@gleanwork/mcp-server-tester 1.0.1 → 1.1.0

package/dist/index-BcUWzQCx.d.ts

@@ -0,0 +1,4282 @@
+import { ZodType, z } from 'zod';
+import { Page, TestInfo, Expect } from '@playwright/test';
+import { OAuthClientProvider } from '@modelcontextprotocol/sdk/client/auth.js';
+import { OAuthClientMetadata, OAuthClientInformationFull, OAuthTokens } from '@modelcontextprotocol/sdk/shared/auth.js';
+import * as oauth from 'oauth4webapi';
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { Tool, CallToolResult, Implementation, ServerCapabilities, Resource, Prompt } from '@modelcontextprotocol/sdk/types.js';
+import * as playwright_test from 'playwright/test';
+
+/**
+ * Validator Types
+ *
+ * Core types for the unified assertion architecture.
+ * These types are used by both Playwright matchers and the eval runner.
+ */
+
+/**
+ * Result of a validation operation
+ */
+interface ValidationResult {
+    /** Whether the validation passed */
+    pass: boolean;
+    /** Human-readable message explaining the result */
+    message: string;
+    /** Additional structured details about the validation */
+    details?: Record<string, unknown>;
+    /**
+     * Optional quantitative metrics from the validation.
+     * Populated by validateToolCalls for precision/recall.
+     */
+    metrics?: {
+        precision?: number;
+        recall?: number;
+    };
+}
+/**
+ * Options for text validation
+ */
+interface TextValidatorOptions {
+    /** Whether to perform case-sensitive matching (default: true) */
+    caseSensitive?: boolean;
+}
+/**
+ * Options for response size validation
+ */
+interface SizeValidatorOptions {
+    /** Maximum allowed size in bytes */
+    maxBytes?: number;
+    /** Minimum required size in bytes */
+    minBytes?: number;
+}
+/**
+ * Options for schema validation
+ */
+interface SchemaValidatorOptions {
+    /** Whether to use strict mode (fail on extra properties) */
+    strict?: boolean;
+}
+/**
+ * Options for pattern validation
+ */
+interface PatternValidatorOptions {
+    /** Whether to perform case-sensitive matching (default: true) */
+    caseSensitive?: boolean;
+}
+/**
+ * Built-in snapshot sanitizer names for use with toMatchToolSnapshot.
+ * Pass these values in the sanitizers array to replace non-deterministic
+ * values with stable placeholders before snapshot comparison.
+ *
+ * @example
+ * expect(result).toMatchToolSnapshot('my-snapshot', [
+ *   SnapshotSanitizers.UUID,
+ *   SnapshotSanitizers.ISO_DATE,
+ * ]);
+ */
+declare const SnapshotSanitizers: {
+    /** Replaces Unix timestamps (seconds and milliseconds) with a stable placeholder */
+    readonly TIMESTAMP: "timestamp";
+    /** Replaces UUID v1-v5 strings with a stable placeholder */
+    readonly UUID: "uuid";
+    /** Replaces ISO 8601 date/datetime strings with a stable placeholder */
+    readonly ISO_DATE: "iso-date";
+    /** Replaces MongoDB ObjectId strings with a stable placeholder */
+    readonly OBJECT_ID: "objectId";
+    /** Replaces JWT tokens with a stable placeholder */
+    readonly JWT: "jwt";
+};
+/**
+ * Built-in sanitizer names for common variable patterns
+ */
+type BuiltInSanitizer = (typeof SnapshotSanitizers)[keyof typeof SnapshotSanitizers];
+/**
+ * Custom regex-based sanitizer
+ */
+interface RegexSanitizer {
+    /** Regex pattern to match */
+    pattern: string | RegExp;
+    /** Replacement string (default: "[SANITIZED]") */
+    replacement?: string;
+}
+/**
+ * Field removal sanitizer - removes specified fields from objects
+ */
+interface FieldRemovalSanitizer {
+    /** Field paths to remove (supports dot notation for nested fields) */
+    remove: string[];
+}
+/**
+ * Snapshot sanitizer configuration
+ *
+ * Sanitizers transform response data before snapshot comparison,
+ * allowing variable content (timestamps, IDs, etc.) to be normalized.
+ *
+ * Can be:
+ * - A built-in sanitizer name: 'timestamp', 'uuid', 'iso-date', 'objectId', 'jwt'
+ * - A regex sanitizer: { pattern: /regex/, replacement: '[REPLACED]' }
+ * - A field removal sanitizer: { remove: ['field1', 'nested.field'] }
+ */
+type SnapshotSanitizer = BuiltInSanitizer | RegexSanitizer | FieldRemovalSanitizer;
+/**
+ * Schema registry for named schemas in datasets
+ */
+type SchemaRegistry = Record<string, ZodType>;
+
+/**
+ * Tool call validators for mcp_host simulation results.
+ *
+ * These validators extract the tool call trace from an MCPHostSimulationResult
+ * and apply assertions against expected call lists and counts.
+ */
+
+interface ToolCallExpectation {
+    calls: Array<{
+        name: string;
+        arguments?: Record<string, unknown>;
+        required?: boolean;
+    }>;
+    order?: 'strict' | 'any';
+    exclusive?: boolean;
+}
+interface ToolCallCountOptions {
+    min?: number;
+    max?: number;
+    exact?: number;
+}
+/**
+ * Validates tool calls made during an MCP host simulation.
+ *
+ * @param response - Must be an MCPHostSimulationResult (from mcp_host mode)
+ * @param expectation - Expected tool call specification
+ */
+declare function validateToolCalls(response: unknown, expectation: ToolCallExpectation): ValidationResult;
+/**
+ * Validates the number of tool calls made during an MCP host simulation.
+ *
+ * @param response - Must be an MCPHostSimulationResult (from mcp_host mode)
+ * @param options - Count constraints (min, max, exact)
+ */
+declare function validateToolCallCount(response: unknown, options: ToolCallCountOptions): ValidationResult;
+
+/**
+ * Built-in judge rubrics matching Glean EvalV2's named judge types.
+ * Use these for consistent, standardized evaluations across teams.
+ *
+ * All built-in rubrics use a 5-point scale: 0.0 / 0.25 / 0.5 / 0.75 / 1.0
+ */
+type BuiltInRubric = 'correctness' | 'completeness' | 'groundedness' | 'instruction-following' | 'conciseness';
+declare const BUILT_IN_RUBRICS: Record<BuiltInRubric, string>;
+/** A rubric specification: either a built-in named rubric or custom text. */
+type RubricSpec = BuiltInRubric | {
+    text: string;
+};
+/**
+ * Returns true if `s` is a built-in rubric name.
+ */
+declare function isBuiltInRubric(s: unknown): s is BuiltInRubric;
+/**
+ * Resolves a RubricSpec to its full rubric text.
+ * - Built-in name → returns the expanded rubric text from BUILT_IN_RUBRICS
+ * - Custom object → returns rubric.text as-is
+ */
+declare function resolveRubric(rubric: RubricSpec): string;
+
+/**
+ * Usage metrics from Claude Agent SDK response
+ */
+interface UsageMetrics {
+    /**
+     * Number of input tokens consumed
+     */
+    inputTokens: number;
+    /**
+     * Number of output tokens generated
+     */
+    outputTokens: number;
+    /**
+     * Total cost in USD
+     */
+    totalCostUsd: number;
+    /**
+     * Execution duration in milliseconds
+     */
+    durationMs: number;
+    /**
+     * API call duration in milliseconds (excluding network overhead)
+     */
+    durationApiMs?: number;
+    /**
+     * Number of tokens read from cache
+     */
+    cacheReadInputTokens?: number;
+    /**
+     * Number of tokens written to cache
+     */
+    cacheCreationInputTokens?: number;
+}
+/** Valid LLM judge provider kinds. */
+type ProviderKind = 'anthropic' | 'vertex-anthropic' | 'anthropic-agent-sdk' | 'openai' | 'google';
+/**
+ * Configuration for an LLM judge
+ */
+interface JudgeConfig {
+    /**
+     * LLM provider to use
+     * @default 'anthropic'
+     */
+    provider?: ProviderKind;
+    /**
+     * Environment variable name containing the API key
+     * @default 'ANTHROPIC_API_KEY'
+     */
+    apiKeyEnvVar?: string;
+    /**
+     * Model to use for judging
+     * @default 'claude-sonnet-4-20250514'
+     */
+    model?: string;
+    /**
+     * Maximum tokens for response
+     * @default 1000
+     */
+    maxTokens?: number;
+    /**
+     * Temperature (0-1, lower is more deterministic)
+     * @default 0.0
+     */
+    temperature?: number;
+    /**
+     * Maximum budget in USD for the judge evaluation
+     * @default 0.10
+     */
+    maxBudgetUsd?: number;
+    /**
+     * Maximum size (in bytes) for tool output before failing the test
+     * When set, the judge will fail if the candidate response exceeds this size
+     */
+    maxToolOutputSize?: number;
+}
+/**
+ * Result from LLM judge evaluation
+ */
+interface JudgeResult {
+    /**
+     * Whether the evaluation passed
+     */
+    pass: boolean;
+    /**
+     * Numeric score (0-1, where 1 is best)
+     */
+    score?: number;
+    /**
+     * Reasoning/explanation from the judge
+     */
+    reasoning?: string;
+    /**
+     * Usage metrics from the Claude Agent SDK
+     */
+    usage?: UsageMetrics;
+    /**
+     * Size of the candidate response in bytes (for maxToolOutputSize tracking)
+     */
+    candidateSizeBytes?: number;
+    /**
+     * Whether the candidate exceeded maxToolOutputSize
+     */
+    exceedsMaxToolOutputSize?: boolean;
+    /**
+     * Standard deviation of individual rep scores.
+     * Only populated when the judge was run with reps > 1.
+     */
+    scoreStdDev?: number;
+    /**
+     * True when the standard deviation across reps exceeds 0.2, indicating
+     * that the rubric may be ambiguous or the judge is non-deterministic.
+     * Only populated when the judge was run with reps > 1.
+     */
+    highVariance?: boolean;
+    /**
+     * Individual scores from each judge rep.
+     * Only populated when the judge was run with reps > 1.
+     */
+    scores?: number[];
+}
+
+/**
+ * LLM judge client interface
+ */
+interface Judge {
+    /**
+     * Evaluates a candidate response against a reference
+     *
+     * @param candidate - The actual response to evaluate
+     * @param reference - The expected/reference response (or null if not applicable)
+     * @param rubric - The evaluation rubric/criteria
+     * @returns Evaluation result with usage metrics
+     */
+    evaluate(candidate: unknown, reference: unknown, rubric: string): Promise<JudgeResult>;
+}
+
+/**
+ * Judge Validator
+ *
+ * Validates a response using an LLM-as-a-judge evaluation.
+ */
+
+/**
+ * Configuration for the judge validator
+ */
+interface JudgeValidatorConfig {
+    /**
+     * The evaluation rubric: a built-in name or custom { text: string }.
+     * Required when no named `judge` is specified.
+     */
+    rubric?: RubricSpec;
+    /** Optional reference response to compare against */
+    reference?: unknown;
+    /** Minimum score required to pass (0-1, default: 0.7) */
+    threshold?: number;
+    /** Number of judge evaluations to run. Scores averaged. @default 1 */
+    reps?: number;
+    /** Judge provider. @default 'claude' */
+    provider?: ProviderKind;
+    /** Model override (e.g., 'claude-opus-4-20250514') */
+    model?: string;
+    /** Environment variable name for API key */
+    apiKeyEnvVar?: string;
+    /** Max tokens for judge response */
+    maxTokens?: number;
+    /** Temperature for judge LLM (0–1) */
+    temperature?: number;
+    /** Max budget in USD per evaluation */
+    maxBudgetUsd?: number;
+    /** Fail if response exceeds this size in bytes before judging */
+    maxToolOutputSize?: number;
+    /**
+     * Name of a registered custom judge executor.
+     * When set, the named judge handles the entire evaluation pipeline
+     * and returns a normalized score. The `threshold` determines pass/fail.
+     * Register judges with `registerJudge()` before tests run.
+     */
+    judge?: string;
+}
+declare function validateJudge(response: unknown, config: JudgeValidatorConfig): Promise<ValidationResult>;
+
+/**
+ * Matcher Types
+ *
+ * TypeScript declarations for custom Playwright matchers.
+ */
+
+/**
+ * Options for the LLM judge matcher
+ */
+interface JudgeMatcherOptions {
+    /** Reference response to compare against */
+    reference?: unknown;
+    /** Score threshold for passing (default: 0.7) */
+    passingThreshold?: number;
+    /** Number of judge evaluations (scores averaged) */
+    reps?: number;
+    /** Override the judge provider */
+    provider?: ProviderKind;
+    /** Override the judge model */
+    model?: string;
+    /**
+     * Name of a registered custom judge executor.
+     * When set, the named judge handles the entire evaluation pipeline
+     * and its `pass` result is authoritative.
+     */
+    judge?: string;
+}
+/**
+ * Declaration merging for Playwright matchers
+ */
+declare global {
+    namespace PlaywrightTest {
+        interface Matchers<R, T = unknown> {
+            /**
+             * Validates that a response exactly matches the expected value
+             *
+             * @param expected - The expected response value
+             *
+             * @example
+             * ```typescript
+             * expect(result).toMatchToolResponse({ status: 'ok', count: 42 });
+             * ```
+             */
+            toMatchToolResponse(expected: unknown): R;
+            /**
+             * Validates that a response matches a Zod schema
+             *
+             * @param schema - The Zod schema to validate against
+             * @param options - Validation options
+             *
+             * @example
+             * ```typescript
+             * const WeatherSchema = z.object({
+             *   temperature: z.number(),
+             *   conditions: z.string(),
+             * });
+             * expect(result).toMatchToolSchema(WeatherSchema);
+             * ```
+             */
+            toMatchToolSchema(schema: ZodType, options?: SchemaValidatorOptions): R;
+            /**
+             * Validates that a response contains expected text substrings
+             *
+             * @param expected - Expected substring(s) to find
+             * @param options - Validation options
+             *
+             * @example
+             * ```typescript
+             * expect(result).toContainToolText('temperature');
+             * expect(result).toContainToolText(['temperature', 'conditions']);
+             * expect(result).toContainToolText('HELLO', { caseSensitive: false });
+             * ```
+             */
+            toContainToolText(expected: string | string[], options?: TextValidatorOptions): R;
+            /**
+             * Validates that a response matches regex patterns
+             *
+             * @param patterns - Expected pattern(s) to match
+             * @param options - Validation options
+             *
+             * @example
+             * ```typescript
+             * expect(result).toMatchToolPattern(/temperature: \d+/);
+             * expect(result).toMatchToolPattern(['temp: \\d+', 'humidity: \\d+%']);
+             * ```
+             */
+            toMatchToolPattern(patterns: string | RegExp | (string | RegExp)[], options?: PatternValidatorOptions): R;
+            /**
+             * Validates that a response matches a saved snapshot
+             *
+             * @param name - Snapshot name
+             * @param sanitizers - Optional sanitizers for non-deterministic values
+             *
+             * @example
+             * ```typescript
+             * expect(result).toMatchToolSnapshot('weather-response');
+             * expect(result).toMatchToolSnapshot('user-data', [
+             *   { pattern: /\d{4}-\d{2}-\d{2}/, replacement: '[DATE]' },
+             * ]);
+             * ```
+             */
+            toMatchToolSnapshot(name: string, sanitizers?: SnapshotSanitizer[]): Promise<R>;
+            /**
+             * Validates that a response is (or is not) an error
+             *
+             * @param expected - What to expect (true for error, false for success, string for specific message)
+             *
+             * @example
+             * ```typescript
+             * expect(result).toBeToolError();  // Expects any error
+             * expect(result).not.toBeToolError();  // Expects success
+             * expect(result).toBeToolError('File not found');  // Expects specific error
+             * ```
+             */
+            toBeToolError(expected?: boolean | string | string[]): R;
+            /**
+             * Validates that a response passes LLM-as-judge evaluation.
+             *
+             * Two call signatures:
+             * - With rubric: `toPassToolJudge(rubric, options?)` — built-in LLM judge
+             * - With named judge: `toPassToolJudge({ judge: 'name' })` — custom judge executor
+             *
+             * @example
+             * ```typescript
+             * // Built-in LLM judge with rubric
+             * expect(result).toPassToolJudge('Response should be helpful and accurate');
+             * expect(result).toPassToolJudge('correctness', {
+             *   reference: expectedOutput,
+             *   passingThreshold: 0.8,
+             * });
+             *
+             * // Named custom judge (registered via registerJudge)
+             * expect(result).toPassToolJudge({ judge: 'glean-completeness' });
+             * ```
+             */
+            toPassToolJudge(rubric: RubricSpec, options?: JudgeMatcherOptions): Promise<R>;
+            toPassToolJudge(options: JudgeMatcherOptions): Promise<R>;
+            toPassToolJudge(judges: Array<JudgeMatcherOptions & {
+                rubric?: RubricSpec;
+            }>): Promise<R>;
+            /**
+             * Validates that a response meets size constraints
+             *
+             * @param options - Size constraints (maxBytes, minBytes)
+             *
+             * @example
+             * ```typescript
+             * expect(result).toHaveToolResponseSize({ maxBytes: 10000 });
+             * expect(result).toHaveToolResponseSize({ minBytes: 100, maxBytes: 50000 });
+             * ```
+             */
+            toHaveToolResponseSize(options: SizeValidatorOptions): R;
+            /**
+             * Validates that a response satisfies a custom predicate function
+             *
+             * Use this as an escape hatch when built-in matchers don't cover your use case.
+             * The predicate receives both the raw response and extracted text for convenience.
+             *
+             * @param predicate - Function that validates the response
+             * @param description - Optional description for error messages
+             *
+             * @example
+             * ```typescript
+             * // Simple boolean predicate
+             * expect(result).toSatisfyToolPredicate((response) => {
+             *   return response.data?.items?.length > 0;
+             * });
+             *
+             * // Predicate with custom message
+             * expect(result).toSatisfyToolPredicate(
+             *   (response, text) => ({
+             *     pass: text.includes('success'),
+             *     message: 'Expected response to contain "success"',
+             *   }),
+             *   'success check'
+             * );
+             *
+             * // Async predicate
+             * expect(result).toSatisfyToolPredicate(async (response) => {
+             *   return await validateWithExternalService(response);
+             * });
+             * ```
+             */
+            toSatisfyToolPredicate(predicate: ToolPredicate, description?: string): Promise<R>;
+            /**
+             * Validates which tools the LLM called during a mcp_host simulation.
+             *
+             * @example
+             * ```typescript
+             * expect(simulationResult).toHaveToolCalls({
+             *   calls: [{ name: 'search', arguments: { query: 'hello' }, required: true }],
+             *   order: 'any',
+             * });
+             * ```
+             */
+            toHaveToolCalls(expectation: ToolCallExpectation): R;
+            /**
+             * Validates the number of tool calls made during a mcp_host simulation.
+             *
+             * @example
+             * ```typescript
+             * expect(simulationResult).toHaveToolCallCount({ min: 1, max: 3 });
+             * expect(simulationResult).toHaveToolCallCount({ exact: 2 });
+             * ```
+             */
+            toHaveToolCallCount(options: ToolCallCountOptions): R;
+        }
+    }
+}
+/**
+ * Predicate result returned by the user's predicate function
+ */
+interface PredicateResult {
+    /** Whether the predicate passed */
+    pass: boolean;
+    /** Message explaining the result (shown on failure) */
+    message?: string;
+}
+/**
+ * A predicate function that validates a response
+ */
+type ToolPredicate = (response: unknown, text: string) => boolean | PredicateResult | Promise<boolean | PredicateResult>;
+
+/**
+ * OAuth configuration for MCP authentication
+ */
+interface MCPOAuthConfig {
+    /**
+     * OAuth authorization server metadata URL
+     * (e.g., https://auth.example.com/.well-known/oauth-authorization-server)
+     */
+    serverUrl: string;
+    /**
+     * Scopes to request during authorization
+     */
+    scopes?: Array<string>;
+    /**
+     * Resource indicator (RFC 8707, required by MCP 2025-06-18 spec)
+     */
+    resource?: string;
+    /**
+     * Path to Playwright auth state file
+     * (e.g., playwright/.auth/oauth-state.json)
+     */
+    authStatePath?: string;
+    /**
+     * Client ID (if pre-registered; otherwise uses Dynamic Client Registration)
+     */
+    clientId?: string;
+    /**
+     * Client secret (for confidential clients)
+     */
+    clientSecret?: string;
+    /**
+     * Redirect URI for OAuth callback
+     */
+    redirectUri?: string;
+}
+/**
+ * OAuth 2.1 client credentials configuration for machine-to-machine (CI/CD) authentication.
+ * Credentials can be provided here or via MCP_CLIENT_ID/MCP_CLIENT_SECRET environment variables.
+ */
+interface MCPClientCredentialsConfig {
+    /**
+     * OAuth client ID (falls back to MCP_CLIENT_ID env var)
+     */
+    clientId?: string;
+    /**
+     * OAuth client secret (falls back to MCP_CLIENT_SECRET env var)
+     */
+    clientSecret?: string;
+    /**
+     * Token endpoint URL (required)
+     */
+    tokenEndpoint?: string;
+    /**
+     * Scopes to request
+     */
+    scopes?: string[];
+}
+/**
+ * Authentication configuration for MCP connections
+ */
+interface MCPAuthConfig {
+    /**
+     * Pre-acquired access token (simplest authentication mode)
+     */
+    accessToken?: string;
+    /**
+     * Full OAuth configuration for browser-based authentication
+     */
+    oauth?: MCPOAuthConfig;
+    /**
+     * OAuth 2.1 client credentials grant for machine-to-machine authentication
+     */
+    clientCredentials?: MCPClientCredentialsConfig;
+}
+/**
+ * MCP host capabilities that can be registered with the server
+ */
+interface MCPHostCapabilities {
+    /**
+     * Sampling capabilities (for LLM sampling)
+     */
+    sampling?: Record<string, unknown>;
+    /**
+     * Roots capabilities (for file system roots)
+     */
+    roots?: {
+        /**
+         * Whether the client can notify the server when roots change
+         */
+        listChanged: boolean;
+    };
+}
+/**
+ * Configuration for MCP client connection via stdio transport (local process)
+ */
+interface StdioMCPConfig {
+    /**
+     * Transport type discriminant
+     */
+    transport: 'stdio';
+    /**
+     * Command to execute (required for stdio transport)
+     */
+    command: string;
+    /**
+     * Command arguments
+     */
+    args?: Array<string>;
+    /**
+     * Working directory for the command
+     */
+    cwd?: string;
+    /**
+     * Environment variables to pass to the subprocess.
+     * Merged with the current process environment.
+     */
+    env?: Record<string, string>;
+    /**
+     * Suppress stderr output from the server process.
+     * When true, server stderr is ignored instead of inherited.
+     */
+    quiet?: boolean;
+    /**
+     * Host capabilities to register with the server
+     */
+    capabilities?: MCPHostCapabilities;
+    /**
+     * Connection timeout in milliseconds
+     */
+    connectTimeoutMs?: number;
+    /**
+     * Request timeout in milliseconds
+     */
+    requestTimeoutMs?: number;
+    /**
+     * Timeout in milliseconds for MCP tool/list operations. Default: 30000
+     */
+    callTimeoutMs?: number;
+}
+/**
+ * Configuration for MCP client connection via HTTP transport (remote server)
+ */
+interface HttpMCPConfig {
+    /**
+     * Transport type discriminant
+     */
+    transport: 'http';
+    /**
+     * Server URL (required for http transport)
+     */
+    serverUrl: string;
+    /**
+     * HTTP headers (e.g., Authorization)
+     */
+    headers?: Record<string, string>;
+    /**
+     * Authentication configuration
+     */
+    auth?: MCPAuthConfig;
+    /**
+     * Host capabilities to register with the server
+     */
+    capabilities?: MCPHostCapabilities;
+    /**
+     * Connection timeout in milliseconds
+     */
+    connectTimeoutMs?: number;
+    /**
+     * Request timeout in milliseconds
+     */
+    requestTimeoutMs?: number;
+    /**
+     * Timeout in milliseconds for MCP tool/list operations. Default: 30000
+     */
+    callTimeoutMs?: number;
+    /**
+     * HTTP proxy configuration. Falls back to HTTPS_PROXY/HTTP_PROXY environment variables.
+     */
+    proxy?: {
+        /**
+         * Proxy URL. Credentials can be embedded directly if required:
+         * `http://user:pass@proxy.example.com:8080`
+         */
+        url: string;
+    };
+    /**
+     * Number of retry attempts for transient connection failures and 429 rate limit responses.
+     * Uses exponential backoff with Retry-After header awareness. Defaults to 0 (no retries).
+     */
+    retryAttempts?: number;
+    /**
+     * TLS/mTLS configuration for custom certificates or disabling cert validation.
+     * File paths should point to PEM-encoded certificate files.
+     */
+    tls?: {
+        /**
+         * Path to CA certificate PEM file (for custom/self-signed CAs)
+         */
+        ca?: string;
+        /**
+         * Path to client certificate PEM file (for mutual TLS)
+         */
+        cert?: string;
+        /**
+         * Path to client private key PEM file (for mutual TLS)
+         */
+        key?: string;
+        /**
+         * Whether to reject unauthorized certificates. Defaults to true.
+         * Set to false to disable certificate validation (not recommended for production).
+         */
+        rejectUnauthorized?: boolean;
+    };
+}
+/**
+ * Configuration for MCP client connection.
+ *
+ * This is a discriminated union — narrow with `isStdioConfig()` or `isHttpConfig()`
+ * before accessing transport-specific fields.
+ *
+ * Supports both stdio (local) and HTTP (remote) transports.
+ */
+type MCPConfig = StdioMCPConfig | HttpMCPConfig;
+/**
+ * Union schema for MCPConfig (validates based on transport type)
+ */
+declare const MCPConfigSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    transport: z.ZodLiteral<"stdio">;
+    command: z.ZodString;
+    args: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    cwd: z.ZodOptional<z.ZodString>;
+    env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    capabilities: z.ZodOptional<z.ZodObject<{
+        sampling: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        roots: z.ZodOptional<z.ZodObject<{
+            listChanged: z.ZodBoolean;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    connectTimeoutMs: z.ZodOptional<z.ZodNumber>;
+    requestTimeoutMs: z.ZodOptional<z.ZodNumber>;
+    callTimeoutMs: z.ZodOptional<z.ZodNumber>;
+    quiet: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strip>, z.ZodObject<{
+    transport: z.ZodLiteral<"http">;
+    serverUrl: z.ZodString;
+    headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    capabilities: z.ZodOptional<z.ZodObject<{
+        sampling: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        roots: z.ZodOptional<z.ZodObject<{
+            listChanged: z.ZodBoolean;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    connectTimeoutMs: z.ZodOptional<z.ZodNumber>;
+    requestTimeoutMs: z.ZodOptional<z.ZodNumber>;
+    callTimeoutMs: z.ZodOptional<z.ZodNumber>;
+    auth: z.ZodOptional<z.ZodObject<{
+        accessToken: z.ZodOptional<z.ZodString>;
+        oauth: z.ZodOptional<z.ZodObject<{
+            serverUrl: z.ZodString;
+            scopes: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            resource: z.ZodOptional<z.ZodString>;
+            authStatePath: z.ZodOptional<z.ZodString>;
+            clientId: z.ZodOptional<z.ZodString>;
+            clientSecret: z.ZodOptional<z.ZodString>;
+            redirectUri: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>;
+        clientCredentials: z.ZodOptional<z.ZodObject<{
+            clientId: z.ZodOptional<z.ZodString>;
+            clientSecret: z.ZodOptional<z.ZodString>;
+            tokenEndpoint: z.ZodOptional<z.ZodString>;
+            scopes: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    proxy: z.ZodOptional<z.ZodObject<{
+        url: z.ZodString;
+    }, z.core.$strip>>;
+    retryAttempts: z.ZodOptional<z.ZodNumber>;
+    tls: z.ZodOptional<z.ZodObject<{
+        ca: z.ZodOptional<z.ZodString>;
+        cert: z.ZodOptional<z.ZodString>;
+        key: z.ZodOptional<z.ZodString>;
+        rejectUnauthorized: z.ZodOptional<z.ZodBoolean>;
+    }, z.core.$strip>>;
+}, z.core.$strip>], "transport">;
+/**
+ * Validates an MCPConfig object
+ *
+ * @param config - The config to validate
+ * @returns The validated config
+ * @throws {z.ZodError} If validation fails
+ */
+declare function validateMCPConfig(config: unknown): MCPConfig;
+/**
+ * Type guard to check if a config is for stdio transport
+ */
+declare function isStdioConfig(config: MCPConfig): config is StdioMCPConfig;
+/**
+ * Type guard to check if a config is for HTTP transport
+ */
+declare function isHttpConfig(config: MCPConfig): config is HttpMCPConfig;
+
+/**
+ * Auth types for MCP OAuth integration
+ */
+
+/**
+ * Stored OAuth tokens
+ */
+interface StoredTokens {
+    /**
+     * OAuth access token
+     */
+    accessToken: string;
+    /**
+     * OAuth refresh token (if provided)
+     */
+    refreshToken?: string;
+    /**
+     * Token expiration timestamp (Unix milliseconds)
+     */
+    expiresAt?: number;
+    /**
+     * Token type (typically "Bearer")
+     */
+    tokenType: string;
+    /**
+     * Client ID that was used to obtain these tokens.
+     * Required for token refresh since refresh tokens are bound to the client.
+     */
+    clientId?: string;
+}
+/**
+ * Stored client information from Dynamic Client Registration
+ */
+interface StoredClientInfo {
+    /**
+     * Client ID from DCR
+     */
+    clientId: string;
+    /**
+     * Client secret from DCR (for confidential clients)
+     */
+    clientSecret?: string;
+    /**
+     * Client ID issued at timestamp
+     */
+    clientIdIssuedAt?: number;
+    /**
+     * Client secret expiration timestamp
+     */
+    clientSecretExpiresAt?: number;
+}
+/**
+ * Complete OAuth state persisted to disk for Playwright auth state pattern
+ */
+interface StoredOAuthState {
+    /**
+     * OAuth tokens
+     */
+    tokens?: StoredTokens;
+    /**
+     * DCR client information
+     */
+    clientInfo?: StoredClientInfo;
+    /**
+     * PKCE code verifier (used during authorization flow)
+     */
+    codeVerifier?: string;
+    /**
+     * OAuth state parameter (for CSRF protection)
+     */
+    state?: string;
+    /**
+     * Timestamp when this state was saved
+     */
+    savedAt: number;
+}
+/**
+ * Login form selectors for standard OAuth login automation
+ */
+interface OAuthLoginSelectors {
+    /** Selector for username/email input field */
+    usernameInput: string;
+    /** Selector for password input field */
+    passwordInput: string;
+    /** Selector for login submit button */
+    submitButton: string;
+    /** Selector for consent/authorize button (optional) */
+    consentButton?: string;
+}
+/**
+ * Base configuration shared by all OAuth setup strategies
+ */
+interface OAuthSetupBaseConfig {
+    /** OAuth authorization server metadata URL */
+    authServerUrl: string;
+    /** Scopes to request */
+    scopes: Array<string>;
+    /** Path to save OAuth state file */
+    outputPath: string;
+    /** Pre-registered client ID (optional, uses DCR if not provided) */
+    clientId?: string;
+    /** Pre-registered client secret (optional) */
+    clientSecret?: string;
+    /** Redirect URI for OAuth callback */
+    redirectUri?: string;
+    /** Resource indicator (RFC 8707) */
+    resource?: string;
+    /** Timeout for login flow in milliseconds (default: 30000) */
+    timeoutMs?: number;
+}
+/**
+ * Standard login strategy: automates a form with username, password, and submit button.
+ * Use when the IdP presents all login fields on a single page.
+ */
+interface StandardLoginConfig {
+    /** Login form selectors for Playwright automation */
+    loginSelectors: OAuthLoginSelectors;
+    /** Test user credentials */
+    credentials: {
+        username: string;
+        password: string;
+    };
+    customLoginFlow?: never;
+}
+/**
+ * Custom login strategy: full control over the browser-based login flow.
+ * Use for multi-step logins, MFA, custom consent screens, or any flow
+ * that doesn't fit the standard username/password/submit pattern.
+ *
+ * The callback receives a Playwright Page already navigated to the OAuth
+ * authorization URL. Complete the login so the IdP redirects to the
+ * callback URL — `performOAuthSetup` handles PKCE, token exchange,
+ * and state persistence automatically.
+ */
+interface CustomLoginConfig {
+    /**
+     * Custom Playwright automation for the IdP login flow.
+     *
+     * @param page - Playwright Page already navigated to the OAuth authorization URL
+     *
+     * @example
+     * ```typescript
+     * customLoginFlow: async (page) => {
+     *   await page.fill('#username', process.env.TEST_USER!);
+     *   await page.click('#continue');
+     *   await page.fill('#password', process.env.TEST_PASS!);
+     *   await page.click('#submit');
+     * }
+     * ```
+     */
+    customLoginFlow: (page: Page) => Promise<void>;
+    loginSelectors?: never;
+    credentials?: never;
+}
+/**
+ * Configuration for OAuth setup flow.
+ *
+ * Provide either `loginSelectors` + `credentials` for standard form-based login,
+ * or `customLoginFlow` for full control over the browser automation.
+ */
+type OAuthSetupConfig = OAuthSetupBaseConfig & (StandardLoginConfig | CustomLoginConfig);
+/**
+ * Result of token exchange or refresh
+ */
+interface TokenResult {
+    /**
+     * Access token
+     */
+    accessToken: string;
+    /**
+     * Token type (typically "Bearer")
+     */
+    tokenType: string;
+    /**
+     * Expires in seconds
+     */
+    expiresIn?: number;
+    /**
+     * Refresh token (if provided)
+     */
+    refreshToken?: string;
+    /**
+     * Granted scopes (space-separated)
+     */
+    scope?: string;
+}
+
+/**
+ * OAuth client provider implementation for MCP SDK
+ *
+ * Implements the MCP SDK's OAuthClientProvider interface using file-based storage
+ * for integration with Playwright's auth state pattern.
+ */
+
+/**
+ * Configuration for the Playwright OAuth client provider
+ */
+interface PlaywrightOAuthClientProviderConfig {
+    /**
+     * Path to the auth state file (e.g., playwright/.auth/oauth-state.json)
+     */
+    storagePath: string;
+    /**
+     * OAuth redirect URI for callback
+     */
+    redirectUri: string;
+    /**
+     * Client metadata for DCR or display
+     */
+    clientMetadata?: Partial<OAuthClientMetadata>;
+    /**
+     * Pre-registered client ID (if not using DCR)
+     */
+    clientId?: string;
+    /**
+     * Pre-registered client secret (if not using DCR)
+     */
+    clientSecret?: string;
+}
+/**
+ * OAuth client provider that implements the MCP SDK's OAuthClientProvider interface
+ *
+ * Uses file-based storage for integration with Playwright's auth state pattern.
+ * Auth state is persisted to disk so it can be reused across test runs.
+ *
+ * @example
+ * ```typescript
+ * const provider = new PlaywrightOAuthClientProvider({
+ *   storagePath: 'playwright/.auth/oauth-state.json',
+ *   redirectUri: 'http://localhost:3000/callback',
+ * });
+ *
+ * const transport = new StreamableHTTPClientTransport(serverUrl, {
+ *   authProvider: provider,
+ * });
+ * ```
+ */
+declare class PlaywrightOAuthClientProvider implements OAuthClientProvider {
+    private readonly config;
+    private cachedState;
+    private stateParam;
+    constructor(config: PlaywrightOAuthClientProviderConfig);
+    /**
+     * The URL to redirect the user agent to after authorization
+     */
+    get redirectUrl(): string;
+    /**
+     * Metadata about this OAuth client
+     */
+    get clientMetadata(): OAuthClientMetadata;
+    /**
+     * Returns an OAuth2 state parameter
+     */
+    state(): string;
+    /**
+     * Loads information about this OAuth client
+     */
+    clientInformation(): Promise<OAuthClientInformationFull | undefined>;
+    /**
+     * Saves client information from Dynamic Client Registration
+     */
+    saveClientInformation(clientInformation: OAuthClientInformationFull): Promise<void>;
+    /**
+     * Loads any existing OAuth tokens for the current session
+     */
+    tokens(): Promise<OAuthTokens | undefined>;
+    /**
+     * Stores new OAuth tokens for the current session
+     *
+     * The code verifier is cleared after a successful token exchange — it is
+     * single-use per PKCE spec and must not persist beyond the exchange.
+     */
+    saveTokens(tokens: OAuthTokens): Promise<void>;
+    /**
+     * Invoked to redirect the user agent to the given URL
+     *
+     * In a testing context, this is typically handled by Playwright automation.
+     * This implementation throws an error to signal that the caller needs to
+     * handle the redirect externally.
+     */
+    redirectToAuthorization(authorizationUrl: URL): Promise<void>;
+    /**
+     * Saves a PKCE code verifier for the current session
+     */
+    saveCodeVerifier(codeVerifier: string): Promise<void>;
+    /**
+     * Loads the PKCE code verifier for the current session
+     */
+    codeVerifier(): Promise<string>;
+    /**
+     * Invalidates the specified credentials
+     */
+    invalidateCredentials(scope: 'all' | 'client' | 'tokens' | 'verifier'): Promise<void>;
+    private loadState;
+    private saveState;
+    private deleteState;
+    private createEmptyState;
+    private generateRandomString;
+}
+
+/**
+ * OAuth flow utilities using oauth4webapi
+ *
+ * Implements OAuth 2.1 with PKCE as required by MCP specification
+ */
+
+/**
+ * Discovered OAuth authorization server metadata
+ */
+interface AuthServerMetadata {
+    /**
+     * The oauth4webapi AuthorizationServer object
+     */
+    server: oauth.AuthorizationServer;
+    /**
+     * Issuer URL
+     */
+    issuer: string;
+}
+/**
+ * Configuration for token refresh
+ */
+interface TokenRefreshConfig {
+    /**
+     * Authorization server metadata
+     */
+    authServer: AuthServerMetadata;
+    /**
+     * Client ID
+     */
+    clientId: string;
+    /**
+     * Client secret (for confidential clients)
+     */
+    clientSecret?: string;
+    /**
+     * Refresh token
+     */
+    refreshToken: string;
+}
+/**
+ * Refreshes an access token using a refresh token
+ *
+ * @param config - Token refresh configuration
+ * @returns New token result
+ */
+declare function refreshAccessToken(config: TokenRefreshConfig): Promise<TokenResult>;
+/**
+ * Configuration for client credentials grant
+ */
+interface ClientCredentialsConfig {
+    /**
+     * Token endpoint URL
+     */
+    tokenEndpoint: string;
+    /**
+     * OAuth client ID
+     */
+    clientId: string;
+    /**
+     * OAuth client secret
+     */
+    clientSecret: string;
+    /**
+     * Scopes to request (optional)
+     */
+    scopes?: string[];
+}
+/**
+ * Performs the OAuth 2.1 client credentials grant to obtain an access token.
+ * Suitable for CI/CD machine-to-machine authentication.
+ *
+ * Uses oauth4webapi for spec-compliant request construction and response validation,
+ * consistent with how the rest of this module handles OAuth flows.
+ *
+ * @param config - Client credentials configuration
+ * @returns Token result
+ */
+declare function performClientCredentialsFlow(config: ClientCredentialsConfig): Promise<TokenResult>;
+
+/**
+ * OAuth Protected Resource and Authorization Server discovery
+ *
+ * Implements RFC 9728 (OAuth Protected Resource Metadata) and
+ * RFC 8414 (Authorization Server Metadata) for MCP servers.
+ */
+
+/**
+ * MCP Protocol version header value
+ */
+declare const MCP_PROTOCOL_VERSION = "2025-06-18";
+/**
+ * Protected Resource Metadata (RFC 9728)
+ */
+interface ProtectedResourceMetadata {
+    /**
+     * The protected resource URL
+     */
+    resource: string;
+    /**
+     * Array of authorization server URLs
+     */
+    authorization_servers?: Array<string>;
+    /**
+     * Scopes supported by the protected resource
+     */
+    scopes_supported?: Array<string>;
+    /**
+     * Bearer token formats supported
+     */
+    bearer_methods_supported?: Array<string>;
+    /**
+     * Resource documentation URL
+     */
+    resource_documentation?: string;
+    /**
+     * Resource signing algorithms
+     */
+    resource_signing_alg_values_supported?: Array<string>;
+}
+/**
+ * Result of protected resource discovery
+ */
+interface ProtectedResourceDiscoveryResult {
+    /**
+     * The discovered metadata
+     */
+    metadata: ProtectedResourceMetadata;
+    /**
+     * The URL where metadata was found
+     */
+    discoveryUrl: string;
+    /**
+     * Whether path-aware discovery was used (vs base discovery)
+     */
+    usedPathAwareDiscovery: boolean;
+}
+/**
+ * Discovers protected resource metadata per RFC 9728
+ *
+ * Follows RFC 9728 Section 4.1 for path-aware discovery:
+ * 1. First tries: {origin}/.well-known/oauth-protected-resource{pathname}
+ * 2. Falls back to: {origin}/.well-known/oauth-protected-resource
+ *
+ * @param mcpServerUrl - The MCP server URL
+ * @returns Protected resource discovery result
+ * @throws Error if discovery fails completely
+ *
+ * @example
+ * const result = await discoverProtectedResource('https://api.example.com/mcp/default');
+ * console.log(result.metadata.authorization_servers);
+ */
+declare function discoverProtectedResource(mcpServerUrl: string): Promise<ProtectedResourceDiscoveryResult>;
+/**
+ * Error thrown when discovery fails
+ */
+declare class DiscoveryError extends Error {
+    readonly status?: number | undefined;
+    readonly url?: string | undefined;
+    constructor(message: string, status?: number | undefined, url?: string | undefined);
+}
+/**
+ * Discovers OAuth Authorization Server metadata per RFC 8414
+ *
+ * Wraps oauth4webapi's discovery with MCP-specific headers.
+ *
+ * @param authServerUrl - The authorization server URL
+ * @returns Authorization server metadata
+ * @throws Error if discovery fails
+ *
+ * @example
+ * const authServer = await discoverAuthorizationServer('https://auth.example.com');
+ * console.log(authServer.server.token_endpoint);
+ */
+declare function discoverAuthorizationServer(authServerUrl: string): Promise<AuthServerMetadata>;
+
+/**
+ * OAuth token storage with environment variable support for CI/CD
+ *
+ * Provides file-based storage for OAuth state per MCP server, with support
+ * for token injection via environment variables for automated testing.
+ */
+
+/**
+ * Combined server metadata (auth server + protected resource)
+ */
+interface StoredServerMetadata {
+    /**
+     * Authorization server metadata
+     */
+    authServer: AuthServerMetadata;
+    /**
+     * Protected resource metadata
+     */
+    protectedResource: ProtectedResourceMetadata;
+    /**
+     * Timestamp when metadata was discovered
+     */
+    discoveredAt: number;
+}
+/**
+ * Environment variable names for CI/CD token injection
+ */
+declare const ENV_VAR_NAMES: {
+    readonly accessToken: "MCP_ACCESS_TOKEN";
+    readonly refreshToken: "MCP_REFRESH_TOKEN";
+    readonly tokenType: "MCP_TOKEN_TYPE";
+    readonly expiresAt: "MCP_TOKEN_EXPIRES_AT";
+};
+/**
+ * Reads tokens from environment variables (for CI/CD)
+ *
+ * @returns StoredTokens if MCP_ACCESS_TOKEN is set, null otherwise
+ */
+declare function loadTokensFromEnv(): StoredTokens | null;
+/**
+ * Programmatically inject tokens into storage (for CI/CD setup)
+ *
+ * @param serverUrl - The MCP server URL
+ * @param tokens - The tokens to inject
+ * @param stateDir - Optional custom state directory
+ */
+declare function injectTokens(serverUrl: string, tokens: StoredTokens, stateDir?: string): Promise<void>;
+/**
+ * Load stored OAuth tokens for an MCP server
+ *
+ * Reads tokens from the standard storage location for the given server URL.
+ * Tokens are stored by `mcp-server-tester login` or `injectTokens()`.
+ *
+ * @param serverUrl - The MCP server URL
+ * @param stateDir - Optional custom state directory
+ * @returns StoredTokens if found, null otherwise
+ *
+ * @example
+ * ```typescript
+ * // After running: npx mcp-server-tester login https://api.example.com/mcp
+ * const tokens = await loadTokens('https://api.example.com/mcp');
+ * if (tokens?.accessToken) {
+ *   // Use the token — never log raw token values
+ *   headers.Authorization = `Bearer ${tokens.accessToken}`;
+ * }
+ * ```
+ */
+declare function loadTokens(serverUrl: string, stateDir?: string): Promise<StoredTokens | null>;
+/**
+ * Check if valid OAuth tokens exist for an MCP server
+ *
+ * Returns true if tokens exist and are not expired (with buffer).
+ * Use this to check if authentication is needed before making requests.
+ *
+ * @param serverUrl - The MCP server URL
+ * @param options - Optional configuration
+ * @param options.stateDir - Custom state directory
+ * @param options.bufferMs - Buffer time before expiration (default: 60000ms)
+ * @returns true if valid (non-expired) tokens exist
+ *
+ * @example
+ * ```typescript
+ * if (await hasValidTokens('https://api.example.com/mcp')) {
+ *   // Use stored tokens
+ *   const tokens = await loadTokens('https://api.example.com/mcp');
+ * } else {
+ *   console.log('Run: npx mcp-server-tester login https://api.example.com/mcp');
+ * }
+ * ```
+ */
+declare function hasValidTokens(serverUrl: string, options?: {
+    stateDir?: string;
+    bufferMs?: number;
+}): Promise<boolean>;
+
+/**
+ * CLI OAuth client for command-line authentication flows
+ *
+ * Provides browser-based OAuth authentication for CLI environments,
+ * with support for environment variable token injection for CI/CD.
+ */
+/**
+ * Configuration for CLI OAuth client
+ */
+interface CLIOAuthClientConfig {
+    /**
+     * MCP server URL (for protected resource discovery)
+     */
+    mcpServerUrl: string;
+    /**
+     * Scopes to request (optional, uses discovered scopes if not provided)
+     */
+    scopes?: Array<string>;
+    /**
+     * Custom storage directory
+     */
+    stateDir?: string;
+    /**
+     * Pre-registered client ID (skips DCR if provided)
+     */
+    clientId?: string;
+    /**
+     * Pre-registered client secret
+     */
+    clientSecret?: string;
+    /**
+     * Preferred callback port (default: random available port)
+     */
+    callbackPort?: number;
+    /**
+     * Timeout for OAuth flow in milliseconds (default: 300000 = 5 min)
+     */
+    timeoutMs?: number;
+    /**
+     * Client name for DCR registration
+     */
+    clientName?: string;
+}
+/**
+ * Result of CLI OAuth authentication
+ */
+interface CLIOAuthResult {
+    /**
+     * Access token
+     */
+    accessToken: string;
+    /**
+     * Token type (typically "Bearer")
+     */
+    tokenType: string;
+    /**
+     * Expiration timestamp (Unix ms)
+     */
+    expiresAt?: number;
+    /**
+     * Whether token was refreshed vs newly acquired
+     */
+    refreshed: boolean;
+    /**
+     * Scopes that were requested (only set for new authentications)
+     */
+    requestedScopes?: string[];
+    /**
+     * Whether token came from environment variables
+     */
+    fromEnv: boolean;
+}
+/**
+ * CLI OAuth client for command-line authentication flows
+ */
+declare class CLIOAuthClient {
+    private readonly config;
+    private readonly storage;
+    constructor(config: CLIOAuthClientConfig);
+    /**
+     * Get a valid access token, authenticating if necessary
+     *
+     * Token resolution priority:
+     * 1. Check environment variables (for CI/CD)
+     * 2. Check file storage for cached tokens
+     * 3. Try to refresh if expired but refresh token exists
+     * 4. Run full OAuth flow if needed
+     */
+    getAccessToken(): Promise<CLIOAuthResult>;
+    /**
+     * Try to get a valid access token without triggering browser auth
+     *
+     * Returns null if no valid token is available (no stored tokens,
+     * expired without refresh token, or refresh failed). Unlike getAccessToken(),
+     * this will NOT open a browser for authentication.
+     *
+     * Use this for CLI commands that should prompt the user to run `login`
+     * instead of automatically starting the OAuth flow.
+     */
+    tryGetAccessToken(): Promise<CLIOAuthResult | null>;
+    /**
+     * Force a new authentication flow
+     */
+    authenticate(): Promise<CLIOAuthResult>;
+    /**
+     * Check if stored credentials exist (may be expired)
+     */
+    hasStoredCredentials(): Promise<boolean>;
+    /**
+     * Clear stored credentials
+     */
+    clearCredentials(): Promise<void>;
+    /**
+     * Discover protected resource and authorization server
+     */
+    private discoverServers;
+    /**
+     * Get existing client or register new one via DCR
+     */
+    private getOrRegisterClient;
+    /**
+     * Register a new client via Dynamic Client Registration
+     */
+    private registerClient;
+    /**
+     * Perform the full OAuth authorization flow
+     */
+    private performOAuthFlow;
+    /**
+     * Refresh an expired token
+     *
+     * Uses the clientId stored with the tokens (if available) to ensure
+     * the refresh request uses the same client that obtained the original tokens.
+     * This is important because refresh tokens are bound to the client_id.
+     */
+    private refreshStoredToken;
+    /**
+     * Start local callback server
+     */
+    private startCallbackServer;
+    /**
+     * Open browser or print URL for headless environments
+     */
+    private openBrowserOrPrintUrl;
+    /**
+     * Convert TokenResult to StoredTokens
+     *
+     * @param result - Token result from exchange or refresh
+     * @param clientId - Client ID that was used to obtain these tokens
+     */
+    private tokenResultToStoredTokens;
+    /**
+     * HTML page for successful authentication
+     */
+    private successHtml;
+    /**
+     * HTML page for authentication error
+     */
+    private errorHtml;
+}
+
+/**
+ * Options for creating an MCP fixture
+ */
+interface MCPFixtureOptions {
+    /**
+     * Authentication type used for this test
+     * - 'oauth': Interactive OAuth 2.1 with PKCE (browser-based authentication)
+     * - 'api-token': Static API token (e.g., from a dashboard or environment variable)
+     * - 'none': No authentication
+     */
+    authType?: AuthType;
+    /**
+     * Playwright project name for this test
+     * Used for filtering and grouping in the reporter
+     */
+    project?: string;
+    /**
+     * Timeout in milliseconds for MCP tool/list operations. Default: 30000
+     */
+    callTimeoutMs?: number;
+}
+/**
+ * High-level API for interacting with MCP servers in tests
+ *
+ * This interface wraps the raw MCP Client with test-friendly methods
+ */
+interface MCPFixtureApi {
+    /**
+     * The underlying MCP client (for advanced usage)
+     */
+    client: Client;
+    /**
+     * Authentication type used for this test session
+     */
+    authType: AuthType;
+    /**
+     * Playwright project name for this test session
+     */
+    project?: string;
+    /**
+     * Lists all available tools from the MCP server
+     *
+     * @returns Array of tool definitions
+     */
+    listTools(): Promise<Array<Tool>>;
+    /**
+     * Calls a tool on the MCP server
+     *
+     * @param name - Tool name
+     * @param args - Tool arguments
+     * @returns Tool call result
+     */
+    callTool<TArgs extends Record<string, unknown> = Record<string, unknown>>(name: string, args: TArgs): Promise<CallToolResult>;
+    /**
+     * Gets information about the connected server
+     */
+    getServerInfo(): {
+        name?: string;
+        version?: string;
+    } | null;
+}
+/**
+ * Creates an MCP fixture wrapper around a Client, providing a high-level
+ * {@link MCPFixtureApi} without requiring Playwright's `test.extend` pattern.
+ *
+ * Use this when you need to set up an MCP fixture manually — for example in
+ * custom fixture hierarchies, non-Playwright test runners (e.g. Vitest,
+ * Jest), or when you want to compose the fixture with other lifecycle
+ * management logic that doesn't fit the standard `test.extend` model.
+ *
+ * For the typical Playwright use case, prefer importing `test` and `mcp`
+ * directly from `@gleanwork/mcp-server-tester/fixtures/mcp`, which wires
+ * this function up automatically.
+ *
+ * When `testInfo` is provided, all MCP operations are automatically wrapped
+ * in `test.step()` calls and attachments are created for the MCP Test
+ * Reporter. Omit `testInfo` for lightweight usage outside Playwright.
+ *
+ * @param client - The MCP client to wrap (created via `createMCPClientForConfig`)
+ * @param testInfo - Optional Playwright TestInfo for auto-tracking and reporter attachments
+ * @param options - Optional fixture options (authType, project)
+ * @returns MCPFixtureApi instance
+ *
+ * @example
+ * ```typescript
+ * // Advanced: custom fixture setup inside test.extend
+ * const test = base.extend<{ mcp: MCPFixtureApi }>({
+ *   mcp: async ({}, use, testInfo) => {
+ *     const client = await createMCPClientForConfig(config);
+ *     const api = createMCPFixture(client, testInfo, { authType: 'api-token' });
+ *     await use(api);
+ *     await closeMCPClient(client);
+ *   }
+ * });
+ *
+ * // Non-Playwright usage (no reporter attachments)
+ * const client = await createMCPClientForConfig(config);
+ * const api = createMCPFixture(client);
+ * const tools = await api.listTools();
+ * ```
+ */
+declare function createMCPFixture(client: Client, testInfo?: TestInfo, options?: MCPFixtureOptions): MCPFixtureApi;
+
+/**
+ * Test-scoped auth fixtures interface
+ */
+interface MCPAuthFixtures {
+    /**
+     * OAuth client provider for MCP authentication
+     */
+    mcpAuthProvider: OAuthClientProvider | undefined;
+}
+/**
+ * Extended Playwright test with MCP auth fixtures
+ *
+ * Use this when you need OAuth authentication for MCP server testing.
+ *
+ * @example
+ * ```typescript
+ * // test.ts
+ * import { test } from '@gleanwork/mcp-server-tester/fixtures/mcpAuth';
+ *
+ * test('authenticated MCP call', async ({ mcpAuthProvider }) => {
+ *   // mcpAuthProvider can be passed to createMCPClientForConfig
+ * });
+ * ```
+ */
+declare const test: playwright_test.TestType<playwright_test.PlaywrightTestArgs & playwright_test.PlaywrightTestOptions & MCPAuthFixtures, playwright_test.PlaywrightWorkerArgs & playwright_test.PlaywrightWorkerOptions>;
+
+/**
+ * Options for creating an MCP client
+ */
+interface CreateMCPClientOptions {
+    /**
+     * Client information (name and version)
+     */
+    clientInfo?: {
+        name?: string;
+        version?: string;
+    };
+    /**
+     * OAuth client provider for authentication
+     *
+     * When provided, the MCP SDK handles OAuth flow automatically.
+     * This takes precedence over static token auth in config.auth.accessToken.
+     */
+    authProvider?: OAuthClientProvider;
+    /**
+     * Sampling handler callback for LLM sampling requests from the server.
+     *
+     * When provided, the client will advertise sampling capability to the server.
+     * When absent, sampling is removed from declared capabilities so the client
+     * does not falsely advertise support it cannot fulfill.
+     */
+    samplingHandler?: (...args: unknown[]) => unknown;
+}
+/**
+ * Creates and connects an MCP client based on the provided configuration
+ *
+ * @param config - MCP configuration (will be validated)
+ * @param options - Optional client options including auth provider
+ * @returns Connected MCP Client instance
+ * @throws {Error} If config is invalid or connection fails
+ *
+ * @example
+ * // Stdio transport
+ * const client = await createMCPClientForConfig({
+ *   transport: 'stdio',
+ *   command: 'node',
+ *   args: ['server.js']
+ * });
+ *
+ * @example
+ * // HTTP transport with static token auth
+ * const client = await createMCPClientForConfig({
+ *   transport: 'http',
+ *   serverUrl: 'http://localhost:3000/mcp',
+ *   auth: { accessToken: 'your-token' }
+ * });
+ *
+ * @example
+ * // HTTP transport with OAuth provider
+ * const client = await createMCPClientForConfig(
+ *   { transport: 'http', serverUrl: 'http://localhost:3000/mcp' },
+ *   { authProvider: myOAuthProvider }
+ * );
+ */
+declare function createMCPClientForConfig(config: MCPConfig, options?: CreateMCPClientOptions): Promise<Client>;
+/**
+ * Safely closes an MCP client connection
+ *
+ * @param client - The client to close
+ */
+declare function closeMCPClient(client: Client): Promise<void>;
+
+/**
+ * A single content block from an MCP response
+ */
+interface ContentBlock {
+    type: string;
+    text?: string;
+    data?: unknown;
+    mimeType?: string;
+}
+/**
+ * Normalized representation of an MCP tool response
+ *
+ * This provides a consistent interface regardless of the response format
+ * returned by the MCP server.
+ */
+interface NormalizedToolResponse {
+    /**
+     * Extracted text content (concatenated from all text blocks)
+     */
+    text: string;
+    /**
+     * Original raw response from the MCP SDK
+     */
+    raw: CallToolResult;
+    /**
+     * Whether the tool call resulted in an error
+     */
+    isError: boolean;
+    /**
+     * Parsed content blocks from the response
+     */
+    contentBlocks: ContentBlock[];
+    /**
+     * Structured content if present (parsed JSON or raw data)
+     */
+    structuredContent: unknown;
+}
+/**
+ * Normalizes an MCP CallToolResult into a consistent format
+ *
+ * @param result - Raw CallToolResult from the MCP SDK
+ * @returns Normalized response with extracted text, content blocks, etc.
+ *
+ * @example
+ * ```typescript
+ * const result = await client.callTool({ name: 'read_file', arguments: { path: 'readme.txt' } });
+ * const normalized = normalizeToolResponse(result);
+ *
+ * console.log(normalized.text);           // "Hello World"
+ * console.log(normalized.isError);        // false
+ * console.log(normalized.contentBlocks);  // [{ type: 'text', text: 'Hello World' }]
+ * ```
+ */
+declare function normalizeToolResponse(result: CallToolResult): NormalizedToolResponse;
+/**
+ * Extracts just the text content from a normalized or raw response
+ *
+ * This is a convenience function that works with both:
+ * - Raw CallToolResult from the MCP SDK
+ * - NormalizedToolResponse from normalizeToolResponse()
+ * - Plain strings or other legacy formats
+ *
+ * @param response - Response in any supported format
+ * @returns Extracted text content
+ */
+declare function extractText(response: unknown): string;
+
+/**
+ * Types and interfaces for MCP host simulation mode
+ *
+ * This module provides types for testing MCP servers through MCP hosts,
+ * validating tool descriptions, parameter clarity, and discoverability.
+ */
+
+/**
+ * Host type for MCP host simulation.
+ *
+ * - 'sdk': Programmatic via Vercel AI SDK (default). The framework's MCP connection is reused.
+ * - 'cli': CLI-based hosts (e.g., Claude Code, Codex). Spawns a process with its own MCP connection.
+ * - 'browser': Web-based hosts (e.g., claude.ai). Uses Playwright/CDP. (Not yet implemented.)
+ * - 'desktop': Desktop app hosts (e.g., Claude Desktop). Uses computer use. (Not yet implemented.)
+ */
+type HostType = 'sdk' | 'cli' | 'browser' | 'desktop';
+/**
+ * LLM provider for SDK-based host simulation.
+ *
+ * Each provider runs through the Vercel AI SDK (`ai` package)
+ * and requires its corresponding @ai-sdk/* package:
+ *
+ *   openai      → npm install ai @ai-sdk/openai
+ *   anthropic   → npm install ai @ai-sdk/anthropic
+ *   google      → npm install ai @ai-sdk/google
+ *   azure       → npm install ai @ai-sdk/azure
+ *   mistral     → npm install ai @ai-sdk/mistral
+ *   deepseek    → npm install ai @ai-sdk/deepseek
+ *   openrouter  → npm install ai @openrouter/ai-sdk-provider
+ *   xai         → npm install ai @ai-sdk/xai
+ */
+type LLMProvider = 'openai' | 'anthropic' | 'azure' | 'google' | 'mistral' | 'deepseek' | 'openrouter' | 'xai'
+/**
+ * Anthropic Claude via Google Vertex AI.
+ * Requires @ai-sdk/google-vertex and Application Default Credentials (gcloud auth).
+ * Set GOOGLE_VERTEX_PROJECT and GOOGLE_VERTEX_LOCATION env vars.
+ * Use this instead of 'anthropic' in environments where api.anthropic.com is blocked.
+ * @example model: 'claude-3-5-haiku@20241022'
+ */
+ | 'vertex-anthropic';
+/**
+ * Output format for CLI host processes.
+ *
+ * - 'stream-json': NDJSON (one JSON object per line). Used by Claude Code (`--output-format stream-json`).
+ * - 'json': Single JSON object on stdout.
+ */
+type CLIOutputFormat = 'stream-json' | 'json';
+/**
+ * Configuration for a CLI host process.
+ *
+ * The process is spawned directly (no shell) with `command` and `args`.
+ * Use `{{scenario}}` in any args entry as a placeholder for the natural
+ * language prompt — the framework replaces it before spawning.
+ *
+ * Because args are passed directly to the process (not through a shell),
+ * special characters in the scenario (quotes, newlines, `$`, etc.) are
+ * handled safely without escaping.
+ *
+ * @example Claude Code
+ * ```json
+ * {
+ *   "command": "claude",
+ *   "args": ["-p", "{{scenario}}", "--output-format", "stream-json",
+ *            "--verbose", "--mcp-config", "{...}"]
+ * }
+ * ```
+ *
+ * @example Custom CLI
+ * ```json
+ * {
+ *   "command": "my-agent",
+ *   "args": ["--prompt", "{{scenario}}", "--config", "./mcp.json"],
+ *   "outputFormat": "json"
+ * }
+ * ```
+ */
+interface CLIConfig {
+    /**
+     * CLI binary to invoke.
+     */
+    command: string;
+    /**
+     * Arguments to pass. Use `{{scenario}}` as a placeholder for the prompt.
+     */
+    args: string[];
+    /**
+     * How to parse stdout.
+     * @default 'stream-json'
+     */
+    outputFormat?: CLIOutputFormat;
+    /**
+     * Timeout in milliseconds.
+     * @default 120000 (2 minutes)
+     */
+    timeout?: number;
+}
+/**
+ * A cookie to inject into the browser context before running the script.
+ * Matches the shape expected by Playwright's `BrowserContext.addCookies()`.
+ */
+interface BrowserCookie {
+    name: string;
+    value: string;
+    url?: string;
+    domain?: string;
+    path?: string;
+    expires?: number;
+    httpOnly?: boolean;
+    secure?: boolean;
+    sameSite?: 'Strict' | 'Lax' | 'None';
+    partitionKey?: string;
+}
+/**
+ * Configuration for a browser-based host.
+ *
+ * Uses Playwright to launch a Chromium instance, inject auth state,
+ * and execute a user-provided script that drives a web-based MCP host
+ * (e.g., claude.ai).
+ */
+interface BrowserConfig {
+    /**
+     * Path to the browser script (resolved relative to cwd).
+     * The script must default-export an async function
+     * `(page: Page, scenario: string) => Promise<MCPHostSimulationResult>`.
+     */
+    script: string;
+    /**
+     * Timeout in milliseconds for the browser script.
+     * @default 120000 (2 minutes)
+     */
+    timeout?: number;
+    /**
+     * Whether to launch in headless mode.
+     * @default true
+     */
+    headless?: boolean;
+    /**
+     * Path to a Playwright storage state JSON file (cookies + localStorage).
+     * Resolved relative to cwd.
+     */
+    storageState?: string;
+    /**
+     * Extra cookies to inject into the browser context.
+     */
+    cookies?: BrowserCookie[];
+}
+/**
+ * Configuration for MCP host simulation
+ */
+interface MCPHostConfig {
+    /**
+     * Host type for the simulation.
+     *
+     * - 'sdk': Programmatic via Vercel AI SDK (default). The framework's MCP connection is reused.
+     * - 'cli': CLI-based hosts (e.g., Claude Code, Codex). Spawns a process with its own MCP connection.
+     * - 'browser': Web-based hosts (not yet implemented).
+     * - 'desktop': Desktop app hosts (not yet implemented).
+     *
+     * @default 'sdk'
+     */
+    hostType?: HostType;
+    /**
+     * LLM provider (required for 'sdk' host type, ignored for 'cli')
+     */
+    provider?: LLMProvider;
+    /**
+     * Environment variable name containing the API key
+     */
+    apiKeyEnvVar?: string;
+    /**
+     * Model to use (provider-specific default if omitted)
+     */
+    model?: string;
+    /**
+     * Maximum tokens for response
+     */
+    maxTokens?: number;
+    /**
+     * Temperature (0-1, lower is more deterministic)
+     * @default 0
+     */
+    temperature?: number;
+    /**
+     * Maximum number of tool call steps to allow in a single conversation
+     * @default 10
+     */
+    maxToolCalls?: number;
+    /**
+     * CLI host configuration (required for 'cli' host type).
+     */
+    cli?: CLIConfig;
+    /**
+     * Browser host configuration (required for 'browser' host type).
+     */
+    browser?: BrowserConfig;
+}
+/**
+ * A tool call made by the LLM
+ */
+interface LLMToolCall {
+    /** Tool name */
+    name: string;
+    /** Tool arguments (as provided by LLM) */
+    arguments: Record<string, unknown>;
+    /** Optional tool call ID (for tracking) */
+    id?: string;
+}
+/**
+ * Result from an MCP host simulation
+ */
+interface MCPHostSimulationResult {
+    /** Whether the simulation succeeded */
+    success: boolean;
+    /** Tool calls made by the LLM */
+    toolCalls: Array<LLMToolCall>;
+    /** Final response from the LLM */
+    response?: string;
+    /** Error message if simulation failed */
+    error?: string;
+    /** The scenario prompt that was given to the LLM */
+    scenario?: string;
+    /** The conversation turns for attribution analysis */
+    conversationHistory?: Array<{
+        role: 'user' | 'assistant' | 'tool';
+        content: string;
+    }>;
+    /**
+     * Milliseconds spent waiting for LLM responses
+     * (excludes MCP tool execution time)
+     */
+    llmDurationMs?: number;
+    /**
+     * Milliseconds spent executing MCP tool calls
+     * (excludes LLM response time)
+     */
+    mcpDurationMs?: number;
+    /**
+     * Token usage from the LLM during simulation.
+     * Populated by SDK-based hosts from the AI SDK response.
+     */
+    usage?: UsageMetrics;
+}
+/**
+ * Interface for MCP host simulators.
+ *
+ * The only built-in implementation is the Vercel AI SDK orchestrator
+ * (src/evals/mcpHost/adapters/vercel.ts). Custom implementations can be
+ * created for specialised testing needs.
+ */
+interface MCPHostSimulator {
+    /**
+     * Simulates an MCP host interacting with an MCP server
+     *
+     * @param mcp - MCP fixture API
+     * @param scenario - Natural language prompt describing what the LLM should do
+     * @param config - MCP host configuration
+     * @returns Simulation result with tool calls and response
+     */
+    simulate(mcp: MCPFixtureApi, scenario: string, config: MCPHostConfig): Promise<MCPHostSimulationResult>;
+}
+
+/**
+ * Evaluation mode
+ */
+type EvalMode = 'direct' | 'mcp_host';
+/**
+ * A single eval test case
+ *
+ * For 'direct' mode: toolName and args are required
+ * For 'mcp_host' mode: scenario and mcpHostConfig are required
+ */
+interface EvalCase {
+    /**
+     * Unique identifier for this test case
+     */
+    id: string;
+    /**
+     * Human-readable description of what this test case validates
+     */
+    description?: string;
+    /**
+     * Evaluation mode
+     * - 'direct': Direct API calls to MCP tools (default)
+     * - 'mcp_host': LLM-driven tool selection via natural language
+     *
+     * @default 'direct'
+     */
+    mode?: EvalMode;
+    /**
+     * Name of the MCP tool to call (required for 'direct' mode, optional for 'mcp_host' mode)
+     */
+    toolName?: string;
+    /**
+     * Arguments to pass to the tool (required for 'direct' mode, optional for 'mcp_host' mode)
+     */
+    args?: Record<string, unknown>;
+    /**
+     * Natural language scenario for LLM to execute (optional, required for 'mcp_host' mode)
+     *
+     * @example "Get the weather for London and tell me if I need an umbrella"
+     */
+    scenario?: string;
+    /**
+     * MCP host configuration (optional for 'mcp_host' mode)
+     *
+     * If not specified, uses default configuration from test environment
+     */
+    mcpHostConfig?: MCPHostConfig;
+    /**
+     * Additional metadata for this test case
+     *
+     * For 'mcp_host' mode, can include 'expectedToolCalls' for validation
+     */
+    metadata?: Record<string, unknown>;
+    /**
+     * Number of times to run this case and compute an assertion pass rate.
+     * When > 1, `EvalCaseResult.assertionPassRate` is populated and `pass` is determined
+     * by `accuracyThreshold` rather than a single run.
+     * @default 1
+     */
+    iterations?: number;
+    /**
+     * Minimum accuracy (0–1) required to pass when `iterations > 1`.
+     * @default 1.0 (all iterations must pass)
+     */
+    accuracyThreshold?: number;
+    /**
+     * Number of times to invoke the LLM judge per `passesJudge` assertion.
+     * Scores are averaged; the mean must meet the threshold to pass.
+     * Reduces judge variance caused by non-determinism.
+     * Per-assertion `passesJudge.reps` overrides this value.
+     * @default 1
+     */
+    judgeReps?: number;
+    /**
+     * Golden/expected answer for this case.
+     * When set, automatically passed as `reference` to the LLM judge
+     * (unless passesJudge.reference is explicitly provided).
+     * Mirrors EvalV2's `canonical_answer` field.
+     */
+    canonicalAnswer?: string;
+    /**
+     * Arbitrary string labels for this case.
+     * Use for filtering eval runs with `EvalRunnerOptions.filterTags`
+     * and for slicing results by category.
+     *
+     * @example ['tool-finding', 'multi-hop', 'search']
+     */
+    tags?: string[];
+    /**
+     * Expectations to validate against the tool response
+     *
+     * Multiple expectations can be combined and will all be validated.
+     *
+     * @example
+     * ```json
+     * {
+     *   "id": "weather-london",
+     *   "toolName": "get_weather",
+     *   "args": { "city": "London" },
+     *   "expect": {
+     *     "containsText": ["temperature", "conditions"],
+     *     "schema": "WeatherResponse",
+     *     "responseSize": { "maxBytes": 10000 },
+     *     "isError": false
+     *   }
+     * }
+     * ```
+     */
+    expect?: EvalExpectBlock;
+}
+/**
+ * Configuration for a single LLM-as-judge evaluation
+ */
+interface JudgeExpectConfig {
+    /**
+     * Name of a registered custom judge executor.
+     * When set, the named judge handles evaluation and returns a normalized score.
+     * The `threshold` determines pass/fail. `reps` and LLM config fields
+     * (provider, model, etc.) are ignored.
+     */
+    judge?: string;
+    /** Built-in rubric name or custom rubric object. Required when no `judge` is specified. */
+    rubric?: BuiltInRubric | {
+        text: string;
+    };
+    /** Reference response to compare against */
+    reference?: unknown;
+    /** Score threshold for passing (0-1, default: 0.7) */
+    threshold?: number;
+    /** Number of judge evaluations for this assertion. Overrides EvalCase.judgeReps. */
+    reps?: number;
+    /** Judge provider. @default 'anthropic' */
+    provider?: 'anthropic' | 'vertex-anthropic' | 'anthropic-agent-sdk' | 'openai' | 'google';
+    /** Model override (e.g., 'claude-opus-4-20250514') */
+    model?: string;
+    /** Environment variable name for API key */
+    apiKeyEnvVar?: string;
+    /** Max tokens for judge response */
+    maxTokens?: number;
+    /** Temperature for judge LLM (0–1) */
+    temperature?: number;
+    /** Max budget in USD per evaluation */
+    maxBudgetUsd?: number;
+    /** Fail if response exceeds this size in bytes before judging */
+    maxToolOutputSize?: number;
+}
+/**
+ * Unified expectation block for eval cases
+ *
+ * Mirrors the Playwright matcher API for consistency.
+ */
+interface EvalExpectBlock {
+    /**
+     * Exact response match (toMatchToolResponse)
+     */
+    response?: unknown;
+    /**
+     * Name of schema to validate against (toMatchToolSchema)
+     */
+    schema?: string;
+    /**
+     * Text substring(s) that must be present (toContainToolText)
+     */
+    containsText?: string | string[];
+    /**
+     * Regex pattern(s) that must match (toMatchToolPattern)
+     */
+    matchesPattern?: string | string[];
+    /**
+     * Snapshot name for comparison (toMatchToolSnapshot)
+     */
+    snapshot?: string;
+    /**
+     * Snapshot sanitizers to apply
+     */
+    snapshotSanitizers?: SnapshotSanitizer[];
+    /**
+     * Error expectation (toBeToolError)
+     * - true: expects any error
+     * - false: expects no error
+     * - string: expects error containing this message
+     */
+    isError?: boolean | string | string[];
+    /**
+     * LLM-as-judge evaluation (toPassToolJudge)
+     *
+     * Accepts a single judge config or an array for multi-judge evaluation.
+     * When an array is provided, all judges must pass (AND semantics).
+     */
+    passesJudge?: JudgeExpectConfig | JudgeExpectConfig[];
+    /**
+     * Response size validation (toHaveToolResponseSize)
+     */
+    responseSize?: {
+        /** Maximum allowed size in bytes */
+        maxBytes?: number;
+        /** Minimum required size in bytes */
+        minBytes?: number;
+    };
+    /**
+     * Asserts which tools the LLM called during a mcp_host simulation.
+     * Only meaningful for mcp_host mode — direct mode has no tool call trace.
+     */
+    toolsTriggered?: {
+        /** Expected tool calls */
+        calls: Array<{
+            /** Tool name */
+            name: string;
+            /** Expected arguments (partial match — extra keys are allowed) */
+            arguments?: Record<string, unknown>;
+            /** Whether this call MUST have been made (default: true) */
+            required?: boolean;
+        }>;
+        /**
+         * 'strict': calls must appear in the exact order listed
+         * 'any': calls can appear in any order (default)
+         */
+        order?: 'strict' | 'any';
+        /** If true, no tool calls outside the `calls` list are allowed */
+        exclusive?: boolean;
+    };
+    /**
+     * Asserts the number of tool calls made during a mcp_host simulation.
+     */
+    toolCallCount?: {
+        /** Minimum number of tool calls */
+        min?: number;
+        /** Maximum number of tool calls */
+        max?: number;
+        /** Exact number of tool calls */
+        exact?: number;
+    };
+}
+/**
+ * A complete eval dataset containing multiple test cases
+ */
+interface EvalDataset {
+    /**
+     * Dataset name
+     */
+    name: string;
+    /**
+     * Dataset description
+     */
+    description?: string;
+    /**
+     * Test cases in this dataset
+     */
+    cases: Array<EvalCase>;
+    /**
+     * Optional schema definitions referenced by test cases
+     */
+    schemas?: Record<string, z.ZodSchema>;
+    /**
+     * Additional dataset metadata
+     */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Zod schema for EvalCase
+ *
+ * toolName and args are optional for mcp_host mode (which uses scenario instead)
+ */
+declare const EvalCaseSchema: z.ZodObject<{
+    id: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    mode: z.ZodOptional<z.ZodEnum<{
+        direct: "direct";
+        mcp_host: "mcp_host";
+    }>>;
+    toolName: z.ZodOptional<z.ZodString>;
+    args: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    scenario: z.ZodOptional<z.ZodString>;
+    mcpHostConfig: z.ZodOptional<z.ZodObject<{
+        hostType: z.ZodOptional<z.ZodEnum<{
+            sdk: "sdk";
+            cli: "cli";
+            browser: "browser";
+            desktop: "desktop";
+        }>>;
+        provider: z.ZodOptional<z.ZodEnum<{
+            openai: "openai";
+            anthropic: "anthropic";
+            azure: "azure";
+            google: "google";
+            mistral: "mistral";
+            deepseek: "deepseek";
+            openrouter: "openrouter";
+            xai: "xai";
+            "vertex-anthropic": "vertex-anthropic";
+        }>>;
+        apiKeyEnvVar: z.ZodOptional<z.ZodString>;
+        model: z.ZodOptional<z.ZodString>;
+        maxTokens: z.ZodOptional<z.ZodNumber>;
+        temperature: z.ZodOptional<z.ZodNumber>;
+        maxToolCalls: z.ZodOptional<z.ZodNumber>;
+        cli: z.ZodOptional<z.ZodObject<{
+            command: z.ZodString;
+            args: z.ZodArray<z.ZodString>;
+            outputFormat: z.ZodOptional<z.ZodEnum<{
+                "stream-json": "stream-json";
+                json: "json";
+            }>>;
+            timeout: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    iterations: z.ZodOptional<z.ZodNumber>;
+    accuracyThreshold: z.ZodOptional<z.ZodNumber>;
+    judgeReps: z.ZodOptional<z.ZodNumber>;
+    canonicalAnswer: z.ZodOptional<z.ZodString>;
+    tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    expect: z.ZodOptional<z.ZodObject<{
+        response: z.ZodOptional<z.ZodUnknown>;
+        schema: z.ZodOptional<z.ZodString>;
+        containsText: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+        matchesPattern: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+        snapshot: z.ZodOptional<z.ZodString>;
+        snapshotSanitizers: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodEnum<{
+            timestamp: "timestamp";
+            uuid: "uuid";
+            "iso-date": "iso-date";
+            objectId: "objectId";
+            jwt: "jwt";
+        }>, z.ZodObject<{
+            pattern: z.ZodString;
+            replacement: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>, z.ZodObject<{
+            remove: z.ZodArray<z.ZodString>;
+        }, z.core.$strip>]>>>;
+        isError: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString, z.ZodArray<z.ZodString>]>>;
+        passesJudge: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
+            judge: z.ZodOptional<z.ZodString>;
+            rubric: z.ZodOptional<z.ZodUnion<readonly [z.ZodEnum<{
+                correctness: "correctness";
+                completeness: "completeness";
+                groundedness: "groundedness";
+                "instruction-following": "instruction-following";
+                conciseness: "conciseness";
+            }>, z.ZodObject<{
+                text: z.ZodString;
+            }, z.core.$strip>]>>;
+            reference: z.ZodOptional<z.ZodUnknown>;
+            threshold: z.ZodOptional<z.ZodNumber>;
+            reps: z.ZodOptional<z.ZodNumber>;
+            provider: z.ZodOptional<z.ZodEnum<{
+                openai: "openai";
+                anthropic: "anthropic";
+                google: "google";
+                "vertex-anthropic": "vertex-anthropic";
+                "anthropic-agent-sdk": "anthropic-agent-sdk";
+            }>>;
+            model: z.ZodOptional<z.ZodString>;
+            apiKeyEnvVar: z.ZodOptional<z.ZodString>;
+            maxTokens: z.ZodOptional<z.ZodNumber>;
+            temperature: z.ZodOptional<z.ZodNumber>;
+            maxBudgetUsd: z.ZodOptional<z.ZodNumber>;
+            maxToolOutputSize: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>, z.ZodArray<z.ZodObject<{
+            judge: z.ZodOptional<z.ZodString>;
+            rubric: z.ZodOptional<z.ZodUnion<readonly [z.ZodEnum<{
+                correctness: "correctness";
+                completeness: "completeness";
+                groundedness: "groundedness";
+                "instruction-following": "instruction-following";
+                conciseness: "conciseness";
+            }>, z.ZodObject<{
+                text: z.ZodString;
+            }, z.core.$strip>]>>;
+            reference: z.ZodOptional<z.ZodUnknown>;
+            threshold: z.ZodOptional<z.ZodNumber>;
+            reps: z.ZodOptional<z.ZodNumber>;
+            provider: z.ZodOptional<z.ZodEnum<{
+                openai: "openai";
+                anthropic: "anthropic";
+                google: "google";
+                "vertex-anthropic": "vertex-anthropic";
+                "anthropic-agent-sdk": "anthropic-agent-sdk";
+            }>>;
+            model: z.ZodOptional<z.ZodString>;
+            apiKeyEnvVar: z.ZodOptional<z.ZodString>;
+            maxTokens: z.ZodOptional<z.ZodNumber>;
+            temperature: z.ZodOptional<z.ZodNumber>;
+            maxBudgetUsd: z.ZodOptional<z.ZodNumber>;
+            maxToolOutputSize: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>>]>>;
+        responseSize: z.ZodOptional<z.ZodObject<{
+            maxBytes: z.ZodOptional<z.ZodNumber>;
+            minBytes: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>>;
+        toolsTriggered: z.ZodOptional<z.ZodObject<{
+            calls: z.ZodArray<z.ZodObject<{
+                name: z.ZodString;
+                arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+                required: z.ZodOptional<z.ZodBoolean>;
+            }, z.core.$strip>>;
+            order: z.ZodOptional<z.ZodEnum<{
+                strict: "strict";
+                any: "any";
+            }>>;
+            exclusive: z.ZodOptional<z.ZodBoolean>;
+        }, z.core.$strip>>;
+        toolCallCount: z.ZodOptional<z.ZodObject<{
+            min: z.ZodOptional<z.ZodNumber>;
+            max: z.ZodOptional<z.ZodNumber>;
+            exact: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+/**
+ * Zod schema for EvalDataset (without schemas field, as schemas aren't serializable)
+ */
+declare const EvalDatasetSchema: z.ZodObject<{
+    name: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    cases: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        description: z.ZodOptional<z.ZodString>;
+        mode: z.ZodOptional<z.ZodEnum<{
+            direct: "direct";
+            mcp_host: "mcp_host";
+        }>>;
+        toolName: z.ZodOptional<z.ZodString>;
+        args: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        scenario: z.ZodOptional<z.ZodString>;
+        mcpHostConfig: z.ZodOptional<z.ZodObject<{
+            hostType: z.ZodOptional<z.ZodEnum<{
+                sdk: "sdk";
+                cli: "cli";
+                browser: "browser";
+                desktop: "desktop";
+            }>>;
+            provider: z.ZodOptional<z.ZodEnum<{
+                openai: "openai";
+                anthropic: "anthropic";
+                azure: "azure";
+                google: "google";
+                mistral: "mistral";
+                deepseek: "deepseek";
+                openrouter: "openrouter";
+                xai: "xai";
+                "vertex-anthropic": "vertex-anthropic";
+            }>>;
+            apiKeyEnvVar: z.ZodOptional<z.ZodString>;
+            model: z.ZodOptional<z.ZodString>;
+            maxTokens: z.ZodOptional<z.ZodNumber>;
+            temperature: z.ZodOptional<z.ZodNumber>;
+            maxToolCalls: z.ZodOptional<z.ZodNumber>;
+            cli: z.ZodOptional<z.ZodObject<{
+                command: z.ZodString;
+                args: z.ZodArray<z.ZodString>;
+                outputFormat: z.ZodOptional<z.ZodEnum<{
+                    "stream-json": "stream-json";
+                    json: "json";
+                }>>;
+                timeout: z.ZodOptional<z.ZodNumber>;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        iterations: z.ZodOptional<z.ZodNumber>;
+        accuracyThreshold: z.ZodOptional<z.ZodNumber>;
+        judgeReps: z.ZodOptional<z.ZodNumber>;
+        canonicalAnswer: z.ZodOptional<z.ZodString>;
+        tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        expect: z.ZodOptional<z.ZodObject<{
+            response: z.ZodOptional<z.ZodUnknown>;
+            schema: z.ZodOptional<z.ZodString>;
+            containsText: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+            matchesPattern: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+            snapshot: z.ZodOptional<z.ZodString>;
+            snapshotSanitizers: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodEnum<{
+                timestamp: "timestamp";
+                uuid: "uuid";
+                "iso-date": "iso-date";
+                objectId: "objectId";
+                jwt: "jwt";
+            }>, z.ZodObject<{
+                pattern: z.ZodString;
+                replacement: z.ZodOptional<z.ZodString>;
+            }, z.core.$strip>, z.ZodObject<{
+                remove: z.ZodArray<z.ZodString>;
+            }, z.core.$strip>]>>>;
+            isError: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString, z.ZodArray<z.ZodString>]>>;
+            passesJudge: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
+                judge: z.ZodOptional<z.ZodString>;
+                rubric: z.ZodOptional<z.ZodUnion<readonly [z.ZodEnum<{
+                    correctness: "correctness";
+                    completeness: "completeness";
+                    groundedness: "groundedness";
+                    "instruction-following": "instruction-following";
+                    conciseness: "conciseness";
+                }>, z.ZodObject<{
+                    text: z.ZodString;
+                }, z.core.$strip>]>>;
+                reference: z.ZodOptional<z.ZodUnknown>;
+                threshold: z.ZodOptional<z.ZodNumber>;
+                reps: z.ZodOptional<z.ZodNumber>;
+                provider: z.ZodOptional<z.ZodEnum<{
+                    openai: "openai";
+                    anthropic: "anthropic";
+                    google: "google";
+                    "vertex-anthropic": "vertex-anthropic";
+                    "anthropic-agent-sdk": "anthropic-agent-sdk";
+                }>>;
+                model: z.ZodOptional<z.ZodString>;
+                apiKeyEnvVar: z.ZodOptional<z.ZodString>;
+                maxTokens: z.ZodOptional<z.ZodNumber>;
+                temperature: z.ZodOptional<z.ZodNumber>;
+                maxBudgetUsd: z.ZodOptional<z.ZodNumber>;
+                maxToolOutputSize: z.ZodOptional<z.ZodNumber>;
+            }, z.core.$strip>, z.ZodArray<z.ZodObject<{
+                judge: z.ZodOptional<z.ZodString>;
+                rubric: z.ZodOptional<z.ZodUnion<readonly [z.ZodEnum<{
+                    correctness: "correctness";
+                    completeness: "completeness";
+                    groundedness: "groundedness";
+                    "instruction-following": "instruction-following";
+                    conciseness: "conciseness";
+                }>, z.ZodObject<{
+                    text: z.ZodString;
+                }, z.core.$strip>]>>;
+                reference: z.ZodOptional<z.ZodUnknown>;
+                threshold: z.ZodOptional<z.ZodNumber>;
+                reps: z.ZodOptional<z.ZodNumber>;
+                provider: z.ZodOptional<z.ZodEnum<{
+                    openai: "openai";
+                    anthropic: "anthropic";
+                    google: "google";
+                    "vertex-anthropic": "vertex-anthropic";
+                    "anthropic-agent-sdk": "anthropic-agent-sdk";
+                }>>;
+                model: z.ZodOptional<z.ZodString>;
+                apiKeyEnvVar: z.ZodOptional<z.ZodString>;
+                maxTokens: z.ZodOptional<z.ZodNumber>;
+                temperature: z.ZodOptional<z.ZodNumber>;
+                maxBudgetUsd: z.ZodOptional<z.ZodNumber>;
+                maxToolOutputSize: z.ZodOptional<z.ZodNumber>;
+            }, z.core.$strip>>]>>;
+            responseSize: z.ZodOptional<z.ZodObject<{
+                maxBytes: z.ZodOptional<z.ZodNumber>;
+                minBytes: z.ZodOptional<z.ZodNumber>;
+            }, z.core.$strip>>;
+            toolsTriggered: z.ZodOptional<z.ZodObject<{
+                calls: z.ZodArray<z.ZodObject<{
+                    name: z.ZodString;
+                    arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+                    required: z.ZodOptional<z.ZodBoolean>;
+                }, z.core.$strip>>;
+                order: z.ZodOptional<z.ZodEnum<{
+                    strict: "strict";
+                    any: "any";
+                }>>;
+                exclusive: z.ZodOptional<z.ZodBoolean>;
+            }, z.core.$strip>>;
+            toolCallCount: z.ZodOptional<z.ZodObject<{
+                min: z.ZodOptional<z.ZodNumber>;
+                max: z.ZodOptional<z.ZodNumber>;
+                exact: z.ZodOptional<z.ZodNumber>;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strip>;
+/**
+ * Type for serialized eval dataset (without Zod schemas)
+ */
+type SerializedEvalDataset = z.infer<typeof EvalDatasetSchema>;
+/**
+ * Validates an eval case
+ *
+ * @param evalCase - The eval case to validate
+ * @returns The validated eval case
+ * @throws {z.ZodError} If validation fails
+ */
+declare function validateEvalCase(evalCase: unknown): EvalCase;
+/**
+ * Validates a serialized eval dataset
+ *
+ * @param dataset - The dataset to validate
+ * @returns The validated dataset
+ * @throws {z.ZodError} If validation fails
+ */
+declare function validateEvalDataset(dataset: unknown): SerializedEvalDataset;
+
+/**
+ * Options for loading an eval dataset
+ */
+interface LoadDatasetOptions {
+    /**
+     * Optional schema definitions to attach to the dataset
+     *
+     * Keys should match the expectedSchemaName in eval cases
+     */
+    schemas?: Record<string, z.ZodSchema>;
+    /**
+     * Whether to validate the loaded dataset
+     * @default true
+     */
+    validate?: boolean;
+}
+/**
+ * Loads an eval dataset from a JSON file
+ *
+ * @param filePath - Absolute path to the JSON file
+ * @param options - Load options
+ * @returns The loaded and validated dataset
+ * @throws {Error} If file cannot be read or JSON is invalid
+ * @throws {z.ZodError} If validation fails
+ *
+ * @example
+ * const dataset = await loadEvalDataset('./data/my-evals.json', {
+ *   schemas: {
+ *     'weather-response': WeatherResponseSchema,
+ *   },
+ * });
+ */
+declare function loadEvalDataset(filePath: string, options?: LoadDatasetOptions): Promise<EvalDataset>;
+/**
+ * Loads an eval dataset from a plain object
+ *
+ * Useful for programmatically creating datasets in tests
+ *
+ * @param data - The dataset data
+ * @param options - Load options
+ * @returns The loaded and validated dataset
+ * @throws {z.ZodError} If validation fails
+ *
+ * @example
+ * const dataset = loadEvalDatasetFromObject({
+ *   name: 'my-test-dataset',
+ *   cases: [
+ *     {
+ *       id: 'case-1',
+ *       toolName: 'get_weather',
+ *       args: { city: 'London' },
+ *     },
+ *   ],
+ * });
+ */
+declare function loadEvalDatasetFromObject(data: unknown, options?: LoadDatasetOptions): EvalDataset;
+
+type StoredArtifactKind = 'eval-runner-result' | 'reporter-run' | 'eval-run-comparison' | 'server-comparison';
+interface StoredEvalArtifactMetadata {
+    datasetName?: string;
+    gitHash?: string;
+    branch?: string;
+    runNumber?: string;
+    trigger?: string;
+    packageVersion?: string;
+    toolOverrideVariantId?: string;
+    mcpHostModel?: string;
+    judgeModel?: string;
+    labels?: Record<string, string>;
+    [key: string]: unknown;
+}
+interface StoredEvalArtifact<T> {
+    schemaVersion: 1;
+    kind: StoredArtifactKind;
+    id: string;
+    createdAt: string;
+    metadata: StoredEvalArtifactMetadata;
+    data: T;
+}
+interface StoredArtifactSummary {
+    kind: StoredArtifactKind;
+    id: string;
+    createdAt: string;
+    metadata: StoredEvalArtifactMetadata;
+}
+interface ListStoredArtifactsOptions {
+    limit?: number;
+}
+interface EvalResultStore {
+    saveArtifact<T>(artifact: StoredEvalArtifact<T>): Promise<void>;
+    loadArtifact<T>(kind: StoredArtifactKind, id: string): Promise<StoredEvalArtifact<T>>;
+    loadLatestArtifact<T>(kind: StoredArtifactKind): Promise<StoredEvalArtifact<T> | null>;
+    listArtifacts(kind: StoredArtifactKind, options?: ListStoredArtifactsOptions): Promise<StoredArtifactSummary[]>;
+}
+interface FileEvalResultStoreConfig {
+    provider: 'file';
+    dir: string;
+}
+interface GCSEvalResultStoreConfig {
+    provider: 'gcs';
+    bucket: string;
+    prefix?: string;
+}
+type EvalResultStoreConfig = FileEvalResultStoreConfig | GCSEvalResultStoreConfig;
+type EvalResultStoreLike = EvalResultStore | EvalResultStoreConfig;
+declare function createEvalResultStore(config: EvalResultStoreConfig): EvalResultStore;
+declare function resolveEvalResultStore(store: EvalResultStoreLike): EvalResultStore;
+declare function isEvalResultStore(value: unknown): value is EvalResultStore;
+declare function createStoredEvalArtifact<T>(options: {
+    kind: StoredArtifactKind;
+    data: T;
+    id?: string;
+    metadata?: StoredEvalArtifactMetadata;
+    createdAt?: string;
+}): StoredEvalArtifact<T>;
+declare function createDefaultArtifactId(timestamp?: string): string;
+declare function defaultEnvironmentMetadata(): StoredEvalArtifactMetadata;
+declare class FileEvalResultStore implements EvalResultStore {
+    private readonly dir;
+    constructor(config: FileEvalResultStoreConfig);
+    saveArtifact<T>(artifact: StoredEvalArtifact<T>): Promise<void>;
+    loadArtifact<T>(kind: StoredArtifactKind, id: string): Promise<StoredEvalArtifact<T>>;
+    loadLatestArtifact<T>(kind: StoredArtifactKind): Promise<StoredEvalArtifact<T> | null>;
+    listArtifacts(kind: StoredArtifactKind, options?: ListStoredArtifactsOptions): Promise<StoredArtifactSummary[]>;
+}
+declare class GCSEvalResultStore implements EvalResultStore {
+    private readonly bucketName;
+    private readonly prefix;
+    private storage;
+    constructor(config: GCSEvalResultStoreConfig);
+    saveArtifact<T>(artifact: StoredEvalArtifact<T>): Promise<void>;
+    loadArtifact<T>(kind: StoredArtifactKind, id: string): Promise<StoredEvalArtifact<T>>;
+    loadLatestArtifact<T>(kind: StoredArtifactKind): Promise<StoredEvalArtifact<T> | null>;
+    listArtifacts(kind: StoredArtifactKind, options?: ListStoredArtifactsOptions): Promise<StoredArtifactSummary[]>;
+    private getBucket;
+    private objectPath;
+}
+
+/**
+ * Reporter-specific type definitions
+ *
+ * These types are used by the MCP reporter and UI.
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * Configuration options for MCP Eval Reporter
+ */
+interface MCPEvalReporterConfig {
+    /**
+     * Output directory for reports and historical data
+     * @default '.mcp-test-results'
+     */
+    outputDir?: string;
+    /**
+     * Auto-open report in browser after test run
+     * @default false
+     */
+    autoOpen?: boolean;
+    /**
+     * Number of historical runs to keep
+     * @default 10
+     */
+    historyLimit?: number;
+    /**
+     * Suppress console output (report still generated)
+     * @default false
+     */
+    quiet?: boolean;
+    /**
+     * Include auto-tracked MCP tool calls from tests without explicit eval results.
+     * When true, any test using the MCP fixture will have its tool calls
+     * included in the report, even without using runEvalCase/runEvalDataset.
+     * When false, only tests with explicit eval results are included.
+     * @default true
+     */
+    includeAutoTracking?: boolean;
+    /**
+     * Optional external result store for durable reporter run history.
+     */
+    resultStore?: EvalResultStoreLike;
+    /**
+     * Optional run ID for externally stored reporter results.
+     * Defaults to a generated timestamp-based ID.
+     */
+    runId?: string;
+    /**
+     * Extra metadata to attach to externally stored reporter results.
+     */
+    runMetadata?: Record<string, unknown>;
+    /**
+     * When true, strips response payloads before storing reporter results externally.
+     * Local report output is unchanged.
+     * @default true
+     */
+    redactStoredResponses?: boolean;
+}
+/**
+ * Experiment tracking metadata for an eval run
+ */
+interface EvalRunMetadata {
+    /** Git commit hash at time of run */
+    gitHash?: string;
+    /** ISO timestamp of the run */
+    timestamp: string;
+    /** Package version from package.json */
+    packageVersion: string;
+    /** Runtime tool override variant identifier, when one was used */
+    toolOverrideVariantId?: string;
+    /** MCP host model identifier (if mcp_host mode) */
+    mcpHostModel?: string;
+    /** Judge model identifier (if judge was used) */
+    judgeModel?: string;
+}
+/**
+ * Individual conformance check result
+ */
+interface MCPConformanceCheck {
+    /**
+     * Check name (e.g., 'server_info_present', 'list_tools_succeeds')
+     */
+    name: string;
+    /**
+     * Whether the check passed
+     */
+    pass: boolean;
+    /**
+     * Human-readable message describing the result
+     */
+    message: string;
+}
+/**
+ * Conformance check result as stored in reporter data
+ */
+interface MCPConformanceResultData {
+    /**
+     * Test title where conformance check was run
+     */
+    testTitle: string;
+    /**
+     * Whether all checks passed
+     */
+    pass: boolean;
+    /**
+     * Individual check results
+     */
+    checks: MCPConformanceCheck[];
+    /**
+     * Server info if available
+     */
+    serverInfo?: {
+        name?: string;
+        version?: string;
+    };
+    /**
+     * Number of tools discovered
+     */
+    toolCount: number;
+    /**
+     * Auth type used for this check
+     */
+    authType?: AuthType;
+    /**
+     * Project name
+     */
+    project?: string;
+}
+/**
+ * Server capabilities data from mcp-list-tools attachment
+ */
+interface MCPServerCapabilitiesData {
+    /**
+     * Test title where listTools was called
+     */
+    testTitle: string;
+    /**
+     * List of tools available on the server
+     */
+    tools: Array<{
+        name: string;
+        description?: string;
+    }>;
+    /**
+     * Total number of tools
+     */
+    toolCount: number;
+    /**
+     * Auth type used for this test
+     */
+    authType?: AuthType;
+    /**
+     * Project name
+     */
+    project?: string;
+}
+/**
+ * Result of a single iteration within a multi-iteration eval case
+ */
+interface IterationResult {
+    /** Whether this iteration passed */
+    pass: boolean;
+    /** Execution time for this iteration */
+    durationMs: number;
+    /** Error message if the iteration failed with an exception */
+    error?: string;
+    /** When true, this iteration failed due to network/infrastructure issues rather than an assertion failure */
+    isInfrastructureError?: boolean;
+    /**
+     * Ordered trace of tool calls made by the LLM during this iteration (mcp_host mode only).
+     * Captures what was actually called so you can distinguish "LLM didn't call the tool"
+     * from "LLM called the wrong tool" from "tool was called but assertion failed".
+     */
+    mcpHostTrace?: {
+        calls: Array<{
+            name: string;
+            arguments: Record<string, unknown>;
+            status: 'expected' | 'unexpected';
+        }>;
+        missed: Array<{
+            name: string;
+        }>;
+    };
+    /** Token usage from mcp_host LLM simulation in this iteration */
+    hostUsage?: UsageMetrics;
+}
+/**
+ * Request data captured from the eval case input.
+ * Preserves what was sent so results are self-contained for debugging.
+ */
+interface EvalCaseRequest {
+    /** Human-readable description of the case */
+    description?: string;
+    /** Runtime tool override variant identifier, when one was used */
+    toolOverrideVariantId?: string;
+    /** Tool arguments (direct mode) */
+    args?: Record<string, unknown>;
+    /** Natural language scenario sent to the LLM (mcp_host mode) */
+    scenario?: string;
+    /** LLM provider/model configuration (mcp_host mode) */
+    mcpHostConfig?: {
+        provider?: string;
+        model?: string;
+    };
+}
+/**
+ * Result of a single eval case
+ */
+interface EvalCaseResult {
+    /**
+     * Case ID
+     */
+    id: string;
+    /**
+     * Dataset name this case belongs to
+     */
+    datasetName: string;
+    /**
+     * MCP tool name that was called
+     */
+    toolName: string;
+    /**
+     * Source of this result
+     */
+    source: ResultSource;
+    /**
+     * Overall pass/fail status
+     */
+    pass: boolean;
+    /**
+     * Request data from the eval case input (tool args, scenario, LLM config).
+     * Populated so results are self-contained for debugging without the original dataset.
+     */
+    request?: EvalCaseRequest;
+    /**
+     * Tool response
+     */
+    response?: unknown;
+    /**
+     * Error if tool call failed
+     */
+    error?: string;
+    /**
+     * Expectation results
+     */
+    expectations: Partial<Record<ExpectationType, EvalExpectationResult>>;
+    /**
+     * Authentication type used for this test
+     */
+    authType?: AuthType;
+    /**
+     * Playwright project name this test belongs to
+     */
+    project?: string;
+    /**
+     * Execution time in milliseconds
+     */
+    durationMs: number;
+    /**
+     * Assertion pass rate (0–1): passes divided by non-infrastructure iterations.
+     * Only present when the case was run with `iterations > 1`.
+     *
+     * Infrastructure errors (network timeouts, rate limits, etc.) are excluded from
+     * the denominator so that environment reliability does not inflate this metric.
+     */
+    assertionPassRate?: number;
+    /**
+     * 95% Wilson score confidence interval for `assertionPassRate`.
+     * Only present when the case was run with `iterations > 1`.
+     *
+     * Interpet as: the true pass rate is likely between `lower` and `upper`.
+     * Wider intervals mean fewer iterations were run; run more iterations to narrow them.
+     *
+     * @example { lower: 0.35, upper: 0.93 } // 7/10 passes → 70% ± wide CI
+     * @example { lower: 0.57, upper: 0.80 } // 35/50 passes → 70% ± narrow CI
+     */
+    assertionPassRateCI?: {
+        /** Lower bound of the 95% confidence interval (0–1) */
+        lower: number;
+        /** Upper bound of the 95% confidence interval (0–1) */
+        upper: number;
+    };
+    /**
+     * Infrastructure error rate (0–1): infra errors divided by total iterations.
+     * Only present when the case was run with `iterations > 1`.
+     */
+    infrastructureErrorRate?: number;
+    /**
+     * Per-iteration pass/fail breakdown.
+     * Only present when the case was run with `iterations > 1`.
+     */
+    iterationResults?: Array<IterationResult>;
+    /**
+     * Tags from the source eval case, for filtering and slicing reports.
+     */
+    tags?: string[];
+    /**
+     * Precision of tool calls made (0–1).
+     * 1.0 means every tool called was expected; <1.0 means unexpected tools were called.
+     * Populated whenever a `toolsTriggered` expectation is evaluated.
+     */
+    toolPrecision?: number;
+    /**
+     * Recall of required tool calls (0–1).
+     * 1.0 means all required tools were called; <1.0 means some were missed.
+     * Only populated when toolsTriggered expectation was evaluated.
+     */
+    toolRecall?: number;
+    /**
+     * Pass/fail status of this case in the baseline run.
+     * Only present when a baseline was provided to runEvalDataset.
+     */
+    baselinePass?: boolean;
+    /**
+     * Number of iterations that failed due to infrastructure errors (network, rate limits, etc.)
+     * Only present when the case was run with `iterations > 1`.
+     */
+    infrastructureErrorCount?: number;
+    /**
+     * Ordered trace of tool calls made by the LLM in mcp_host mode.
+     * Only populated when the eval case uses toolsTriggered expectations.
+     */
+    mcpHostTrace?: {
+        /** The ordered sequence of tool calls made by the LLM */
+        calls: Array<{
+            name: string;
+            arguments: Record<string, unknown>;
+            /** 'expected' = was in the expected set, 'unexpected' = was not expected */
+            status: 'expected' | 'unexpected';
+        }>;
+        /** Tools that were required but never called */
+        missed: Array<{
+            name: string;
+        }>;
+    };
+    /**
+     * Aggregate token usage from mcp_host LLM simulation for this case.
+     * Summed across all iterations. Only populated for mcp_host mode cases.
+     */
+    hostUsage?: UsageMetrics;
+}
+/**
+ * Aggregated MCP eval run data
+ */
+interface MCPEvalRunData {
+    /**
+     * Run timestamp (ISO 8601)
+     */
+    timestamp: string;
+    /**
+     * Total duration in milliseconds
+     */
+    durationMs: number;
+    /**
+     * Environment info
+     */
+    environment: {
+        ci: boolean;
+        node: string;
+        platform: string;
+    };
+    /**
+     * Aggregate metrics
+     */
+    metrics: {
+        /**
+         * Total number of eval cases
+         */
+        total: number;
+        /**
+         * Number of passed cases
+         */
+        passed: number;
+        /**
+         * Number of failed cases
+         */
+        failed: number;
+        /**
+         * Pass rate (0-1)
+         */
+        passRate: number;
+        /**
+         * Dataset breakdown: dataset name -> count
+         */
+        datasetBreakdown: Record<string, number>;
+        /**
+         * Expectation type breakdown
+         */
+        expectationBreakdown: ExpectationBreakdown;
+        /**
+         * Aggregate token usage from all mcp_host LLM simulations in this run.
+         */
+        totalHostUsage?: UsageMetrics;
+    };
+    /**
+     * All eval results from this run
+     */
+    results: EvalCaseResult[];
+    /**
+     * Conformance check results (optional)
+     */
+    conformanceChecks?: MCPConformanceResultData[];
+    /**
+     * Server capabilities discovered via listTools (optional)
+     */
+    serverCapabilities?: MCPServerCapabilitiesData[];
+}
+/**
+ * Historical summary for trend charts
+ */
+interface MCPEvalHistoricalSummary {
+    timestamp: string;
+    total: number;
+    passed: number;
+    failed: number;
+    passRate: number;
+    durationMs: number;
+}
+/**
+ * Complete data structure passed to UI
+ */
+interface MCPEvalData {
+    runData: MCPEvalRunData;
+    historical: MCPEvalHistoricalSummary[];
+}
+
+/**
+ * Context passed to the eval runner
+ */
+interface EvalContext {
+    /**
+     * MCP fixture API for interacting with the server
+     */
+    mcp: MCPFixtureApi;
+    /**
+     * Optional Playwright TestInfo for reporter integration
+     * When provided, eval results will be attached to the test for the MCP reporter
+     */
+    testInfo?: TestInfo;
+    /**
+     * Optional Playwright expect function for snapshot testing
+     * Required for snapshot expectations to work properly
+     */
+    expect?: Expect;
+}
+
+/**
+ * Metadata overrides for a single existing MCP tool.
+ */
+interface ToolMetadataOverride {
+    /**
+     * Replacement tool description shown to MCP hosts.
+     */
+    description?: string;
+    /**
+     * Replacement input schema shown to MCP hosts.
+     */
+    inputSchema?: Record<string, unknown>;
+}
+/**
+ * Runtime metadata variant for experimenting with MCP tool discoverability.
+ *
+ * Tool keys are canonical MCP server tool names. Overrides affect only the
+ * metadata returned from listTools(); callTool() still forwards canonical tool
+ * names and arguments to the original MCP server.
+ */
+interface ToolOverrideVariant {
+    /**
+     * Stable identifier for this runtime variant.
+     */
+    id: string;
+    /**
+     * Optional human-readable explanation of what this variant is testing.
+     */
+    description?: string;
+    /**
+     * Per-tool metadata overrides keyed by canonical tool name.
+     */
+    tools: Record<string, ToolMetadataOverride>;
+}
+/**
+ * Overall result of running an eval dataset
+ */
+interface EvalRunnerResult {
+    /**
+     * Total number of cases
+     */
+    total: number;
+    /**
+     * Number of passing cases
+     */
+    passed: number;
+    /**
+     * Number of failing cases
+     */
+    failed: number;
+    /**
+     * Individual case results
+     */
+    caseResults: Array<EvalCaseResult>;
+    /**
+     * Overall execution time in milliseconds
+     */
+    durationMs: number;
+    /**
+     * Difference between current pass rate and baseline pass rate.
+     * Positive = improvement, negative = regression.
+     * Only present when `baselineResultsFrom` was provided.
+     */
+    deltaPassRate?: number;
+    /**
+     * Number of cases that regressed: passed in baseline, failed now.
+     * Only present when `baselineResultsFrom` was provided.
+     */
+    regressions?: number;
+    /**
+     * Number of cases that improved: failed in baseline, passed now.
+     * Only present when `baselineResultsFrom` was provided.
+     */
+    improvements?: number;
+    /**
+     * Average tool precision across all mcp_host cases that have a
+     * `toolsTriggered` expectation (precision = fraction of called tools
+     * that were expected). Only present when at least one such case ran.
+     */
+    datasetToolPrecision?: number;
+    /**
+     * Average tool recall across all mcp_host cases that have a
+     * `toolsTriggered` expectation (recall = fraction of required tools
+     * that were actually called). Only present when at least one such case ran.
+     */
+    datasetToolRecall?: number;
+    /**
+     * Harmonic mean of `datasetToolPrecision` and `datasetToolRecall`.
+     * Only present when at least one case contributes precision/recall data.
+     */
+    datasetToolF1?: number;
+    /**
+     * Experiment tracking metadata captured at run time.
+     */
+    metadata?: EvalRunMetadata;
+    /**
+     * Aggregate token usage from all mcp_host LLM simulations across all cases.
+     */
+    totalHostUsage?: UsageMetrics;
+}
+type StoredEvalResultRef = 'latest' | {
+    id: string;
+};
+interface StoredEvalResultLoadOptions {
+    store: true;
+    ref: StoredEvalResultRef;
+}
+interface StoredEvalResultSaveOptions {
+    store: true;
+    ref?: 'latest' | {
+        id?: string;
+    };
+}
+/**
+ * Options for running eval dataset
+ */
+interface EvalRunnerOptions {
+    /**
+     * The dataset to run
+     */
+    dataset: EvalDataset;
+    /**
+     * Schema registry for schema validation by name
+     *
+     * Maps schema names to Zod schemas for use with expect.schema
+     *
+     * @example
+     * ```typescript
+     * {
+     *   schemas: {
+     *     WeatherResponse: z.object({ temperature: z.number() }),
+     *     ErrorResponse: z.object({ error: z.string() }),
+     *   }
+     * }
+     * ```
+     */
+    schemas?: Record<string, ZodType>;
+    /**
+     * Whether to stop on first failure
+     * @default false
+     */
+    stopOnFailure?: boolean;
+    /**
+     * Optional callback called after each case
+     */
+    onCaseComplete?: (result: EvalCaseResult) => void | Promise<void>;
+    /**
+     * Maximum number of eval cases to run concurrently.
+     * When > 1, cases run in parallel (ignores stopOnFailure ordering).
+     * @default 1 (sequential)
+     */
+    concurrency?: number;
+    /**
+     * Default iteration count for `mcp_host` mode cases that do not specify
+     * `iterations` explicitly. Has no effect on `direct` mode cases (which are
+     * deterministic and always default to 1 iteration).
+     *
+     * Set to 10 for standard runs or 20 for release gates. Individual cases can
+     * still override this with their own `iterations` field.
+     *
+     * @default 1 (preserves historical behaviour when not set)
+     *
+     * @example
+     * ```typescript
+     * // Run all mcp_host cases 10 times each by default
+     * await runEvalDataset({ dataset, defaultLlmIterations: 10 }, { mcp });
+     * ```
+     */
+    defaultLlmIterations?: number;
+    /**
+     * Default number of judge evaluations for cases that do not specify
+     * `judgeReps` explicitly. Applies to any case with a `passesJudge`
+     * expectation. Per-case `judgeReps` overrides this.
+     *
+     * @default 1 (single judge run)
+     */
+    defaultJudgeReps?: number;
+    /**
+     * When set, only eval cases whose `tags` array contains at least one of
+     * the specified tags are run. Cases without a `tags` field are excluded.
+     * When undefined or empty, all cases run (default behavior).
+     */
+    filterTags?: string[];
+    /**
+     * If set, saves the run results to this file path after completion.
+     * Use with `baselineResultsFrom` on the next run for regression detection.
+     *
+     * @example '.mcp-test-results/baseline.json'
+     */
+    saveResultsTo?: string | StoredEvalResultSaveOptions;
+    /**
+     * When true (default), strips the `response` field from each case result
+     * before saving the baseline file. Keeps baseline files small and git-friendly —
+     * the full tool response is not needed for pass/fail regression detection.
+     *
+     * Set to false to preserve complete responses in the saved file.
+     *
+     * @default true
+     */
+    omitResponsesFromBaseline?: boolean;
+    /**
+     * When true (default), strips response bodies from each case result before
+     * saving to an external result store. Stored artifacts only need the pass/fail
+     * shape and tool-call metadata — full response payloads are not necessary
+     * for regression detection or history comparison. Set to false when you
+     * specifically need stored artifacts to retain complete responses.
+     *
+     * Defaults to `true` to match the reporter's `redactStoredResponses` default
+     * (see `MCPReporter`). Both write paths produce the same redaction shape, so
+     * users with both configured don't end up with a mix of redacted and
+     * non-redacted artifacts depending on which code path wrote them.
+     *
+     * @default true
+     */
+    redactStoredResponses?: boolean;
+    /**
+     * Optional external result store for loading/saving eval run artifacts.
+     */
+    resultStore?: EvalResultStoreLike;
+    /**
+     * If set, loads this file or stored result as the baseline and computes delta metrics vs the current run.
+     * Populates `EvalRunnerResult.deltaPassRate`, `.regressions`, `.improvements`,
+     * and tags each `EvalCaseResult.baselinePass`.
+     */
+    baselineResultsFrom?: string | StoredEvalResultLoadOptions;
+    /**
+     * Runtime MCP tool metadata overrides used for variant experiments.
+     *
+     * Overrides are applied to the tool list shown to MCP hosts without changing
+     * the eval dataset or mutating the underlying MCP server. Tool keys must be
+     * canonical tool names exposed by the server.
+     */
+    toolOverrides?: ToolOverrideVariant;
+    /**
+     * MCP host model identifier to record in run metadata.
+     * Use this to identify which model was used when running mcp_host cases.
+     *
+     * @example 'claude-opus-4-20250514'
+     */
+    mcpHostModel?: string;
+    /**
+     * Judge model identifier to record in run metadata.
+     * Use this to identify which model was used for judge evaluations.
+     *
+     * @example 'claude-sonnet-4-20250514'
+     */
+    judgeModel?: string;
+}
+/**
+ * Options for running a single eval case
+ */
+interface EvalCaseOptions {
+    /**
+     * Dataset name for the result (defaults to 'single-case')
+     */
+    datasetName?: string;
+    /**
+     * Schema registry for schema validation by name
+     */
+    schemas?: Record<string, ZodType>;
+    /**
+     * Runtime tool override variant id for reporter/debug metadata.
+     */
+    toolOverrideVariantId?: string;
+}
+/**
+ * Runs a single eval case and returns the result.
+ * When `evalCase.iterations > 1`, runs the case N times and returns accuracy.
+ *
+ * @param evalCase - The eval case to run
+ * @param context - Context containing mcp, testInfo, expect
+ * @param options - Optional configuration (datasetName, schemas)
+ * @returns The result of running the eval case
+ *
+ * @example
+ * ```typescript
+ * const result = await runEvalCase(
+ *   evalCase,
+ *   { mcp, testInfo, expect },
+ *   { schemas: { WeatherResponse: WeatherSchema } }
+ * );
+ *
+ * expect(result.pass).toBe(true);
+ * ```
+ */
+declare function runEvalCase(evalCase: EvalCase, context: EvalContext, options?: EvalCaseOptions): Promise<EvalCaseResult>;
+declare function runEvalDataset(options: EvalRunnerOptions, context: EvalContext): Promise<EvalRunnerResult>;
+
+/** Outcome of comparing two servers on a single eval case. */
+type ComparisonOutcome = 'A_WINS' | 'B_WINS' | 'TIE' | 'BOTH_FAIL';
+/** Result of comparing a single eval case across two servers. */
+interface CaseComparisonResult {
+    /** Case ID */
+    id: string;
+    /** Comparison outcome */
+    outcome: ComparisonOutcome;
+    /** Result from server A */
+    serverA: EvalCaseResult;
+    /** Result from server B */
+    serverB: EvalCaseResult;
+}
+/** Aggregated result of running a dataset against two servers. */
+interface ServerComparisonResult {
+    /** Dataset name */
+    dataset: string;
+    /** Total cases compared (cases present in both runs) */
+    total: number;
+    /** Cases where server A passed and server B failed */
+    aWins: number;
+    /** Cases where server B passed and server A failed */
+    bWins: number;
+    /** Cases where both passed */
+    ties: number;
+    /** Cases where both failed */
+    bothFail: number;
+    /** Cases with a decisive outcome (aWins + bWins + ties, excludes BOTH_FAIL) */
+    decidedCases: number;
+    /** Fraction of total cases where both servers failed (bothFail / total) */
+    failureAlignment: number;
+    /** A win rate (aWins / decidedCases, excludes BOTH_FAIL) */
+    aWinRate: number;
+    /** B win rate (bWins / decidedCases, excludes BOTH_FAIL) */
+    bWinRate: number;
+    /** Tie rate (ties / decidedCases, excludes BOTH_FAIL) */
+    tieRate: number;
+    /** Per-case comparison results */
+    cases: CaseComparisonResult[];
+    /** Full result from server A */
+    serverAResult: EvalRunnerResult;
+    /** Full result from server B */
+    serverBResult: EvalRunnerResult;
+    /** Total duration in milliseconds */
+    durationMs: number;
+}
+/**
+ * Options for `runServerComparison`.
+ * Same as `EvalRunnerOptions` without baseline-specific fields.
+ */
+type ServerComparisonOptions = Omit<EvalRunnerOptions, 'saveResultsTo' | 'baselineResultsFrom'> & {
+    comparisonStore?: EvalResultStoreLike;
+    comparisonId?: string;
+    comparisonMetadata?: StoredEvalArtifactMetadata;
+    redactStoredResponses?: boolean;
+};
+/**
+ * Runs the same eval dataset against two MCP servers in parallel and
+ * returns a detailed per-case comparison of results.
+ *
+ * Both servers receive identical cases and options. The comparison uses
+ * simple pass/fail per case: A_WINS means A passed and B failed, etc.
+ *
+ * @param options - Eval dataset and runner options (shared between both servers)
+ * @param contextA - MCP context for server A (e.g., Glean MCP)
+ * @param contextB - MCP context for server B (e.g., native MCP)
+ * @returns Comparison result with per-case outcomes and aggregate win rates
+ *
+ * @example
+ * ```typescript
+ * const comparison = await runServerComparison(
+ *   { dataset },
+ *   { mcp: gleanMcpFixture },
+ *   { mcp: nativeMcpFixture }
+ * );
+ * console.log(`Glean MCP wins: ${(comparison.aWinRate * 100).toFixed(1)}%`);
+ * console.log(`Native MCP wins: ${(comparison.bWinRate * 100).toFixed(1)}%`);
+ * ```
+ */
+declare function runServerComparison(options: ServerComparisonOptions, contextA: EvalContext, contextB: EvalContext): Promise<ServerComparisonResult>;
+interface SaveServerComparisonOptions {
+    store: EvalResultStoreLike;
+    comparison: ServerComparisonResult;
+    id?: string;
+    metadata?: StoredEvalArtifactMetadata;
+    redactStoredResponses?: boolean;
+}
+declare function saveServerComparison(options: SaveServerComparisonOptions): Promise<StoredEvalArtifact<ServerComparisonResult>>;
+
+/** Labels used when presenting an eval run comparison. */
+interface EvalRunComparisonLabels {
+    /** Label for the baseline run. Defaults to "baseline". */
+    baseline?: string;
+    /** Label for the candidate run. Defaults to the candidate variant id or "candidate". */
+    candidate?: string;
+}
+/** Outcome of comparing one eval case across two completed eval runs. */
+type EvalCaseComparisonOutcome = 'IMPROVED' | 'REGRESSED' | 'UNCHANGED_PASS' | 'UNCHANGED_FAIL' | 'MISSING_FROM_BASELINE' | 'MISSING_FROM_CANDIDATE';
+/** Per-case comparison between a baseline run and candidate run. */
+interface EvalCaseComparison {
+    /** Case ID */
+    id: string;
+    /** Outcome for this case */
+    outcome: EvalCaseComparisonOutcome;
+    /** Baseline case result, absent when the case only exists in the candidate run */
+    baseline?: EvalCaseResult;
+    /** Candidate case result, absent when the case only exists in the baseline run */
+    candidate?: EvalCaseResult;
+}
+/** Options for comparing two completed eval runs. */
+interface CompareEvalRunsOptions {
+    /** Baseline run result */
+    baseline: EvalRunnerResult;
+    /** Candidate run result */
+    candidate: EvalRunnerResult;
+    /** Optional human-readable run labels */
+    labels?: EvalRunComparisonLabels;
+}
+/** Aggregated comparison between two completed eval runs. */
+interface EvalRunComparisonResult {
+    /** Baseline label */
+    baselineLabel: string;
+    /** Candidate label */
+    candidateLabel: string;
+    /** Baseline pass rate (passed / total, or 0 when total is 0) */
+    baselinePassRate: number;
+    /** Candidate pass rate (passed / total, or 0 when total is 0) */
+    candidatePassRate: number;
+    /** Candidate pass rate minus baseline pass rate */
+    deltaPassRate: number;
+    /** Baseline dataset tool precision, when present */
+    baselineToolPrecision?: number;
+    /** Candidate dataset tool precision, when present */
+    candidateToolPrecision?: number;
+    /** Candidate precision minus baseline precision, when both are present */
+    deltaToolPrecision?: number;
+    /** Baseline dataset tool recall, when present */
+    baselineToolRecall?: number;
+    /** Candidate dataset tool recall, when present */
+    candidateToolRecall?: number;
+    /** Candidate recall minus baseline recall, when both are present */
+    deltaToolRecall?: number;
+    /** Baseline dataset tool F1, when present */
+    baselineToolF1?: number;
+    /** Candidate dataset tool F1, when present */
+    candidateToolF1?: number;
+    /** Candidate F1 minus baseline F1, when both are present */
+    deltaToolF1?: number;
+    /** All per-case comparison records in deterministic order */
+    cases: EvalCaseComparison[];
+    /** Cases that failed in baseline and passed in candidate */
+    improvedCases: EvalCaseComparison[];
+    /** Cases that passed in baseline and failed in candidate */
+    regressedCases: EvalCaseComparison[];
+    /** Cases that passed in both runs */
+    unchangedPasses: EvalCaseComparison[];
+    /** Cases that failed in both runs */
+    unchangedFailures: EvalCaseComparison[];
+    /** Cases present only in candidate */
+    missingFromBaseline: EvalCaseComparison[];
+    /** Cases present only in baseline */
+    missingFromCandidate: EvalCaseComparison[];
+}
+type StoredEvalRunRef = 'latest' | {
+    id: string;
+};
+interface SaveEvalRunComparisonOptions {
+    store: EvalResultStoreLike;
+    comparison: EvalRunComparisonResult;
+    id?: string;
+    metadata?: StoredEvalArtifactMetadata;
+    redactStoredResponses?: boolean;
+}
+/**
+ * Compares two completed eval runs without running any evals or reading files.
+ *
+ * Use this after running a baseline and candidate (for example a toolOverrides
+ * variant) to compute pass-rate deltas, tool metric deltas, and per-case
+ * improvement/regression buckets.
+ */
+declare function compareEvalRuns(options: CompareEvalRunsOptions): EvalRunComparisonResult;
+declare function loadStoredEvalRunnerResult(storeLike: EvalResultStoreLike, ref: StoredEvalRunRef): Promise<StoredEvalArtifact<EvalRunnerResult>>;
+declare function saveEvalRunComparison(options: SaveEvalRunComparisonOptions): Promise<StoredEvalArtifact<EvalRunComparisonResult>>;
+
+/**
+ * Metric used to rank variant candidates and decide improvement.
+ *
+ * - `passRate`: passed / total across the dataset (always available).
+ * - `toolF1` / `toolPrecision` / `toolRecall`: dataset-level tool-call metrics,
+ *   only available when the dataset has `mcp_host` cases with `toolsTriggered`
+ *   expectations. Choosing one of these when no such cases exist throws a clear
+ *   error rather than silently ranking on nothing.
+ */
+type ExperimentMetric = 'passRate' | 'toolF1' | 'toolPrecision' | 'toolRecall';
+/**
+ * Why a variant experiment stopped.
+ *
+ * - `no-variants`: no candidates were ever produced (round 0 yielded none).
+ * - `no-improvement`: a round's best candidate did not beat the best-so-far by
+ *   at least `minImprovement`, or `proposeVariants` returned no further
+ *   candidates.
+ * - `max-rounds`: the configured `maxRounds` budget was exhausted.
+ * - `threshold-met`: reserved for future absolute-target convergence; not
+ *   emitted by the current delta-based logic.
+ */
+type VariantExperimentReason = 'threshold-met' | 'no-improvement' | 'max-rounds' | 'no-variants';
+/** Whether a winning variant should be applied, rejected, or is inconclusive. */
+type VariantRecommendation = 'apply' | 'reject' | 'inconclusive';
+/** Result of running and scoring a single candidate variant. */
+interface VariantCandidateResult {
+    /** The variant that was injected via `toolOverrides`. */
+    variant: ToolOverrideVariant;
+    /** The eval run produced for this variant. */
+    result: EvalRunnerResult;
+    /** Comparison of this candidate against the original baseline run. */
+    comparison: EvalRunComparisonResult;
+    /** The selected metric's value for this candidate. */
+    metricValue: number;
+    /** `metricValue` minus the baseline's metric value. */
+    metricDelta: number;
+    /**
+     * True when this candidate regressed at least one case and `allowRegressions`
+     * is not set. Disqualified candidates can never become the winner.
+     */
+    disqualified: boolean;
+}
+/** All candidates tried in a single round, plus the round's best non-disqualified pick. */
+interface VariantExperimentRound {
+    /** 0-based round index. */
+    round: number;
+    /** Every candidate scored this round, in input order. */
+    candidates: VariantCandidateResult[];
+    /** Highest-scoring non-disqualified candidate this round, if any. */
+    best?: VariantCandidateResult;
+}
+/** Context passed to a `proposeVariants` callback before each round. */
+interface ProposeVariantsContext {
+    /** 0-based index of the round about to run. */
+    round: number;
+    /** The original baseline run (no overrides). */
+    baseline: EvalRunnerResult;
+    /** The metric the experiment is optimizing. */
+    metric: ExperimentMetric;
+    /** All completed rounds so far, in order. */
+    history: VariantExperimentRound[];
+    /** Best non-disqualified candidate across all prior rounds, if any. */
+    bestSoFar?: VariantCandidateResult;
+}
+/** A structured, ready-to-act proposal derived from the best attempted candidate. */
+interface VariantImprovementProposal {
+    /** `id` of the variant this proposal describes. */
+    variantId: string;
+    /** Metric the experiment optimized. */
+    metric: ExperimentMetric;
+    /** Baseline metric value. */
+    baselineValue: number;
+    /** Candidate metric value. */
+    candidateValue: number;
+    /** `candidateValue` minus `baselineValue`. */
+    delta: number;
+    /** Per-tool overrides this variant applied, keyed by canonical tool name. */
+    toolChanges: Record<string, ToolMetadataOverride>;
+    /** IDs of cases that failed in baseline and passed with this variant. */
+    improvedCaseIds: string[];
+    /** IDs of cases that passed in baseline and failed with this variant. */
+    regressedCaseIds: string[];
+    /**
+     * `apply` when the variant improved the metric without disqualifying
+     * regressions; `reject` when the best attempt regressed cases (and
+     * regressions are not allowed); `inconclusive` when nothing beat baseline.
+     */
+    recommendation: VariantRecommendation;
+}
+/** Options for {@link runVariantExperiment}. */
+interface VariantExperimentOptions {
+    /** The eval dataset. Treated as the stable behavioral contract; never mutated. */
+    dataset: EvalDataset;
+    /** Static candidates tried in round 0. */
+    variants?: ToolOverrideVariant[];
+    /**
+     * AI hook returning the next candidates given prior-round results. Invoked for
+     * rounds >= 1, and for round 0 when `variants` is omitted. Return `[]` to stop.
+     */
+    proposeVariants?: (context: ProposeVariantsContext) => Promise<ToolOverrideVariant[]>;
+    /** Metric to optimize. @default 'passRate' */
+    metric?: ExperimentMetric;
+    /** Maximum number of rounds to run. @default 1 */
+    maxRounds?: number;
+    /**
+     * Convergence threshold. Stop when a round's best metric improvement over the
+     * prior best-so-far is below this value. @default 0
+     */
+    minImprovement?: number;
+    /**
+     * When false (default), any candidate that regresses a case is disqualified
+     * from winning and surfaced with `recommendation: 'reject'`. When true,
+     * regressions do not disqualify.
+     * @default false
+     */
+    allowRegressions?: boolean;
+    /** Default `mcp_host` iterations per case. Forwarded to `runEvalDataset`. */
+    defaultLlmIterations?: number;
+    /** Default judge repetitions per case. Forwarded to `runEvalDataset`. */
+    defaultJudgeReps?: number;
+    /** Max eval cases to run concurrently within each run. Forwarded to `runEvalDataset`. */
+    concurrency?: number;
+    /** Run only cases with at least one of these tags. Forwarded to `runEvalDataset`. */
+    filterTags?: string[];
+    /** Schema registry for `expect.schema` cases. Forwarded to `runEvalDataset`. */
+    schemas?: Record<string, ZodType>;
+    /** MCP host model identifier recorded in run metadata. */
+    mcpHostModel?: string;
+    /** Judge model identifier recorded in run metadata. */
+    judgeModel?: string;
+}
+/** Aggregated result of a variant experiment. */
+interface VariantExperimentResult {
+    /** Metric that was optimized. */
+    metric: ExperimentMetric;
+    /** The original baseline run (no overrides). */
+    baseline: EvalRunnerResult;
+    /** Every round that ran, in order. */
+    rounds: VariantExperimentRound[];
+    /** Best non-disqualified candidate across all rounds, if any. */
+    winner?: VariantCandidateResult;
+    /** Structured proposal derived from the best attempted candidate, if any ran. */
+    proposal?: VariantImprovementProposal;
+    /** True when the experiment stopped on its own terms (always true today). */
+    converged: boolean;
+    /** Why the experiment stopped. */
+    reason: VariantExperimentReason;
+}
+/**
+ * Runs a tool-metadata variant experiment: establishes a baseline, then injects
+ * each candidate variant via `toolOverrides`, compares it to the baseline,
+ * ranks by the chosen metric, guards against regressions, and emits a structured
+ * improvement proposal.
+ *
+ * The library owns the experiment mechanism; the *policy* — which variant to try
+ * next — is the caller's, supplied either as a static `variants` list or an
+ * iterative `proposeVariants` callback. This is the programmatic spine an AI or
+ * skill drives to optimize tool descriptions/schemas for better host triggering.
+ *
+ * Candidates are always compared against the original baseline (not the prior
+ * round), so the resulting proposal is directly applicable. Multi-round
+ * convergence is tracked separately via best-so-far.
+ *
+ * @example
+ * ```typescript
+ * const result = await runVariantExperiment(
+ *   { dataset, variants: [variantA, variantB], metric: 'passRate' },
+ *   { mcp, testInfo }
+ * );
+ * if (result.proposal?.recommendation === 'apply') {
+ *   console.log('Apply:', result.winner?.variant.id, '+', result.proposal.delta);
+ * }
+ * ```
+ */
+declare function runVariantExperiment(options: VariantExperimentOptions, context: EvalContext): Promise<VariantExperimentResult>;
+
+/**
+ * Options for saveBaseline
+ */
+interface SaveBaselineOptions {
+    /**
+     * When true (default), strips the `response` field from each case result
+     * before saving. Keeps baseline files small and git-friendly — the baseline
+     * is a pass/fail record and the full response is not needed for comparison.
+     *
+     * Set to false to preserve the complete response in the saved file.
+     *
+     * @default true
+     */
+    omitResponses?: boolean;
+}
+/**
+ * Saves eval results to a JSON file for use as a baseline in future runs.
+ *
+ * @param result - The eval run result to save
+ * @param filePath - Path to write the JSON file (parent dirs created automatically)
+ * @param options - Save options
+ */
+declare function saveBaseline(result: EvalRunnerResult, filePath: string, options?: SaveBaselineOptions): Promise<void>;
+/**
+ * Loads a previously saved baseline from a JSON file.
+ *
+ * @param filePath - Path to the JSON file written by saveBaseline
+ * @returns The saved EvalRunnerResult
+ * @throws If the file cannot be read or parsed
+ */
+declare function loadBaseline(filePath: string): Promise<EvalRunnerResult>;
+
+/**
+ * Custom Judge Registry
+ *
+ * Allows consumers to register named judge executors that can be referenced
+ * by string ID in eval fixtures and programmatic tests. This enables
+ * multi-step judge pipelines (LLM call + post-processing), custom scoring
+ * logic, and reusable judge configurations without duplicating rubrics.
+ */
+/**
+ * Result returned by a custom judge executor.
+ *
+ * Custom judges must return a normalized score (0–1). The framework applies
+ * the caller's `threshold` (default 0.7) to determine pass/fail. This keeps
+ * judges reusable — the same judge can be used with different thresholds in
+ * different tests.
+ */
+interface CustomJudgeResult {
+    /** Normalized score (0–1, where 1 is best) */
+    score: number;
+    /** Optional reasoning/explanation */
+    reasoning?: string;
+}
+/**
+ * A user-defined judge executor function.
+ *
+ * Custom executors own their entire evaluation pipeline — prompt construction,
+ * LLM calls, and post-processing — but return a normalized score. The framework
+ * determines pass/fail by comparing the score against the caller's threshold.
+ *
+ * @param candidate - The actual response to evaluate
+ * @param reference - Optional reference/expected response
+ * @returns Evaluation result with a normalized score and optional reasoning
+ *
+ * @example
+ * ```typescript
+ * const completenessJudge: CustomJudgeExecutor = async (candidate, reference) => {
+ *   // Step 1: LLM call with your own prompt and schema
+ *   const llmResult = await callLLM(COMPLETENESS_PROMPT, candidate);
+ *   const { verdict, reasoning } = JSON.parse(llmResult);
+ *
+ *   // Step 2: Deterministic post-processing into a normalized score
+ *   const score = { Complete: 1.0, Incomplete: 0.5 }[verdict] ?? 0.0;
+ *
+ *   return { score, reasoning };
+ * };
+ * ```
+ */
+type CustomJudgeExecutor = (candidate: unknown, reference?: unknown) => Promise<CustomJudgeResult>;
+/**
+ * Registers a named custom judge executor.
+ *
+ * Call this in your test setup (e.g., `playwright.config.ts` or a global setup file)
+ * before tests run. The name can then be referenced in JSON eval fixtures via the
+ * `judge` field on `passesJudge`.
+ *
+ * @param name - Unique identifier for the judge
+ * @param executor - The judge executor function
+ * @throws {Error} If a judge with the same name is already registered
+ *
+ * @example
+ * ```typescript
+ * import { registerJudge } from '@gleanwork/mcp-server-tester';
+ *
+ * registerJudge('glean-completeness', async (candidate, reference) => {
+ *   // Step 1: LLM call with your own prompt and schema
+ *   const llmResult = await callLLM(COMPLETENESS_PROMPT, candidate);
+ *   const { verdict, reasoning } = JSON.parse(llmResult);
+ *
+ *   // Step 2: Deterministic post-processing into a normalized score
+ *   const score = { Complete: 1.0, Incomplete: 0.5 }[verdict] ?? 0.0;
+ *
+ *   return { score, reasoning };
+ * });
+ *
+ * // Then in tests — same judge, different thresholds:
+ * // expect(result).toPassToolJudge({ judge: 'glean-completeness', passingThreshold: 0.8 });
+ * // expect(result).toPassToolJudge({ judge: 'glean-completeness', passingThreshold: 0.5 });
+ * ```
+ */
+declare function registerJudge(name: string, executor: CustomJudgeExecutor): void;
+/**
+ * Retrieves a registered custom judge executor by name.
+ *
+ * @param name - The judge name to look up
+ * @returns The registered executor
+ * @throws {Error} If no judge with the given name is registered
+ */
+declare function getRegisteredJudge(name: string): CustomJudgeExecutor;
+/**
+ * Clears all registered judges. Intended for test teardown.
+ */
+declare function clearJudgeRegistry(): void;
+
+/**
+ * Options for conformance checks
+ */
+interface MCPConformanceOptions {
+    /**
+     * List of tools that must be present
+     */
+    requiredTools?: Array<string>;
+    /**
+     * Whether to validate tool schemas
+     * @default true
+     */
+    validateSchemas?: boolean;
+    /**
+     * Whether to check server info is present
+     * @default true
+     */
+    checkServerInfo?: boolean;
+    /**
+     * Whether to check resources capability (if declared by server)
+     * @default true
+     */
+    checkResources?: boolean;
+    /**
+     * Whether to check prompts capability (if declared by server)
+     * @default true
+     */
+    checkPrompts?: boolean;
+}
+/**
+ * Raw MCP responses for snapshotting
+ */
+interface MCPConformanceRaw {
+    /**
+     * Server info (name, version)
+     * null if not available
+     */
+    serverInfo: Implementation | null;
+    /**
+     * Server capabilities
+     * null if not available
+     */
+    capabilities: ServerCapabilities | null;
+    /**
+     * List of tools from the server
+     */
+    tools: Tool[];
+    /**
+     * List of resources from the server
+     * null if server doesn't declare resources capability
+     */
+    resources: Resource[] | null;
+    /**
+     * List of prompts from the server
+     * null if server doesn't declare prompts capability
+     */
+    prompts: Prompt[] | null;
+}
+/**
+ * Result of conformance checks
+ */
+interface MCPConformanceResult {
+    /**
+     * Whether all checks passed
+     */
+    pass: boolean;
+    /**
+     * List of check results
+     */
+    checks: MCPConformanceCheck[];
+    /**
+     * Raw MCP responses for snapshotting
+     *
+     * @example
+     * ```typescript
+     * const result = await runConformanceChecks(mcp);
+     * expect(result.raw.tools).toMatchSnapshot();
+     * expect(result.raw.capabilities).toMatchSnapshot();
+     * ```
+     */
+    raw: MCPConformanceRaw;
+}
+/**
+ * Runs MCP protocol conformance checks
+ *
+ * Validates that the MCP server conforms to expected protocol behavior.
+ * Returns both assertion results and raw MCP responses for snapshotting.
+ *
+ * When testInfo is provided, results are automatically attached for the MCP reporter.
+ *
+ * @param mcp - MCP fixture API
+ * @param options - Conformance check options
+ * @param testInfo - Optional Playwright TestInfo for reporter integration
+ * @returns Conformance check results with raw responses
+ *
+ * @example
+ * ```typescript
+ * // Basic usage
+ * const result = await runConformanceChecks(mcp, {
+ *   requiredTools: ['get_weather', 'search_docs'],
+ *   validateSchemas: true,
+ * });
+ *
+ * // Check assertions
+ * expect(result.pass).toBe(true);
+ *
+ * // With reporter integration (recommended in Playwright tests)
+ * const result = await runConformanceChecks(mcp, {
+ *   requiredTools: ['search'],
+ * }, testInfo);
+ *
+ * // Snapshot raw responses
+ * expect(result.raw.tools).toMatchSnapshot();
+ * expect(result.raw.capabilities).toMatchSnapshot();
+ * ```
+ */
+declare function runConformanceChecks(mcp: MCPFixtureApi, options?: MCPConformanceOptions, testInfo?: TestInfo): Promise<MCPConformanceResult>;
+
+/**
+ * Canonical type definitions for @gleanwork/mcp-server-tester
+ *
+ * This module is the single source of truth for shared types.
+ * All other modules should import from here rather than defining their own.
+ *
+ * @packageDocumentation
+ */
+/**
+ * Authentication type for MCP connections
+ *
+ * - 'oauth': Interactive OAuth 2.1 with PKCE (browser-based authentication)
+ * - 'api-token': Static API token (e.g., from a dashboard or environment variable)
+ * - 'none': No authentication
+ */
+type AuthType = 'oauth' | 'api-token' | 'none';
+/**
+ * Source of test results
+ *
+ * - 'eval': From runEvalDataset() using JSON eval datasets
+ * - 'test': From direct API test tracking (MCP fixture calls)
+ */
+type ResultSource = 'eval' | 'test';
+/**
+ * Known expectation types supported by the framework
+ */
+type ExpectationType = 'exact' | 'schema' | 'textContains' | 'regex' | 'snapshot' | 'judge' | 'error' | 'size' | 'toolsTriggered' | 'toolCallCount';
+/**
+ * Result of an expectation check
+ */
+interface EvalExpectationResult {
+    /**
+     * Whether the expectation passed
+     */
+    pass: boolean;
+    /**
+     * Optional details about the result
+     */
+    details?: string;
+    /**
+     * Judge score (0-1). Populated for passesJudge expectations.
+     */
+    score?: number;
+    /**
+     * Judge reasoning. Populated for passesJudge expectations.
+     */
+    reasoning?: string;
+    /**
+     * Judge name — rubric name (e.g. 'correctness') or custom judge name.
+     * Populated for passesJudge expectations.
+     */
+    judgeName?: string;
+    /**
+     * Judge provider used. Populated for passesJudge expectations.
+     */
+    judgeProvider?: string;
+    /**
+     * Judge model used. Populated for passesJudge expectations.
+     */
+    judgeModel?: string;
+    /**
+     * Per-judge breakdown when multiple judges are used.
+     * Each entry contains the individual judge's result.
+     * Only populated when passesJudge is an array with 2+ entries.
+     */
+    judgeResults?: EvalExpectationResult[];
+}
+/**
+ * Map of expectation type to result
+ */
+type ExpectationResultMap = Partial<Record<ExpectationType, EvalExpectationResult>>;
+/**
+ * Breakdown of expectation types used in a run
+ */
+type ExpectationBreakdown = Partial<Record<ExpectationType, number>>;
+
+export { type EvalRunComparisonResult as $, type AuthType as A, BUILT_IN_RUBRICS as B, type CLIConfig as C, DiscoveryError as D, ENV_VAR_NAMES as E, type EvalCaseRequest as F, type EvalCaseResult as G, EvalCaseSchema as H, type EvalContext as I, type JudgeMatcherOptions as J, type EvalDataset as K, type LLMProvider as L, type MCPFixtureApi as M, EvalDatasetSchema as N, type OAuthSetupConfig as O, type PatternValidatorOptions as P, type EvalExpectBlock as Q, type RubricSpec as R, type SchemaValidatorOptions as S, type TextValidatorOptions as T, type EvalExpectationResult as U, type ValidationResult as V, type EvalMode as W, type EvalResultStore as X, type EvalResultStoreConfig as Y, type EvalResultStoreLike as Z, type EvalRunComparisonLabels as _, type MCPHostConfig as a, type StoredEvalArtifact as a$, type EvalRunMetadata as a0, type EvalRunnerOptions as a1, type EvalRunnerResult as a2, type ExpectationBreakdown as a3, type ExpectationResultMap as a4, type ExpectationType as a5, type ExperimentMetric as a6, type FieldRemovalSanitizer as a7, FileEvalResultStore as a8, type FileEvalResultStoreConfig as a9, type MCPHostCapabilities as aA, type MCPHostSimulator as aB, type MCPOAuthConfig as aC, type MCPServerCapabilitiesData as aD, MCP_PROTOCOL_VERSION as aE, type NormalizedToolResponse as aF, PlaywrightOAuthClientProvider as aG, type PlaywrightOAuthClientProviderConfig as aH, type PredicateResult as aI, type ProposeVariantsContext as aJ, type ProtectedResourceDiscoveryResult as aK, type ProtectedResourceMetadata as aL, type ProviderKind as aM, type RegexSanitizer as aN, type ResultSource as aO, type SaveBaselineOptions as aP, type SaveEvalRunComparisonOptions as aQ, type SaveServerComparisonOptions as aR, type SchemaRegistry as aS, type SerializedEvalDataset as aT, type ServerComparisonOptions as aU, type ServerComparisonResult as aV, SnapshotSanitizers as aW, type StdioMCPConfig as aX, type StoredArtifactKind as aY, type StoredArtifactSummary as aZ, type StoredClientInfo as a_, GCSEvalResultStore as aa, type GCSEvalResultStoreConfig as ab, type HostType as ac, type HttpMCPConfig as ad, type IterationResult as ae, type JudgeExpectConfig as af, type JudgeResult as ag, type JudgeValidatorConfig as ah, type LLMToolCall as ai, type ListStoredArtifactsOptions as aj, type LoadDatasetOptions as ak, type MCPAuthConfig as al, type MCPAuthFixtures as am, type MCPClientCredentialsConfig as an, type MCPConfig as ao, MCPConfigSchema as ap, type MCPConformanceCheck as aq, type MCPConformanceOptions as ar, type MCPConformanceRaw as as, type MCPConformanceResult as at, type MCPConformanceResultData as au, type MCPEvalData as av, type MCPEvalHistoricalSummary as aw, type MCPEvalReporterConfig as ax, type MCPEvalRunData as ay, type MCPFixtureOptions as az, type MCPHostSimulationResult as b, validateToolCallCount as b$, type StoredEvalArtifactMetadata as b0, type StoredEvalResultLoadOptions as b1, type StoredEvalResultRef as b2, type StoredEvalResultSaveOptions as b3, type StoredEvalRunRef as b4, type StoredOAuthState as b5, type StoredServerMetadata as b6, type StoredTokens as b7, type TokenResult as b8, type ToolMetadataOverride as b9, isHttpConfig as bA, isStdioConfig as bB, loadBaseline as bC, loadEvalDataset as bD, loadEvalDatasetFromObject as bE, loadStoredEvalRunnerResult as bF, loadTokens as bG, loadTokensFromEnv as bH, test as bI, normalizeToolResponse as bJ, performClientCredentialsFlow as bK, refreshAccessToken as bL, registerJudge as bM, resolveEvalResultStore as bN, resolveRubric as bO, runConformanceChecks as bP, runEvalCase as bQ, runEvalDataset as bR, runServerComparison as bS, runVariantExperiment as bT, saveBaseline as bU, saveEvalRunComparison as bV, saveServerComparison as bW, validateEvalCase as bX, validateEvalDataset as bY, validateJudge as bZ, validateMCPConfig as b_, type ToolOverrideVariant as ba, type UsageMetrics as bb, type VariantCandidateResult as bc, type VariantExperimentOptions as bd, type VariantExperimentReason as be, type VariantExperimentResult as bf, type VariantExperimentRound as bg, type VariantImprovementProposal as bh, type VariantRecommendation as bi, clearJudgeRegistry as bj, closeMCPClient as bk, compareEvalRuns as bl, createDefaultArtifactId as bm, createEvalResultStore as bn, createMCPClientForConfig as bo, createMCPFixture as bp, createStoredEvalArtifact as bq, defaultEnvironmentMetadata as br, discoverAuthorizationServer as bs, discoverProtectedResource as bt, extractText as bu, getRegisteredJudge as bv, hasValidTokens as bw, injectTokens as bx, isBuiltInRubric as by, isEvalResultStore as bz, type SizeValidatorOptions as c, validateToolCalls as c0, type SnapshotSanitizer as d, type ToolPredicate as e, type ToolCallExpectation as f, type ToolCallCountOptions as g, type JudgeConfig as h, type Judge as i, type BuiltInRubric as j, type BuiltInSanitizer as k, CLIOAuthClient as l, type CLIOAuthClientConfig as m, type CLIOAuthResult as n, type CLIOutputFormat as o, type CaseComparisonResult as p, type ClientCredentialsConfig as q, type CompareEvalRunsOptions as r, type ComparisonOutcome as s, type ContentBlock as t, type CreateMCPClientOptions as u, type CustomJudgeExecutor as v, type CustomJudgeResult as w, type EvalCase as x, type EvalCaseComparison as y, type EvalCaseComparisonOutcome as z };