@gleanwork/mcp-server-tester 1.0.1 → 1.1.1

package/dist/reporters/mcpReporter.d.ts

@@ -1,10 +1,412 @@
 import { Reporter, FullConfig, Suite, TestCase, TestResult, FullResult } from '@playwright/test/reporter';
+import { ZodType } from 'zod';
 
 /**
- * Reporter types - re-exported from canonical source
+ * Validator Types
  *
- * This module re-exports types from the canonical types module for backwards compatibility.
- * All type definitions now live in src/types/.
+ * Core types for the unified assertion architecture.
+ * These types are used by both Playwright matchers and the eval runner.
+ */
+
+/**
+ * Options for text validation
+ */
+interface TextValidatorOptions {
+    /** Whether to perform case-sensitive matching (default: true) */
+    caseSensitive?: boolean;
+}
+/**
+ * Options for response size validation
+ */
+interface SizeValidatorOptions {
+    /** Maximum allowed size in bytes */
+    maxBytes?: number;
+    /** Minimum required size in bytes */
+    minBytes?: number;
+}
+/**
+ * Options for schema validation
+ */
+interface SchemaValidatorOptions {
+    /** Whether to use strict mode (fail on extra properties) */
+    strict?: boolean;
+}
+/**
+ * Options for pattern validation
+ */
+interface PatternValidatorOptions {
+    /** Whether to perform case-sensitive matching (default: true) */
+    caseSensitive?: boolean;
+}
+/**
+ * Built-in snapshot sanitizer names for use with toMatchToolSnapshot.
+ * Pass these values in the sanitizers array to replace non-deterministic
+ * values with stable placeholders before snapshot comparison.
+ *
+ * @example
+ * expect(result).toMatchToolSnapshot('my-snapshot', [
+ *   SnapshotSanitizers.UUID,
+ *   SnapshotSanitizers.ISO_DATE,
+ * ]);
+ */
+declare const SnapshotSanitizers: {
+    /** Replaces Unix timestamps (seconds and milliseconds) with a stable placeholder */
+    readonly TIMESTAMP: "timestamp";
+    /** Replaces UUID v1-v5 strings with a stable placeholder */
+    readonly UUID: "uuid";
+    /** Replaces ISO 8601 date/datetime strings with a stable placeholder */
+    readonly ISO_DATE: "iso-date";
+    /** Replaces MongoDB ObjectId strings with a stable placeholder */
+    readonly OBJECT_ID: "objectId";
+    /** Replaces JWT tokens with a stable placeholder */
+    readonly JWT: "jwt";
+};
+/**
+ * Built-in sanitizer names for common variable patterns
+ */
+type BuiltInSanitizer = (typeof SnapshotSanitizers)[keyof typeof SnapshotSanitizers];
+/**
+ * Custom regex-based sanitizer
+ */
+interface RegexSanitizer {
+    /** Regex pattern to match */
+    pattern: string | RegExp;
+    /** Replacement string (default: "[SANITIZED]") */
+    replacement?: string;
+}
+/**
+ * Field removal sanitizer - removes specified fields from objects
+ */
+interface FieldRemovalSanitizer {
+    /** Field paths to remove (supports dot notation for nested fields) */
+    remove: string[];
+}
+/**
+ * Snapshot sanitizer configuration
+ *
+ * Sanitizers transform response data before snapshot comparison,
+ * allowing variable content (timestamps, IDs, etc.) to be normalized.
+ *
+ * Can be:
+ * - A built-in sanitizer name: 'timestamp', 'uuid', 'iso-date', 'objectId', 'jwt'
+ * - A regex sanitizer: { pattern: /regex/, replacement: '[REPLACED]' }
+ * - A field removal sanitizer: { remove: ['field1', 'nested.field'] }
+ */
+type SnapshotSanitizer = BuiltInSanitizer | RegexSanitizer | FieldRemovalSanitizer;
+
+/**
+ * Tool call validators for mcp_host simulation results.
+ *
+ * These validators extract the tool call trace from an MCPHostSimulationResult
+ * and apply assertions against expected call lists and counts.
+ */
+
+interface ToolCallExpectation {
+    calls: Array<{
+        name: string;
+        arguments?: Record<string, unknown>;
+        required?: boolean;
+    }>;
+    order?: 'strict' | 'any';
+    exclusive?: boolean;
+}
+interface ToolCallCountOptions {
+    min?: number;
+    max?: number;
+    exact?: number;
+}
+
+/**
+ * Built-in judge rubrics matching Glean EvalV2's named judge types.
+ * Use these for consistent, standardized evaluations across teams.
+ *
+ * All built-in rubrics use a 5-point scale: 0.0 / 0.25 / 0.5 / 0.75 / 1.0
+ */
+type BuiltInRubric = 'correctness' | 'completeness' | 'groundedness' | 'instruction-following' | 'conciseness';
+/** A rubric specification: either a built-in named rubric or custom text. */
+type RubricSpec = BuiltInRubric | {
+    text: string;
+};
+
+/** Valid LLM judge provider kinds. */
+type ProviderKind = 'anthropic' | 'vertex-anthropic' | 'anthropic-agent-sdk' | 'openai' | 'google';
+
+/**
+ * Matcher Types
+ *
+ * TypeScript declarations for custom Playwright matchers.
+ */
+
+/**
+ * Options for the LLM judge matcher
+ */
+interface JudgeMatcherOptions {
+    /** Reference response to compare against */
+    reference?: unknown;
+    /** Score threshold for passing (default: 0.7) */
+    passingThreshold?: number;
+    /** Number of judge evaluations (scores averaged) */
+    reps?: number;
+    /** Override the judge provider */
+    provider?: ProviderKind;
+    /** Override the judge model */
+    model?: string;
+    /**
+     * Name of a registered custom judge executor.
+     * When set, the named judge handles the entire evaluation pipeline
+     * and its `pass` result is authoritative.
+     */
+    judge?: string;
+}
+/**
+ * Declaration merging for Playwright matchers
+ */
+declare global {
+    namespace PlaywrightTest {
+        interface Matchers<R, T = unknown> {
+            /**
+             * Validates that a response exactly matches the expected value
+             *
+             * @param expected - The expected response value
+             *
+             * @example
+             * ```typescript
+             * expect(result).toMatchToolResponse({ status: 'ok', count: 42 });
+             * ```
+             */
+            toMatchToolResponse(expected: unknown): R;
+            /**
+             * Validates that a response matches a Zod schema
+             *
+             * @param schema - The Zod schema to validate against
+             * @param options - Validation options
+             *
+             * @example
+             * ```typescript
+             * const WeatherSchema = z.object({
+             *   temperature: z.number(),
+             *   conditions: z.string(),
+             * });
+             * expect(result).toMatchToolSchema(WeatherSchema);
+             * ```
+             */
+            toMatchToolSchema(schema: ZodType, options?: SchemaValidatorOptions): R;
+            /**
+             * Validates that a response contains expected text substrings
+             *
+             * @param expected - Expected substring(s) to find
+             * @param options - Validation options
+             *
+             * @example
+             * ```typescript
+             * expect(result).toContainToolText('temperature');
+             * expect(result).toContainToolText(['temperature', 'conditions']);
+             * expect(result).toContainToolText('HELLO', { caseSensitive: false });
+             * ```
+             */
+            toContainToolText(expected: string | string[], options?: TextValidatorOptions): R;
+            /**
+             * Validates that a response matches regex patterns
+             *
+             * @param patterns - Expected pattern(s) to match
+             * @param options - Validation options
+             *
+             * @example
+             * ```typescript
+             * expect(result).toMatchToolPattern(/temperature: \d+/);
+             * expect(result).toMatchToolPattern(['temp: \\d+', 'humidity: \\d+%']);
+             * ```
+             */
+            toMatchToolPattern(patterns: string | RegExp | (string | RegExp)[], options?: PatternValidatorOptions): R;
+            /**
+             * Validates that a response matches a saved snapshot
+             *
+             * @param name - Snapshot name
+             * @param sanitizers - Optional sanitizers for non-deterministic values
+             *
+             * @example
+             * ```typescript
+             * expect(result).toMatchToolSnapshot('weather-response');
+             * expect(result).toMatchToolSnapshot('user-data', [
+             *   { pattern: /\d{4}-\d{2}-\d{2}/, replacement: '[DATE]' },
+             * ]);
+             * ```
+             */
+            toMatchToolSnapshot(name: string, sanitizers?: SnapshotSanitizer[]): Promise<R>;
+            /**
+             * Validates that a response is (or is not) an error
+             *
+             * @param expected - What to expect (true for error, false for success, string for specific message)
+             *
+             * @example
+             * ```typescript
+             * expect(result).toBeToolError();  // Expects any error
+             * expect(result).not.toBeToolError();  // Expects success
+             * expect(result).toBeToolError('File not found');  // Expects specific error
+             * ```
+             */
+            toBeToolError(expected?: boolean | string | string[]): R;
+            /**
+             * Validates that a response passes LLM-as-judge evaluation.
+             *
+             * Two call signatures:
+             * - With rubric: `toPassToolJudge(rubric, options?)` — built-in LLM judge
+             * - With named judge: `toPassToolJudge({ judge: 'name' })` — custom judge executor
+             *
+             * @example
+             * ```typescript
+             * // Built-in LLM judge with rubric
+             * expect(result).toPassToolJudge('Response should be helpful and accurate');
+             * expect(result).toPassToolJudge('correctness', {
+             *   reference: expectedOutput,
+             *   passingThreshold: 0.8,
+             * });
+             *
+             * // Named custom judge (registered via registerJudge)
+             * expect(result).toPassToolJudge({ judge: 'glean-completeness' });
+             * ```
+             */
+            toPassToolJudge(rubric: RubricSpec, options?: JudgeMatcherOptions): Promise<R>;
+            toPassToolJudge(options: JudgeMatcherOptions): Promise<R>;
+            toPassToolJudge(judges: Array<JudgeMatcherOptions & {
+                rubric?: RubricSpec;
+            }>): Promise<R>;
+            /**
+             * Validates that a response meets size constraints
+             *
+             * @param options - Size constraints (maxBytes, minBytes)
+             *
+             * @example
+             * ```typescript
+             * expect(result).toHaveToolResponseSize({ maxBytes: 10000 });
+             * expect(result).toHaveToolResponseSize({ minBytes: 100, maxBytes: 50000 });
+             * ```
+             */
+            toHaveToolResponseSize(options: SizeValidatorOptions): R;
+            /**
+             * Validates that a response satisfies a custom predicate function
+             *
+             * Use this as an escape hatch when built-in matchers don't cover your use case.
+             * The predicate receives both the raw response and extracted text for convenience.
+             *
+             * @param predicate - Function that validates the response
+             * @param description - Optional description for error messages
+             *
+             * @example
+             * ```typescript
+             * // Simple boolean predicate
+             * expect(result).toSatisfyToolPredicate((response) => {
+             *   return response.data?.items?.length > 0;
+             * });
+             *
+             * // Predicate with custom message
+             * expect(result).toSatisfyToolPredicate(
+             *   (response, text) => ({
+             *     pass: text.includes('success'),
+             *     message: 'Expected response to contain "success"',
+             *   }),
+             *   'success check'
+             * );
+             *
+             * // Async predicate
+             * expect(result).toSatisfyToolPredicate(async (response) => {
+             *   return await validateWithExternalService(response);
+             * });
+             * ```
+             */
+            toSatisfyToolPredicate(predicate: ToolPredicate, description?: string): Promise<R>;
+            /**
+             * Validates which tools the LLM called during a mcp_host simulation.
+             *
+             * @example
+             * ```typescript
+             * expect(simulationResult).toHaveToolCalls({
+             *   calls: [{ name: 'search', arguments: { query: 'hello' }, required: true }],
+             *   order: 'any',
+             * });
+             * ```
+             */
+            toHaveToolCalls(expectation: ToolCallExpectation): R;
+            /**
+             * Validates the number of tool calls made during a mcp_host simulation.
+             *
+             * @example
+             * ```typescript
+             * expect(simulationResult).toHaveToolCallCount({ min: 1, max: 3 });
+             * expect(simulationResult).toHaveToolCallCount({ exact: 2 });
+             * ```
+             */
+            toHaveToolCallCount(options: ToolCallCountOptions): R;
+        }
+    }
+}
+/**
+ * Predicate result returned by the user's predicate function
+ */
+interface PredicateResult {
+    /** Whether the predicate passed */
+    pass: boolean;
+    /** Message explaining the result (shown on failure) */
+    message?: string;
+}
+/**
+ * A predicate function that validates a response
+ */
+type ToolPredicate = (response: unknown, text: string) => boolean | PredicateResult | Promise<boolean | PredicateResult>;
+
+type StoredArtifactKind = 'eval-runner-result' | 'reporter-run' | 'eval-run-comparison' | 'server-comparison';
+interface StoredEvalArtifactMetadata {
+    datasetName?: string;
+    gitHash?: string;
+    branch?: string;
+    runNumber?: string;
+    trigger?: string;
+    packageVersion?: string;
+    toolOverrideVariantId?: string;
+    mcpHostModel?: string;
+    judgeModel?: string;
+    labels?: Record<string, string>;
+    [key: string]: unknown;
+}
+interface StoredEvalArtifact<T> {
+    schemaVersion: 1;
+    kind: StoredArtifactKind;
+    id: string;
+    createdAt: string;
+    metadata: StoredEvalArtifactMetadata;
+    data: T;
+}
+interface StoredArtifactSummary {
+    kind: StoredArtifactKind;
+    id: string;
+    createdAt: string;
+    metadata: StoredEvalArtifactMetadata;
+}
+interface ListStoredArtifactsOptions {
+    limit?: number;
+}
+interface EvalResultStore {
+    saveArtifact<T>(artifact: StoredEvalArtifact<T>): Promise<void>;
+    loadArtifact<T>(kind: StoredArtifactKind, id: string): Promise<StoredEvalArtifact<T>>;
+    loadLatestArtifact<T>(kind: StoredArtifactKind): Promise<StoredEvalArtifact<T> | null>;
+    listArtifacts(kind: StoredArtifactKind, options?: ListStoredArtifactsOptions): Promise<StoredArtifactSummary[]>;
+}
+interface FileEvalResultStoreConfig {
+    provider: 'file';
+    dir: string;
+}
+interface GCSEvalResultStoreConfig {
+    provider: 'gcs';
+    bucket: string;
+    prefix?: string;
+}
+type EvalResultStoreConfig = FileEvalResultStoreConfig | GCSEvalResultStoreConfig;
+type EvalResultStoreLike = EvalResultStore | EvalResultStoreConfig;
+
+/**
+ * Reporter-specific type definitions
+ *
+ * These types are used by the MCP reporter and UI.
  *
  * @packageDocumentation
  */
@@ -41,6 +443,25 @@ interface MCPEvalReporterConfig {
      * @default true
      */
     includeAutoTracking?: boolean;
+    /**
+     * Optional external result store for durable reporter run history.
+     */
+    resultStore?: EvalResultStoreLike;
+    /**
+     * Optional run ID for externally stored reporter results.
+     * Defaults to a generated timestamp-based ID.
+     */
+    runId?: string;
+    /**
+     * Extra metadata to attach to externally stored reporter results.
+     */
+    runMetadata?: Record<string, unknown>;
+    /**
+     * When true, strips response payloads before storing reporter results externally.
+     * Local report output is unchanged.
+     * @default true
+     */
+    redactStoredResponses?: boolean;
 }
 
 /**
@@ -67,6 +488,7 @@ declare class MCPReporter implements Reporter {
     private allResults;
     private conformanceChecks;
     private serverCapabilities;
+    private variantExperiment;
     constructor(options?: MCPEvalReporterConfig);
     private log;
     private logError;
@@ -83,6 +505,8 @@ declare class MCPReporter implements Reporter {
     private buildRunData;
     private loadHistoricalData;
     private saveRunData;
+    private saveRunDataToStore;
+    private loadHistoricalDataFromStore;
     private cleanupOldRuns;
     private openReport;
 }