@gleanwork/mcp-server-tester 1.0.1-beta.0 → 1.1.0

package/dist/types/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/assertions/validators/types.ts"],"names":[],"mappings":";;;AA0EO,IAAM,kBAAA,GAAqB;AAAA;AAAA,EAEhC,SAAA,EAAW,WAAA;AAAA;AAAA,EAEX,IAAA,EAAM,MAAA;AAAA;AAAA,EAEN,QAAA,EAAU,UAAA;AAAA;AAAA,EAEV,SAAA,EAAW,UAAA;AAAA;AAAA,EAEX,GAAA,EAAK;AACP","file":"index.cjs","sourcesContent":["/**\n * Validator Types\n *\n * Core types for the unified assertion architecture.\n * These types are used by both Playwright matchers and the eval runner.\n */\n\nimport type { ZodType } from 'zod';\n\n/**\n * Result of a validation operation\n */\nexport interface ValidationResult {\n  /** Whether the validation passed */\n  pass: boolean;\n  /** Human-readable message explaining the result */\n  message: string;\n  /** Additional structured details about the validation */\n  details?: Record<string, unknown>;\n  /**\n   * Optional quantitative metrics from the validation.\n   * Populated by validateToolCalls for precision/recall.\n   */\n  metrics?: {\n    precision?: number;\n    recall?: number;\n  };\n}\n\n/**\n * Options for text validation\n */\nexport interface TextValidatorOptions {\n  /** Whether to perform case-sensitive matching (default: true) */\n  caseSensitive?: boolean;\n}\n\n/**\n * Options for response size validation\n */\nexport interface SizeValidatorOptions {\n  /** Maximum allowed size in bytes */\n  maxBytes?: number;\n  /** Minimum required size in bytes */\n  minBytes?: number;\n}\n\n/**\n * Options for schema validation\n */\nexport interface SchemaValidatorOptions {\n  /** Whether to use strict mode (fail on extra properties) */\n  strict?: boolean;\n}\n\n/**\n * Options for pattern validation\n */\nexport interface PatternValidatorOptions {\n  /** Whether to perform case-sensitive matching (default: true) */\n  caseSensitive?: boolean;\n}\n\n/**\n * Built-in snapshot sanitizer names for use with toMatchToolSnapshot.\n * Pass these values in the sanitizers array to replace non-deterministic\n * values with stable placeholders before snapshot comparison.\n *\n * @example\n * expect(result).toMatchToolSnapshot('my-snapshot', [\n *   SnapshotSanitizers.UUID,\n *   SnapshotSanitizers.ISO_DATE,\n * ]);\n */\nexport const SnapshotSanitizers = {\n  /** Replaces Unix timestamps (seconds and milliseconds) with a stable placeholder */\n  TIMESTAMP: 'timestamp' as const,\n  /** Replaces UUID v1-v5 strings with a stable placeholder */\n  UUID: 'uuid' as const,\n  /** Replaces ISO 8601 date/datetime strings with a stable placeholder */\n  ISO_DATE: 'iso-date' as const,\n  /** Replaces MongoDB ObjectId strings with a stable placeholder */\n  OBJECT_ID: 'objectId' as const,\n  /** Replaces JWT tokens with a stable placeholder */\n  JWT: 'jwt' as const,\n} as const;\n\n/**\n * Built-in sanitizer names for common variable patterns\n */\nexport type BuiltInSanitizer =\n  (typeof SnapshotSanitizers)[keyof typeof SnapshotSanitizers];\n\n/**\n * Custom regex-based sanitizer\n */\nexport interface RegexSanitizer {\n  /** Regex pattern to match */\n  pattern: string | RegExp;\n  /** Replacement string (default: \"[SANITIZED]\") */\n  replacement?: string;\n}\n\n/**\n * Field removal sanitizer - removes specified fields from objects\n */\nexport interface FieldRemovalSanitizer {\n  /** Field paths to remove (supports dot notation for nested fields) */\n  remove: string[];\n}\n\n/**\n * Snapshot sanitizer configuration\n *\n * Sanitizers transform response data before snapshot comparison,\n * allowing variable content (timestamps, IDs, etc.) to be normalized.\n *\n * Can be:\n * - A built-in sanitizer name: 'timestamp', 'uuid', 'iso-date', 'objectId', 'jwt'\n * - A regex sanitizer: { pattern: /regex/, replacement: '[REPLACED]' }\n * - A field removal sanitizer: { remove: ['field1', 'nested.field'] }\n */\nexport type SnapshotSanitizer =\n  | BuiltInSanitizer\n  | RegexSanitizer\n  | FieldRemovalSanitizer;\n\n/**\n * Schema registry for named schemas in datasets\n */\nexport type SchemaRegistry = Record<string, ZodType>;\n"]}

package/dist/types/index.d.cts

@@ -0,0 +1,9 @@
+export { A as AuthType, j as BuiltInRubric, k as BuiltInSanitizer, C as CLIConfig, m as CLIOAuthClientConfig, n as CLIOAuthResult, o as CLIOutputFormat, p as CaseComparisonResult, q as ClientCredentialsConfig, r as CompareEvalRunsOptions, s as ComparisonOutcome, t as ContentBlock, u as CreateMCPClientOptions, v as CustomJudgeExecutor, w as CustomJudgeResult, x as EvalCase, y as EvalCaseComparison, z as EvalCaseComparisonOutcome, F as EvalCaseRequest, G as EvalCaseResult, I as EvalContext, K as EvalDataset, Q as EvalExpectBlock, U as EvalExpectationResult, W as EvalMode, _ as EvalRunComparisonLabels, $ as EvalRunComparisonResult, a0 as EvalRunMetadata, a1 as EvalRunnerOptions, a2 as EvalRunnerResult, a3 as ExpectationBreakdown, a4 as ExpectationResultMap, a5 as ExpectationType, a6 as ExperimentMetric, a7 as FieldRemovalSanitizer, ac as HostType, ad as HttpMCPConfig, ae as IterationResult, i as Judge, h as JudgeConfig, af as JudgeExpectConfig, J as JudgeMatcherOptions, ag as JudgeResult, ah as JudgeValidatorConfig, L as LLMProvider, ai as LLMToolCall, ak as LoadDatasetOptions, al as MCPAuthConfig, am as MCPAuthFixtures, an as MCPClientCredentialsConfig, ao as MCPConfig, aq as MCPConformanceCheck, ar as MCPConformanceOptions, as as MCPConformanceRaw, at as MCPConformanceResult, au as MCPConformanceResultData, av as MCPEvalData, aw as MCPEvalHistoricalSummary, ax as MCPEvalReporterConfig, ay as MCPEvalRunData, M as MCPFixtureApi, az as MCPFixtureOptions, aA as MCPHostCapabilities, a as MCPHostConfig, b as MCPHostSimulationResult, aB as MCPHostSimulator, aC as MCPOAuthConfig, aD as MCPServerCapabilitiesData, aF as NormalizedToolResponse, O as OAuthSetupConfig, P as PatternValidatorOptions, aH as PlaywrightOAuthClientProviderConfig, aI as PredicateResult, aJ as ProposeVariantsContext, aK as ProtectedResourceDiscoveryResult, aL as ProtectedResourceMetadata, aM as ProviderKind, aN as RegexSanitizer, aO as ResultSource, R as RubricSpec, aP as SaveBaselineOptions, aQ as SaveEvalRunComparisonOptions, aR as SaveServerComparisonOptions, aS as SchemaRegistry, S as SchemaValidatorOptions, aT as SerializedEvalDataset, aU as ServerComparisonOptions, aV as ServerComparisonResult, c as SizeValidatorOptions, d as SnapshotSanitizer, aW as SnapshotSanitizers, aX as StdioMCPConfig, a_ as StoredClientInfo, b1 as StoredEvalResultLoadOptions, b2 as StoredEvalResultRef, b3 as StoredEvalResultSaveOptions, b4 as StoredEvalRunRef, b5 as StoredOAuthState, b6 as StoredServerMetadata, b7 as StoredTokens, T as TextValidatorOptions, b8 as TokenResult, g as ToolCallCountOptions, f as ToolCallExpectation, b9 as ToolMetadataOverride, ba as ToolOverrideVariant, e as ToolPredicate, bb as UsageMetrics, V as ValidationResult, bc as VariantCandidateResult, bd as VariantExperimentOptions, be as VariantExperimentReason, bf as VariantExperimentResult, bg as VariantExperimentRound, bh as VariantImprovementProposal, bi as VariantRecommendation } from '../index-BcUWzQCx.cjs';
+import 'zod';
+import '@playwright/test';
+import '@modelcontextprotocol/sdk/client/auth.js';
+import '@modelcontextprotocol/sdk/shared/auth.js';
+import 'oauth4webapi';
+import '@modelcontextprotocol/sdk/client/index.js';
+import '@modelcontextprotocol/sdk/types.js';
+import 'playwright/test';

package/dist/types/index.d.ts

@@ -0,0 +1,9 @@
+export { A as AuthType, j as BuiltInRubric, k as BuiltInSanitizer, C as CLIConfig, m as CLIOAuthClientConfig, n as CLIOAuthResult, o as CLIOutputFormat, p as CaseComparisonResult, q as ClientCredentialsConfig, r as CompareEvalRunsOptions, s as ComparisonOutcome, t as ContentBlock, u as CreateMCPClientOptions, v as CustomJudgeExecutor, w as CustomJudgeResult, x as EvalCase, y as EvalCaseComparison, z as EvalCaseComparisonOutcome, F as EvalCaseRequest, G as EvalCaseResult, I as EvalContext, K as EvalDataset, Q as EvalExpectBlock, U as EvalExpectationResult, W as EvalMode, _ as EvalRunComparisonLabels, $ as EvalRunComparisonResult, a0 as EvalRunMetadata, a1 as EvalRunnerOptions, a2 as EvalRunnerResult, a3 as ExpectationBreakdown, a4 as ExpectationResultMap, a5 as ExpectationType, a6 as ExperimentMetric, a7 as FieldRemovalSanitizer, ac as HostType, ad as HttpMCPConfig, ae as IterationResult, i as Judge, h as JudgeConfig, af as JudgeExpectConfig, J as JudgeMatcherOptions, ag as JudgeResult, ah as JudgeValidatorConfig, L as LLMProvider, ai as LLMToolCall, ak as LoadDatasetOptions, al as MCPAuthConfig, am as MCPAuthFixtures, an as MCPClientCredentialsConfig, ao as MCPConfig, aq as MCPConformanceCheck, ar as MCPConformanceOptions, as as MCPConformanceRaw, at as MCPConformanceResult, au as MCPConformanceResultData, av as MCPEvalData, aw as MCPEvalHistoricalSummary, ax as MCPEvalReporterConfig, ay as MCPEvalRunData, M as MCPFixtureApi, az as MCPFixtureOptions, aA as MCPHostCapabilities, a as MCPHostConfig, b as MCPHostSimulationResult, aB as MCPHostSimulator, aC as MCPOAuthConfig, aD as MCPServerCapabilitiesData, aF as NormalizedToolResponse, O as OAuthSetupConfig, P as PatternValidatorOptions, aH as PlaywrightOAuthClientProviderConfig, aI as PredicateResult, aJ as ProposeVariantsContext, aK as ProtectedResourceDiscoveryResult, aL as ProtectedResourceMetadata, aM as ProviderKind, aN as RegexSanitizer, aO as ResultSource, R as RubricSpec, aP as SaveBaselineOptions, aQ as SaveEvalRunComparisonOptions, aR as SaveServerComparisonOptions, aS as SchemaRegistry, S as SchemaValidatorOptions, aT as SerializedEvalDataset, aU as ServerComparisonOptions, aV as ServerComparisonResult, c as SizeValidatorOptions, d as SnapshotSanitizer, aW as SnapshotSanitizers, aX as StdioMCPConfig, a_ as StoredClientInfo, b1 as StoredEvalResultLoadOptions, b2 as StoredEvalResultRef, b3 as StoredEvalResultSaveOptions, b4 as StoredEvalRunRef, b5 as StoredOAuthState, b6 as StoredServerMetadata, b7 as StoredTokens, T as TextValidatorOptions, b8 as TokenResult, g as ToolCallCountOptions, f as ToolCallExpectation, b9 as ToolMetadataOverride, ba as ToolOverrideVariant, e as ToolPredicate, bb as UsageMetrics, V as ValidationResult, bc as VariantCandidateResult, bd as VariantExperimentOptions, be as VariantExperimentReason, bf as VariantExperimentResult, bg as VariantExperimentRound, bh as VariantImprovementProposal, bi as VariantRecommendation } from '../index-BcUWzQCx.js';
+import 'zod';
+import '@playwright/test';
+import '@modelcontextprotocol/sdk/client/auth.js';
+import '@modelcontextprotocol/sdk/shared/auth.js';
+import 'oauth4webapi';
+import '@modelcontextprotocol/sdk/client/index.js';
+import '@modelcontextprotocol/sdk/types.js';
+import 'playwright/test';

package/dist/types/index.js

@@ -0,0 +1,17 @@
+// src/assertions/validators/types.ts
+var SnapshotSanitizers = {
+  /** Replaces Unix timestamps (seconds and milliseconds) with a stable placeholder */
+  TIMESTAMP: "timestamp",
+  /** Replaces UUID v1-v5 strings with a stable placeholder */
+  UUID: "uuid",
+  /** Replaces ISO 8601 date/datetime strings with a stable placeholder */
+  ISO_DATE: "iso-date",
+  /** Replaces MongoDB ObjectId strings with a stable placeholder */
+  OBJECT_ID: "objectId",
+  /** Replaces JWT tokens with a stable placeholder */
+  JWT: "jwt"
+};
+
+export { SnapshotSanitizers };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map

package/dist/types/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/assertions/validators/types.ts"],"names":[],"mappings":";AA0EO,IAAM,kBAAA,GAAqB;AAAA;AAAA,EAEhC,SAAA,EAAW,WAAA;AAAA;AAAA,EAEX,IAAA,EAAM,MAAA;AAAA;AAAA,EAEN,QAAA,EAAU,UAAA;AAAA;AAAA,EAEV,SAAA,EAAW,UAAA;AAAA;AAAA,EAEX,GAAA,EAAK;AACP","file":"index.js","sourcesContent":["/**\n * Validator Types\n *\n * Core types for the unified assertion architecture.\n * These types are used by both Playwright matchers and the eval runner.\n */\n\nimport type { ZodType } from 'zod';\n\n/**\n * Result of a validation operation\n */\nexport interface ValidationResult {\n  /** Whether the validation passed */\n  pass: boolean;\n  /** Human-readable message explaining the result */\n  message: string;\n  /** Additional structured details about the validation */\n  details?: Record<string, unknown>;\n  /**\n   * Optional quantitative metrics from the validation.\n   * Populated by validateToolCalls for precision/recall.\n   */\n  metrics?: {\n    precision?: number;\n    recall?: number;\n  };\n}\n\n/**\n * Options for text validation\n */\nexport interface TextValidatorOptions {\n  /** Whether to perform case-sensitive matching (default: true) */\n  caseSensitive?: boolean;\n}\n\n/**\n * Options for response size validation\n */\nexport interface SizeValidatorOptions {\n  /** Maximum allowed size in bytes */\n  maxBytes?: number;\n  /** Minimum required size in bytes */\n  minBytes?: number;\n}\n\n/**\n * Options for schema validation\n */\nexport interface SchemaValidatorOptions {\n  /** Whether to use strict mode (fail on extra properties) */\n  strict?: boolean;\n}\n\n/**\n * Options for pattern validation\n */\nexport interface PatternValidatorOptions {\n  /** Whether to perform case-sensitive matching (default: true) */\n  caseSensitive?: boolean;\n}\n\n/**\n * Built-in snapshot sanitizer names for use with toMatchToolSnapshot.\n * Pass these values in the sanitizers array to replace non-deterministic\n * values with stable placeholders before snapshot comparison.\n *\n * @example\n * expect(result).toMatchToolSnapshot('my-snapshot', [\n *   SnapshotSanitizers.UUID,\n *   SnapshotSanitizers.ISO_DATE,\n * ]);\n */\nexport const SnapshotSanitizers = {\n  /** Replaces Unix timestamps (seconds and milliseconds) with a stable placeholder */\n  TIMESTAMP: 'timestamp' as const,\n  /** Replaces UUID v1-v5 strings with a stable placeholder */\n  UUID: 'uuid' as const,\n  /** Replaces ISO 8601 date/datetime strings with a stable placeholder */\n  ISO_DATE: 'iso-date' as const,\n  /** Replaces MongoDB ObjectId strings with a stable placeholder */\n  OBJECT_ID: 'objectId' as const,\n  /** Replaces JWT tokens with a stable placeholder */\n  JWT: 'jwt' as const,\n} as const;\n\n/**\n * Built-in sanitizer names for common variable patterns\n */\nexport type BuiltInSanitizer =\n  (typeof SnapshotSanitizers)[keyof typeof SnapshotSanitizers];\n\n/**\n * Custom regex-based sanitizer\n */\nexport interface RegexSanitizer {\n  /** Regex pattern to match */\n  pattern: string | RegExp;\n  /** Replacement string (default: \"[SANITIZED]\") */\n  replacement?: string;\n}\n\n/**\n * Field removal sanitizer - removes specified fields from objects\n */\nexport interface FieldRemovalSanitizer {\n  /** Field paths to remove (supports dot notation for nested fields) */\n  remove: string[];\n}\n\n/**\n * Snapshot sanitizer configuration\n *\n * Sanitizers transform response data before snapshot comparison,\n * allowing variable content (timestamps, IDs, etc.) to be normalized.\n *\n * Can be:\n * - A built-in sanitizer name: 'timestamp', 'uuid', 'iso-date', 'objectId', 'jwt'\n * - A regex sanitizer: { pattern: /regex/, replacement: '[REPLACED]' }\n * - A field removal sanitizer: { remove: ['field1', 'nested.field'] }\n */\nexport type SnapshotSanitizer =\n  | BuiltInSanitizer\n  | RegexSanitizer\n  | FieldRemovalSanitizer;\n\n/**\n * Schema registry for named schemas in datasets\n */\nexport type SchemaRegistry = Record<string, ZodType>;\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gleanwork/mcp-server-tester",
-  "version": "1.0.1-beta.0",
+  "version": "1.1.0",
   "description": "Playwright-based testing and evaluation framework for MCP servers",
   "keywords": [
     "playwright",
@@ -28,6 +28,11 @@
       "import": "./dist/index.js",
       "require": "./dist/index.cjs"
     },
+    "./types": {
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/types/index.js",
+      "require": "./dist/types/index.cjs"
+    },
     "./fixtures/mcp": {
       "types": "./dist/fixtures/mcp.d.ts",
       "import": "./dist/fixtures/mcp.js"
@@ -104,6 +109,7 @@
     "lucide-react": "^0.460.0",
     "markdown-code": "^0.6.1",
     "memfs": "^4.51.1",
+    "nock": "^14.0.10",
     "postcss": "^8.4.49",
     "prettier": "^3.4.2",
     "react-dom": "^18.3.1",
@@ -183,6 +189,7 @@
     }
   },
   "optionalDependencies": {
+    "@google-cloud/storage": "^7.17.1",
     "@ai-sdk/anthropic": "^3.0.46",
     "@ai-sdk/azure": "^3.0.0",
     "@ai-sdk/deepseek": "^2.0.0",