@gleanwork/mcp-server-tester 1.0.0-beta.0 → 1.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. package/dist/cli/index.js +14 -3
  2. package/dist/fixtures/mcp.d.ts +1 -1
  3. package/dist/fixtures/mcp.js +28 -7
  4. package/dist/fixtures/mcp.js.map +1 -1
  5. package/dist/index.cjs +147 -20
  6. package/dist/index.cjs.map +1 -1
  7. package/dist/index.d.cts +81 -49
  8. package/dist/index.d.ts +81 -49
  9. package/dist/index.js +147 -21
  10. package/dist/index.js.map +1 -1
  11. package/package.json +2 -1
package/dist/index.d.ts CHANGED
@@ -119,6 +119,11 @@ interface StdioMCPConfig {
119
119
  * Working directory for the command
120
120
  */
121
121
  cwd?: string;
122
+ /**
123
+ * Environment variables to pass to the subprocess.
124
+ * Merged with the current process environment.
125
+ */
126
+ env?: Record<string, string>;
122
127
  /**
123
128
  * Suppress stderr output from the server process.
124
129
  * When true, server stderr is ignored instead of inherited.
@@ -233,6 +238,7 @@ declare const MCPConfigSchema: z.ZodDiscriminatedUnion<"transport", [z.ZodObject
233
238
  command: z.ZodString;
234
239
  args: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
235
240
  cwd: z.ZodOptional<z.ZodString>;
241
+ env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
236
242
  capabilities: z.ZodOptional<z.ZodObject<{
237
243
  sampling: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
238
244
  roots: z.ZodOptional<z.ZodObject<{
@@ -262,6 +268,7 @@ declare const MCPConfigSchema: z.ZodDiscriminatedUnion<"transport", [z.ZodObject
262
268
  command: string;
263
269
  args?: string[] | undefined;
264
270
  cwd?: string | undefined;
271
+ env?: Record<string, string> | undefined;
265
272
  capabilities?: {
266
273
  sampling?: Record<string, unknown> | undefined;
267
274
  roots?: {
@@ -277,6 +284,7 @@ declare const MCPConfigSchema: z.ZodDiscriminatedUnion<"transport", [z.ZodObject
277
284
  command: string;
278
285
  args?: string[] | undefined;
279
286
  cwd?: string | undefined;
287
+ env?: Record<string, string> | undefined;
280
288
  capabilities?: {
281
289
  sampling?: Record<string, unknown> | undefined;
282
290
  roots?: {
@@ -1366,7 +1374,7 @@ interface CreateMCPClientOptions {
1366
1374
  * When absent, sampling is removed from declared capabilities so the client
1367
1375
  * does not falsely advertise support it cannot fulfill.
1368
1376
  */
1369
- samplingHandler?: unknown;
1377
+ samplingHandler?: (...args: unknown[]) => unknown;
1370
1378
  }
1371
1379
  /**
1372
1380
  * Creates and connects an MCP client based on the provided configuration
@@ -1874,14 +1882,14 @@ interface UsageMetrics {
1874
1882
  cacheCreationInputTokens?: number;
1875
1883
  }
1876
1884
  /** Valid LLM judge provider kinds. */
1877
- type ProviderKind = 'claude' | 'anthropic' | 'openai' | 'google';
1885
+ type ProviderKind = 'anthropic' | 'openai' | 'google';
1878
1886
  /**
1879
1887
  * Configuration for an LLM judge
1880
1888
  */
1881
1889
  interface JudgeConfig {
1882
1890
  /**
1883
1891
  * LLM provider to use
1884
- * @default 'claude'
1892
+ * @default 'anthropic'
1885
1893
  */
1886
1894
  provider?: ProviderKind;
1887
1895
  /**
@@ -2683,7 +2691,33 @@ type MCPFixtures = {
2683
2691
  * expect(tools.length).toBeGreaterThan(0);
2684
2692
  * });
2685
2693
  */
2686
- declare const test: playwright_test.TestType<playwright_test.PlaywrightTestArgs & playwright_test.PlaywrightTestOptions & MCPFixtures, playwright_test.PlaywrightWorkerArgs & playwright_test.PlaywrightWorkerOptions>;
2694
+ declare const test$1: playwright_test.TestType<playwright_test.PlaywrightTestArgs & playwright_test.PlaywrightTestOptions & MCPFixtures, playwright_test.PlaywrightWorkerArgs & playwright_test.PlaywrightWorkerOptions>;
2695
+
2696
+ /**
2697
+ * Test-scoped auth fixtures interface
2698
+ */
2699
+ interface MCPAuthFixtures {
2700
+ /**
2701
+ * OAuth client provider for MCP authentication
2702
+ */
2703
+ mcpAuthProvider: OAuthClientProvider | undefined;
2704
+ }
2705
+ /**
2706
+ * Extended Playwright test with MCP auth fixtures
2707
+ *
2708
+ * Use this when you need OAuth authentication for MCP server testing.
2709
+ *
2710
+ * @example
2711
+ * ```typescript
2712
+ * // test.ts
2713
+ * import { test } from '@gleanwork/mcp-server-tester/fixtures/mcpAuth';
2714
+ *
2715
+ * test('authenticated MCP call', async ({ mcpAuthProvider }) => {
2716
+ * // mcpAuthProvider can be passed to createMCPClientForConfig
2717
+ * });
2718
+ * ```
2719
+ */
2720
+ declare const test: playwright_test.TestType<playwright_test.PlaywrightTestArgs & playwright_test.PlaywrightTestOptions & MCPAuthFixtures, playwright_test.PlaywrightWorkerArgs & playwright_test.PlaywrightWorkerOptions>;
2687
2721
 
2688
2722
  /**
2689
2723
  * Types and interfaces for LLM host simulation mode
@@ -2703,12 +2737,11 @@ declare const test: playwright_test.TestType<playwright_test.PlaywrightTestArgs
2703
2737
  * google → npm install ai @ai-sdk/google
2704
2738
  * azure → npm install ai @ai-sdk/azure
2705
2739
  * mistral → npm install ai @ai-sdk/mistral
2706
- * ollama → npm install ai @ai-sdk/ollama (local, no API key)
2707
2740
  * deepseek → npm install ai @ai-sdk/deepseek
2708
2741
  * openrouter → npm install ai @openrouter/ai-sdk-provider
2709
2742
  * xai → npm install ai @ai-sdk/xai
2710
2743
  */
2711
- type LLMProvider = 'openai' | 'anthropic' | 'azure' | 'google' | 'mistral' | 'ollama' | 'deepseek' | 'openrouter' | 'xai'
2744
+ type LLMProvider = 'openai' | 'anthropic' | 'azure' | 'google' | 'mistral' | 'deepseek' | 'openrouter' | 'xai'
2712
2745
  /**
2713
2746
  * Anthropic Claude via Google Vertex AI.
2714
2747
  * Requires @ai-sdk/google-vertex and Application Default Credentials (gcloud auth).
@@ -2969,8 +3002,8 @@ interface EvalExpectBlock {
2969
3002
  threshold?: number;
2970
3003
  /** Number of judge evaluations for this assertion. Overrides EvalCase.judgeReps. */
2971
3004
  reps?: number;
2972
- /** Judge provider. @default 'claude' */
2973
- provider?: 'claude' | 'anthropic' | 'openai' | 'google';
3005
+ /** Judge provider. @default 'anthropic' */
3006
+ provider?: 'anthropic' | 'openai' | 'google';
2974
3007
  /** Model override (e.g., 'claude-opus-4-20250514') */
2975
3008
  model?: string;
2976
3009
  /** Environment variable name for API key */
@@ -3065,21 +3098,21 @@ declare const EvalCaseSchema: z.ZodObject<{
3065
3098
  args: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
3066
3099
  scenario: z.ZodOptional<z.ZodString>;
3067
3100
  llmHostConfig: z.ZodOptional<z.ZodObject<{
3068
- provider: z.ZodEnum<["openai", "anthropic", "azure", "google", "mistral", "ollama", "deepseek", "openrouter", "xai", "vertex-anthropic"]>;
3101
+ provider: z.ZodEnum<["openai", "anthropic", "azure", "google", "mistral", "deepseek", "openrouter", "xai", "vertex-anthropic"]>;
3069
3102
  apiKeyEnvVar: z.ZodOptional<z.ZodString>;
3070
3103
  model: z.ZodOptional<z.ZodString>;
3071
3104
  maxTokens: z.ZodOptional<z.ZodNumber>;
3072
3105
  temperature: z.ZodOptional<z.ZodNumber>;
3073
3106
  maxToolCalls: z.ZodOptional<z.ZodNumber>;
3074
3107
  }, "strip", z.ZodTypeAny, {
3075
- provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "ollama" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3108
+ provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3076
3109
  model?: string | undefined;
3077
3110
  maxTokens?: number | undefined;
3078
3111
  apiKeyEnvVar?: string | undefined;
3079
3112
  temperature?: number | undefined;
3080
3113
  maxToolCalls?: number | undefined;
3081
3114
  }, {
3082
- provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "ollama" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3115
+ provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3083
3116
  model?: string | undefined;
3084
3117
  maxTokens?: number | undefined;
3085
3118
  apiKeyEnvVar?: string | undefined;
@@ -3126,7 +3159,7 @@ declare const EvalCaseSchema: z.ZodObject<{
3126
3159
  reference: z.ZodOptional<z.ZodUnknown>;
3127
3160
  threshold: z.ZodOptional<z.ZodNumber>;
3128
3161
  reps: z.ZodOptional<z.ZodNumber>;
3129
- provider: z.ZodOptional<z.ZodEnum<["claude", "anthropic", "openai", "google"]>>;
3162
+ provider: z.ZodOptional<z.ZodEnum<["anthropic", "openai", "google"]>>;
3130
3163
  model: z.ZodOptional<z.ZodString>;
3131
3164
  apiKeyEnvVar: z.ZodOptional<z.ZodString>;
3132
3165
  maxTokens: z.ZodOptional<z.ZodNumber>;
@@ -3143,7 +3176,7 @@ declare const EvalCaseSchema: z.ZodObject<{
3143
3176
  reference?: unknown;
3144
3177
  threshold?: number | undefined;
3145
3178
  reps?: number | undefined;
3146
- provider?: "openai" | "anthropic" | "google" | "claude" | undefined;
3179
+ provider?: "openai" | "anthropic" | "google" | undefined;
3147
3180
  apiKeyEnvVar?: string | undefined;
3148
3181
  temperature?: number | undefined;
3149
3182
  maxToolOutputSize?: number | undefined;
@@ -3157,7 +3190,7 @@ declare const EvalCaseSchema: z.ZodObject<{
3157
3190
  reference?: unknown;
3158
3191
  threshold?: number | undefined;
3159
3192
  reps?: number | undefined;
3160
- provider?: "openai" | "anthropic" | "google" | "claude" | undefined;
3193
+ provider?: "openai" | "anthropic" | "google" | undefined;
3161
3194
  apiKeyEnvVar?: string | undefined;
3162
3195
  temperature?: number | undefined;
3163
3196
  maxToolOutputSize?: number | undefined;
@@ -3239,7 +3272,7 @@ declare const EvalCaseSchema: z.ZodObject<{
3239
3272
  } | undefined;
3240
3273
  containsText?: string | string[] | undefined;
3241
3274
  matchesPattern?: string | string[] | undefined;
3242
- snapshotSanitizers?: ("timestamp" | "uuid" | "iso-date" | "objectId" | "jwt" | {
3275
+ snapshotSanitizers?: ("uuid" | "jwt" | "timestamp" | "iso-date" | "objectId" | {
3243
3276
  pattern: string;
3244
3277
  replacement?: string | undefined;
3245
3278
  } | {
@@ -3255,7 +3288,7 @@ declare const EvalCaseSchema: z.ZodObject<{
3255
3288
  reference?: unknown;
3256
3289
  threshold?: number | undefined;
3257
3290
  reps?: number | undefined;
3258
- provider?: "openai" | "anthropic" | "google" | "claude" | undefined;
3291
+ provider?: "openai" | "anthropic" | "google" | undefined;
3259
3292
  apiKeyEnvVar?: string | undefined;
3260
3293
  temperature?: number | undefined;
3261
3294
  maxToolOutputSize?: number | undefined;
@@ -3285,7 +3318,7 @@ declare const EvalCaseSchema: z.ZodObject<{
3285
3318
  } | undefined;
3286
3319
  containsText?: string | string[] | undefined;
3287
3320
  matchesPattern?: string | string[] | undefined;
3288
- snapshotSanitizers?: ("timestamp" | "uuid" | "iso-date" | "objectId" | "jwt" | {
3321
+ snapshotSanitizers?: ("uuid" | "jwt" | "timestamp" | "iso-date" | "objectId" | {
3289
3322
  pattern: string;
3290
3323
  replacement?: string | undefined;
3291
3324
  } | {
@@ -3301,7 +3334,7 @@ declare const EvalCaseSchema: z.ZodObject<{
3301
3334
  reference?: unknown;
3302
3335
  threshold?: number | undefined;
3303
3336
  reps?: number | undefined;
3304
- provider?: "openai" | "anthropic" | "google" | "claude" | undefined;
3337
+ provider?: "openai" | "anthropic" | "google" | undefined;
3305
3338
  apiKeyEnvVar?: string | undefined;
3306
3339
  temperature?: number | undefined;
3307
3340
  maxToolOutputSize?: number | undefined;
@@ -3320,7 +3353,7 @@ declare const EvalCaseSchema: z.ZodObject<{
3320
3353
  toolName?: string | undefined;
3321
3354
  scenario?: string | undefined;
3322
3355
  llmHostConfig?: {
3323
- provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "ollama" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3356
+ provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3324
3357
  model?: string | undefined;
3325
3358
  maxTokens?: number | undefined;
3326
3359
  apiKeyEnvVar?: string | undefined;
@@ -3353,7 +3386,7 @@ declare const EvalCaseSchema: z.ZodObject<{
3353
3386
  } | undefined;
3354
3387
  containsText?: string | string[] | undefined;
3355
3388
  matchesPattern?: string | string[] | undefined;
3356
- snapshotSanitizers?: ("timestamp" | "uuid" | "iso-date" | "objectId" | "jwt" | {
3389
+ snapshotSanitizers?: ("uuid" | "jwt" | "timestamp" | "iso-date" | "objectId" | {
3357
3390
  pattern: string;
3358
3391
  replacement?: string | undefined;
3359
3392
  } | {
@@ -3369,7 +3402,7 @@ declare const EvalCaseSchema: z.ZodObject<{
3369
3402
  reference?: unknown;
3370
3403
  threshold?: number | undefined;
3371
3404
  reps?: number | undefined;
3372
- provider?: "openai" | "anthropic" | "google" | "claude" | undefined;
3405
+ provider?: "openai" | "anthropic" | "google" | undefined;
3373
3406
  apiKeyEnvVar?: string | undefined;
3374
3407
  temperature?: number | undefined;
3375
3408
  maxToolOutputSize?: number | undefined;
@@ -3388,7 +3421,7 @@ declare const EvalCaseSchema: z.ZodObject<{
3388
3421
  toolName?: string | undefined;
3389
3422
  scenario?: string | undefined;
3390
3423
  llmHostConfig?: {
3391
- provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "ollama" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3424
+ provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3392
3425
  model?: string | undefined;
3393
3426
  maxTokens?: number | undefined;
3394
3427
  apiKeyEnvVar?: string | undefined;
@@ -3421,7 +3454,7 @@ declare const EvalCaseSchema: z.ZodObject<{
3421
3454
  } | undefined;
3422
3455
  containsText?: string | string[] | undefined;
3423
3456
  matchesPattern?: string | string[] | undefined;
3424
- snapshotSanitizers?: ("timestamp" | "uuid" | "iso-date" | "objectId" | "jwt" | {
3457
+ snapshotSanitizers?: ("uuid" | "jwt" | "timestamp" | "iso-date" | "objectId" | {
3425
3458
  pattern: string;
3426
3459
  replacement?: string | undefined;
3427
3460
  } | {
@@ -3437,7 +3470,7 @@ declare const EvalCaseSchema: z.ZodObject<{
3437
3470
  reference?: unknown;
3438
3471
  threshold?: number | undefined;
3439
3472
  reps?: number | undefined;
3440
- provider?: "openai" | "anthropic" | "google" | "claude" | undefined;
3473
+ provider?: "openai" | "anthropic" | "google" | undefined;
3441
3474
  apiKeyEnvVar?: string | undefined;
3442
3475
  temperature?: number | undefined;
3443
3476
  maxToolOutputSize?: number | undefined;
@@ -3462,21 +3495,21 @@ declare const EvalDatasetSchema: z.ZodObject<{
3462
3495
  args: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
3463
3496
  scenario: z.ZodOptional<z.ZodString>;
3464
3497
  llmHostConfig: z.ZodOptional<z.ZodObject<{
3465
- provider: z.ZodEnum<["openai", "anthropic", "azure", "google", "mistral", "ollama", "deepseek", "openrouter", "xai", "vertex-anthropic"]>;
3498
+ provider: z.ZodEnum<["openai", "anthropic", "azure", "google", "mistral", "deepseek", "openrouter", "xai", "vertex-anthropic"]>;
3466
3499
  apiKeyEnvVar: z.ZodOptional<z.ZodString>;
3467
3500
  model: z.ZodOptional<z.ZodString>;
3468
3501
  maxTokens: z.ZodOptional<z.ZodNumber>;
3469
3502
  temperature: z.ZodOptional<z.ZodNumber>;
3470
3503
  maxToolCalls: z.ZodOptional<z.ZodNumber>;
3471
3504
  }, "strip", z.ZodTypeAny, {
3472
- provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "ollama" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3505
+ provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3473
3506
  model?: string | undefined;
3474
3507
  maxTokens?: number | undefined;
3475
3508
  apiKeyEnvVar?: string | undefined;
3476
3509
  temperature?: number | undefined;
3477
3510
  maxToolCalls?: number | undefined;
3478
3511
  }, {
3479
- provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "ollama" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3512
+ provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3480
3513
  model?: string | undefined;
3481
3514
  maxTokens?: number | undefined;
3482
3515
  apiKeyEnvVar?: string | undefined;
@@ -3523,7 +3556,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3523
3556
  reference: z.ZodOptional<z.ZodUnknown>;
3524
3557
  threshold: z.ZodOptional<z.ZodNumber>;
3525
3558
  reps: z.ZodOptional<z.ZodNumber>;
3526
- provider: z.ZodOptional<z.ZodEnum<["claude", "anthropic", "openai", "google"]>>;
3559
+ provider: z.ZodOptional<z.ZodEnum<["anthropic", "openai", "google"]>>;
3527
3560
  model: z.ZodOptional<z.ZodString>;
3528
3561
  apiKeyEnvVar: z.ZodOptional<z.ZodString>;
3529
3562
  maxTokens: z.ZodOptional<z.ZodNumber>;
@@ -3540,7 +3573,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3540
3573
  reference?: unknown;
3541
3574
  threshold?: number | undefined;
3542
3575
  reps?: number | undefined;
3543
- provider?: "openai" | "anthropic" | "google" | "claude" | undefined;
3576
+ provider?: "openai" | "anthropic" | "google" | undefined;
3544
3577
  apiKeyEnvVar?: string | undefined;
3545
3578
  temperature?: number | undefined;
3546
3579
  maxToolOutputSize?: number | undefined;
@@ -3554,7 +3587,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3554
3587
  reference?: unknown;
3555
3588
  threshold?: number | undefined;
3556
3589
  reps?: number | undefined;
3557
- provider?: "openai" | "anthropic" | "google" | "claude" | undefined;
3590
+ provider?: "openai" | "anthropic" | "google" | undefined;
3558
3591
  apiKeyEnvVar?: string | undefined;
3559
3592
  temperature?: number | undefined;
3560
3593
  maxToolOutputSize?: number | undefined;
@@ -3636,7 +3669,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3636
3669
  } | undefined;
3637
3670
  containsText?: string | string[] | undefined;
3638
3671
  matchesPattern?: string | string[] | undefined;
3639
- snapshotSanitizers?: ("timestamp" | "uuid" | "iso-date" | "objectId" | "jwt" | {
3672
+ snapshotSanitizers?: ("uuid" | "jwt" | "timestamp" | "iso-date" | "objectId" | {
3640
3673
  pattern: string;
3641
3674
  replacement?: string | undefined;
3642
3675
  } | {
@@ -3652,7 +3685,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3652
3685
  reference?: unknown;
3653
3686
  threshold?: number | undefined;
3654
3687
  reps?: number | undefined;
3655
- provider?: "openai" | "anthropic" | "google" | "claude" | undefined;
3688
+ provider?: "openai" | "anthropic" | "google" | undefined;
3656
3689
  apiKeyEnvVar?: string | undefined;
3657
3690
  temperature?: number | undefined;
3658
3691
  maxToolOutputSize?: number | undefined;
@@ -3682,7 +3715,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3682
3715
  } | undefined;
3683
3716
  containsText?: string | string[] | undefined;
3684
3717
  matchesPattern?: string | string[] | undefined;
3685
- snapshotSanitizers?: ("timestamp" | "uuid" | "iso-date" | "objectId" | "jwt" | {
3718
+ snapshotSanitizers?: ("uuid" | "jwt" | "timestamp" | "iso-date" | "objectId" | {
3686
3719
  pattern: string;
3687
3720
  replacement?: string | undefined;
3688
3721
  } | {
@@ -3698,7 +3731,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3698
3731
  reference?: unknown;
3699
3732
  threshold?: number | undefined;
3700
3733
  reps?: number | undefined;
3701
- provider?: "openai" | "anthropic" | "google" | "claude" | undefined;
3734
+ provider?: "openai" | "anthropic" | "google" | undefined;
3702
3735
  apiKeyEnvVar?: string | undefined;
3703
3736
  temperature?: number | undefined;
3704
3737
  maxToolOutputSize?: number | undefined;
@@ -3717,7 +3750,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3717
3750
  toolName?: string | undefined;
3718
3751
  scenario?: string | undefined;
3719
3752
  llmHostConfig?: {
3720
- provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "ollama" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3753
+ provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3721
3754
  model?: string | undefined;
3722
3755
  maxTokens?: number | undefined;
3723
3756
  apiKeyEnvVar?: string | undefined;
@@ -3750,7 +3783,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3750
3783
  } | undefined;
3751
3784
  containsText?: string | string[] | undefined;
3752
3785
  matchesPattern?: string | string[] | undefined;
3753
- snapshotSanitizers?: ("timestamp" | "uuid" | "iso-date" | "objectId" | "jwt" | {
3786
+ snapshotSanitizers?: ("uuid" | "jwt" | "timestamp" | "iso-date" | "objectId" | {
3754
3787
  pattern: string;
3755
3788
  replacement?: string | undefined;
3756
3789
  } | {
@@ -3766,7 +3799,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3766
3799
  reference?: unknown;
3767
3800
  threshold?: number | undefined;
3768
3801
  reps?: number | undefined;
3769
- provider?: "openai" | "anthropic" | "google" | "claude" | undefined;
3802
+ provider?: "openai" | "anthropic" | "google" | undefined;
3770
3803
  apiKeyEnvVar?: string | undefined;
3771
3804
  temperature?: number | undefined;
3772
3805
  maxToolOutputSize?: number | undefined;
@@ -3785,7 +3818,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3785
3818
  toolName?: string | undefined;
3786
3819
  scenario?: string | undefined;
3787
3820
  llmHostConfig?: {
3788
- provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "ollama" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3821
+ provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3789
3822
  model?: string | undefined;
3790
3823
  maxTokens?: number | undefined;
3791
3824
  apiKeyEnvVar?: string | undefined;
@@ -3818,7 +3851,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3818
3851
  } | undefined;
3819
3852
  containsText?: string | string[] | undefined;
3820
3853
  matchesPattern?: string | string[] | undefined;
3821
- snapshotSanitizers?: ("timestamp" | "uuid" | "iso-date" | "objectId" | "jwt" | {
3854
+ snapshotSanitizers?: ("uuid" | "jwt" | "timestamp" | "iso-date" | "objectId" | {
3822
3855
  pattern: string;
3823
3856
  replacement?: string | undefined;
3824
3857
  } | {
@@ -3834,7 +3867,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3834
3867
  reference?: unknown;
3835
3868
  threshold?: number | undefined;
3836
3869
  reps?: number | undefined;
3837
- provider?: "openai" | "anthropic" | "google" | "claude" | undefined;
3870
+ provider?: "openai" | "anthropic" | "google" | undefined;
3838
3871
  apiKeyEnvVar?: string | undefined;
3839
3872
  temperature?: number | undefined;
3840
3873
  maxToolOutputSize?: number | undefined;
@@ -3857,7 +3890,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3857
3890
  toolName?: string | undefined;
3858
3891
  scenario?: string | undefined;
3859
3892
  llmHostConfig?: {
3860
- provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "ollama" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3893
+ provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3861
3894
  model?: string | undefined;
3862
3895
  maxTokens?: number | undefined;
3863
3896
  apiKeyEnvVar?: string | undefined;
@@ -3890,7 +3923,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3890
3923
  } | undefined;
3891
3924
  containsText?: string | string[] | undefined;
3892
3925
  matchesPattern?: string | string[] | undefined;
3893
- snapshotSanitizers?: ("timestamp" | "uuid" | "iso-date" | "objectId" | "jwt" | {
3926
+ snapshotSanitizers?: ("uuid" | "jwt" | "timestamp" | "iso-date" | "objectId" | {
3894
3927
  pattern: string;
3895
3928
  replacement?: string | undefined;
3896
3929
  } | {
@@ -3906,7 +3939,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3906
3939
  reference?: unknown;
3907
3940
  threshold?: number | undefined;
3908
3941
  reps?: number | undefined;
3909
- provider?: "openai" | "anthropic" | "google" | "claude" | undefined;
3942
+ provider?: "openai" | "anthropic" | "google" | undefined;
3910
3943
  apiKeyEnvVar?: string | undefined;
3911
3944
  temperature?: number | undefined;
3912
3945
  maxToolOutputSize?: number | undefined;
@@ -3930,7 +3963,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3930
3963
  toolName?: string | undefined;
3931
3964
  scenario?: string | undefined;
3932
3965
  llmHostConfig?: {
3933
- provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "ollama" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3966
+ provider: "openai" | "anthropic" | "azure" | "google" | "mistral" | "deepseek" | "openrouter" | "xai" | "vertex-anthropic";
3934
3967
  model?: string | undefined;
3935
3968
  maxTokens?: number | undefined;
3936
3969
  apiKeyEnvVar?: string | undefined;
@@ -3963,7 +3996,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3963
3996
  } | undefined;
3964
3997
  containsText?: string | string[] | undefined;
3965
3998
  matchesPattern?: string | string[] | undefined;
3966
- snapshotSanitizers?: ("timestamp" | "uuid" | "iso-date" | "objectId" | "jwt" | {
3999
+ snapshotSanitizers?: ("uuid" | "jwt" | "timestamp" | "iso-date" | "objectId" | {
3967
4000
  pattern: string;
3968
4001
  replacement?: string | undefined;
3969
4002
  } | {
@@ -3979,7 +4012,7 @@ declare const EvalDatasetSchema: z.ZodObject<{
3979
4012
  reference?: unknown;
3980
4013
  threshold?: number | undefined;
3981
4014
  reps?: number | undefined;
3982
- provider?: "openai" | "anthropic" | "google" | "claude" | undefined;
4015
+ provider?: "openai" | "anthropic" | "google" | undefined;
3983
4016
  apiKeyEnvVar?: string | undefined;
3984
4017
  temperature?: number | undefined;
3985
4018
  maxToolOutputSize?: number | undefined;
@@ -4681,7 +4714,7 @@ declare function runServerComparison(options: ServerComparisonOptions, contextA:
4681
4714
  /**
4682
4715
  * LLM Host Simulation - Main entry point
4683
4716
  *
4684
- * All providers (openai, anthropic, google, azure, mistral, ollama, deepseek,
4717
+ * All providers (openai, anthropic, google, azure, mistral, deepseek,
4685
4718
  * openrouter, xai) run through the Vercel AI SDK orchestrator, which uses
4686
4719
  * generateText + stopWhen for a uniform multi-turn tool-calling loop with
4687
4720
  * built-in latency decomposition.
@@ -4692,7 +4725,6 @@ declare function runServerComparison(options: ServerComparisonOptions, contextA:
4692
4725
  * google → npm install ai @ai-sdk/google
4693
4726
  * azure → npm install ai @ai-sdk/azure
4694
4727
  * mistral → npm install ai @ai-sdk/mistral
4695
- * ollama → npm install ai @ai-sdk/ollama (local, no API key)
4696
4728
  * deepseek → npm install ai @ai-sdk/deepseek
4697
4729
  * openrouter → npm install ai @openrouter/ai-sdk-provider
4698
4730
  * xai → npm install ai @ai-sdk/xai
@@ -4944,4 +4976,4 @@ interface MCPEvalReporterConfig {
4944
4976
  includeAutoTracking?: boolean;
4945
4977
  }
4946
4978
 
4947
- export { type AuthType, BUILT_IN_RUBRICS, type BuiltInRubric, type BuiltInSanitizer, CLIOAuthClient, type CLIOAuthClientConfig, type CLIOAuthResult, type CaseComparisonResult, type ClientCredentialsConfig, type ComparisonOutcome, type ContentBlock, type CreateMCPClientOptions, DiscoveryError, ENV_VAR_NAMES, type EvalCase, type EvalCaseResult, EvalCaseSchema, type EvalContext, type EvalDataset, EvalDatasetSchema, type EvalExpectBlock, type EvalExpectationResult, type EvalMode, type EvalRunnerOptions, type EvalRunnerResult, type ExpectationBreakdown, type ExpectationResultMap, type ExpectationType, type FieldRemovalSanitizer, type HttpMCPConfig, type IterationResult, type Judge, type JudgeConfig, type JudgeMatcherOptions, type JudgeResult, type JudgeValidatorConfig, type LLMHostConfig, type LLMHostSimulationResult, type LLMHostSimulator, type LLMProvider, type LLMToolCall, type LoadDatasetOptions, type MCPAuthConfig, type MCPClientCredentialsConfig, type MCPConfig, MCPConfigSchema, type MCPConformanceCheck, type MCPConformanceOptions, type MCPConformanceRaw, type MCPConformanceResult, type MCPConformanceResultData, type MCPEvalData, type MCPEvalHistoricalSummary, type MCPEvalReporterConfig, type MCPEvalRunData, type MCPFixtureApi, type MCPFixtureOptions, type MCPHostCapabilities, type MCPOAuthConfig, type MCPServerCapabilitiesData, MCP_PROTOCOL_VERSION, type NormalizedToolResponse, type OAuthSetupConfig, type PatternValidatorOptions, PlaywrightOAuthClientProvider, type PlaywrightOAuthClientProviderConfig, type PredicateResult, type ProtectedResourceDiscoveryResult, type ProtectedResourceMetadata, type ProviderKind, type RegexSanitizer, type ResultSource, type RubricSpec, type SchemaRegistry, type SchemaValidatorOptions, type SerializedEvalDataset, type ServerComparisonOptions, type ServerComparisonResult, type SizeValidatorOptions, type SnapshotSanitizer, SnapshotSanitizers, type StdioMCPConfig, type StoredClientInfo, type StoredOAuthState, type StoredServerMetadata, type StoredTokens, type TextValidatorOptions, type TokenResult, type ToolCallCountOptions, type ToolCallExpectation, type ToolPredicate, type UsageMetrics, type ValidationResult, closeMCPClient, createJudge, createMCPClientForConfig, createMCPFixture, createTokenAuthHeaders, discoverAuthorizationServer, discoverProtectedResource, expect, extractText, getMissingDependencyMessage, getResponseSizeBytes, hasValidTokens, injectTokens, isBuiltInRubric, isHttpConfig, isProviderAvailable, isStdioConfig, isTokenExpired, isTokenExpiringSoon, loadBaseline, loadEvalDataset, loadEvalDatasetFromObject, loadTokens, loadTokensFromEnv, normalizeToolResponse, normalizeWhitespace, performClientCredentialsFlow, performOAuthSetup, performOAuthSetupIfNeeded, resolveRubric, runConformanceChecks, runEvalCase, runEvalDataset, runServerComparison, saveBaseline, simulateLLMHost, test, validateAccessToken, validateError, validateEvalCase, validateEvalDataset, validateJudge, validateMCPConfig, validatePattern, validateResponse, validateSchema, validateSize, validateText, validateToolCallCount, validateToolCalls };
4979
+ export { type AuthType, BUILT_IN_RUBRICS, type BuiltInRubric, type BuiltInSanitizer, CLIOAuthClient, type CLIOAuthClientConfig, type CLIOAuthResult, type CaseComparisonResult, type ClientCredentialsConfig, type ComparisonOutcome, type ContentBlock, type CreateMCPClientOptions, DiscoveryError, ENV_VAR_NAMES, type EvalCase, type EvalCaseResult, EvalCaseSchema, type EvalContext, type EvalDataset, EvalDatasetSchema, type EvalExpectBlock, type EvalExpectationResult, type EvalMode, type EvalRunnerOptions, type EvalRunnerResult, type ExpectationBreakdown, type ExpectationResultMap, type ExpectationType, type FieldRemovalSanitizer, type HttpMCPConfig, type IterationResult, type Judge, type JudgeConfig, type JudgeMatcherOptions, type JudgeResult, type JudgeValidatorConfig, type LLMHostConfig, type LLMHostSimulationResult, type LLMHostSimulator, type LLMProvider, type LLMToolCall, type LoadDatasetOptions, type MCPAuthConfig, type MCPAuthFixtures, type MCPClientCredentialsConfig, type MCPConfig, MCPConfigSchema, type MCPConformanceCheck, type MCPConformanceOptions, type MCPConformanceRaw, type MCPConformanceResult, type MCPConformanceResultData, type MCPEvalData, type MCPEvalHistoricalSummary, type MCPEvalReporterConfig, type MCPEvalRunData, type MCPFixtureApi, type MCPFixtureOptions, type MCPHostCapabilities, type MCPOAuthConfig, type MCPServerCapabilitiesData, MCP_PROTOCOL_VERSION, type NormalizedToolResponse, type OAuthSetupConfig, type PatternValidatorOptions, PlaywrightOAuthClientProvider, type PlaywrightOAuthClientProviderConfig, type PredicateResult, type ProtectedResourceDiscoveryResult, type ProtectedResourceMetadata, type ProviderKind, type RegexSanitizer, type ResultSource, type RubricSpec, type SchemaRegistry, type SchemaValidatorOptions, type SerializedEvalDataset, type ServerComparisonOptions, type ServerComparisonResult, type SizeValidatorOptions, type SnapshotSanitizer, SnapshotSanitizers, type StdioMCPConfig, type StoredClientInfo, type StoredOAuthState, type StoredServerMetadata, type StoredTokens, type TextValidatorOptions, type TokenResult, type ToolCallCountOptions, type ToolCallExpectation, type ToolPredicate, type UsageMetrics, type ValidationResult, closeMCPClient, createJudge, createMCPClientForConfig, createMCPFixture, createTokenAuthHeaders, discoverAuthorizationServer, discoverProtectedResource, expect, extractText, getMissingDependencyMessage, getResponseSizeBytes, hasValidTokens, injectTokens, isBuiltInRubric, isHttpConfig, isProviderAvailable, isStdioConfig, isTokenExpired, isTokenExpiringSoon, loadBaseline, loadEvalDataset, loadEvalDatasetFromObject, loadTokens, loadTokensFromEnv, test as mcpAuthTest, normalizeToolResponse, normalizeWhitespace, performClientCredentialsFlow, performOAuthSetup, performOAuthSetupIfNeeded, resolveRubric, runConformanceChecks, runEvalCase, runEvalDataset, runServerComparison, saveBaseline, simulateLLMHost, test$1 as test, validateAccessToken, validateError, validateEvalCase, validateEvalDataset, validateJudge, validateMCPConfig, validatePattern, validateResponse, validateSchema, validateSize, validateText, validateToolCallCount, validateToolCalls };