@gleanwork/mcp-server-tester 1.0.0-beta.0 → 1.0.0-beta.2

package/dist/index.cjs

@@ -1836,11 +1836,11 @@ function parseNullableDef(def, refs) {
       ]
     };
   }
-  const base2 = parseDef(def.innerType._def, {
+  const base3 = parseDef(def.innerType._def, {
     ...refs,
     currentPath: [...refs.currentPath, "anyOf", "0"]
   });
-  return base2 && { anyOf: [base2, { type: "null" }] };
+  return base3 && { anyOf: [base3, { type: "null" }] };
 }
 function parseNumberDef(def) {
   const res = {
@@ -3160,6 +3160,7 @@ var StdioConfigSchema = zod.z.object({
   command: zod.z.string().min(1, "command is required for stdio transport"),
   args: zod.z.array(zod.z.string()).optional(),
   cwd: zod.z.string().optional(),
+  env: zod.z.record(zod.z.string(), zod.z.string()).optional(),
   capabilities: MCPHostCapabilitiesSchema.optional(),
   connectTimeoutMs: zod.z.number().positive().optional(),
   requestTimeoutMs: zod.z.number().positive().optional(),
@@ -4406,7 +4407,7 @@ function escapeHtml(text) {
 
 // package.json
 var package_default = {
-  version: "1.0.0-beta.0"};
+  version: "1.0.0-beta.2"};
 
 // src/mcp/clientFactory.ts
 function getRetryAfterDelayMs(err) {
@@ -4478,7 +4479,14 @@ async function createMCPClientForConfig(config, options) {
       args: validatedConfig.args ?? [],
       ...validatedConfig.cwd && { cwd: validatedConfig.cwd },
       // Suppress server stderr when quiet mode is enabled
-      ...validatedConfig.quiet && { stderr: "ignore" }
+      ...validatedConfig.quiet && { stderr: "ignore" },
+      ...validatedConfig.env && {
+        env: Object.fromEntries(
+          Object.entries({ ...process.env, ...validatedConfig.env }).filter(
+            (entry) => entry[1] !== void 0
+          )
+        )
+      }
     });
     debugClient("Connecting via stdio: %O", {
       command: validatedConfig.command,
@@ -4617,7 +4625,10 @@ async function closeMCPClient(client) {
   try {
     await client.close();
   } catch (error) {
-    console.error("[MCP] Error closing client:", error);
+    debugClient(
+      "Error closing client: %s",
+      error instanceof Error ? error.message : String(error)
+    );
     throw error;
   } finally {
     const agent = agentRegistry.get(client);
@@ -5147,7 +5158,7 @@ function validateToolCalls(response, expectation) {
   ).length;
   const recall = requiredCalls.length > 0 ? calledRequiredCount / requiredCalls.length : 1;
   const allowedNames = new Set(expectation.calls.map((c) => c.name));
-  const precision = actual.length > 0 && expectation.exclusive === true ? actual.filter((c) => allowedNames.has(c.name)).length / actual.length : 1;
+  const precision = actual.length > 0 ? actual.filter((c) => allowedNames.has(c.name)).length / actual.length : 1;
   const metrics = { precision, recall };
   const order = expectation.order ?? "any";
   if (order === "strict") {
@@ -5550,9 +5561,8 @@ Validation errors: ${JSON.stringify(validation.error.issues)}`
 
 // src/judge/judgeClient.ts
 function createJudge(config = {}) {
-  const provider = config.provider ?? "claude";
+  const provider = config.provider ?? "anthropic";
   switch (provider) {
-    case "claude":
     case "anthropic":
       return createClaudeAgentJudge(config);
     case "openai":
@@ -5905,7 +5915,18 @@ function applySanitizers(value, sanitizers) {
       continue;
     }
     if (isRegexSanitizer(sanitizer)) {
-      const pattern = sanitizer.pattern instanceof RegExp ? sanitizer.pattern : new RegExp(sanitizer.pattern, "g");
+      let pattern;
+      if (sanitizer.pattern instanceof RegExp) {
+        pattern = sanitizer.pattern;
+      } else {
+        try {
+          pattern = new RegExp(sanitizer.pattern, "g");
+        } catch {
+          throw new Error(
+            `toMatchToolSnapshot: invalid regex pattern "${sanitizer.pattern}" in sanitizer`
+          );
+        }
+      }
       const replacement = sanitizer.replacement ?? "[SANITIZED]";
       result = result.replace(pattern, replacement);
       continue;
@@ -6209,6 +6230,106 @@ var test = test$1.test.extend({
     await use(api);
   }
 });
+
+// src/fixtures/mcpAuth.ts
+init_oauthClientProvider();
+var StaticTokenAuthProvider = class {
+  accessToken;
+  constructor(accessToken) {
+    this.accessToken = accessToken;
+  }
+  get redirectUrl() {
+    throw new Error("StaticTokenAuthProvider does not support OAuth redirects");
+  }
+  get clientMetadata() {
+    return {
+      redirect_uris: [],
+      token_endpoint_auth_method: "none",
+      grant_types: [],
+      response_types: [],
+      client_name: "@gleanwork/mcp-server-tester"
+    };
+  }
+  async clientInformation() {
+    return void 0;
+  }
+  async tokens() {
+    return {
+      access_token: this.accessToken,
+      token_type: "Bearer"
+    };
+  }
+  async saveTokens() {
+  }
+  async redirectToAuthorization() {
+    throw new Error("StaticTokenAuthProvider does not support OAuth redirects");
+  }
+  async saveCodeVerifier() {
+    throw new Error("StaticTokenAuthProvider does not support PKCE");
+  }
+  async codeVerifier() {
+    throw new Error("StaticTokenAuthProvider does not support PKCE");
+  }
+};
+var test2 = test$1.test.extend({
+  /**
+   * Create auth provider based on environment configuration
+   */
+  // eslint-disable-next-line no-empty-pattern
+  mcpAuthProvider: async ({}, use) => {
+    const authConfig = getAuthConfigFromEnv();
+    if (!authConfig) {
+      await use(void 0);
+      return;
+    }
+    if (authConfig.accessToken) {
+      const provider = new StaticTokenAuthProvider(authConfig.accessToken);
+      await use(provider);
+      return;
+    }
+    if (authConfig.oauth) {
+      const provider = createOAuthProvider(authConfig.oauth);
+      await use(provider);
+      return;
+    }
+    await use(void 0);
+  }
+});
+function createOAuthProvider(oauthConfig) {
+  if (!oauthConfig.authStatePath) {
+    throw new Error(
+      "OAuth configuration requires authStatePath. Use performOAuthSetup() in globalSetup to create auth state first."
+    );
+  }
+  const providerConfig = {
+    storagePath: oauthConfig.authStatePath,
+    redirectUri: oauthConfig.redirectUri ?? "http://localhost:3000/oauth/callback",
+    clientId: oauthConfig.clientId,
+    clientSecret: oauthConfig.clientSecret
+  };
+  return new exports.PlaywrightOAuthClientProvider(providerConfig);
+}
+function getAuthConfigFromEnv() {
+  const accessToken = process.env.MCP_ACCESS_TOKEN;
+  if (accessToken) {
+    return { accessToken };
+  }
+  const oauthServerUrl = process.env.MCP_OAUTH_SERVER_URL;
+  const authStatePath = process.env.MCP_AUTH_STATE_PATH;
+  if (oauthServerUrl || authStatePath) {
+    return {
+      oauth: {
+        serverUrl: oauthServerUrl ?? "",
+        authStatePath,
+        clientId: process.env.MCP_OAUTH_CLIENT_ID,
+        clientSecret: process.env.MCP_OAUTH_CLIENT_SECRET,
+        scopes: process.env.MCP_OAUTH_SCOPES?.split(","),
+        resource: process.env.MCP_OAUTH_RESOURCE
+      }
+    };
+  }
+  return void 0;
+}
 var LLMHostConfigSchema = zod.z.object({
   provider: zod.z.enum([
     "openai",
@@ -6216,7 +6337,6 @@ var LLMHostConfigSchema = zod.z.object({
     "azure",
     "google",
     "mistral",
-    "ollama",
     "deepseek",
     "openrouter",
     "xai",
@@ -6263,7 +6383,7 @@ var EvalExpectBlockSchema = zod.z.object({
     reference: zod.z.unknown().optional(),
     threshold: zod.z.number().min(0).max(1).optional(),
     reps: zod.z.number().int().min(1).optional(),
-    provider: zod.z.enum(["claude", "anthropic", "openai", "google"]).optional(),
+    provider: zod.z.enum(["anthropic", "openai", "google"]).optional(),
     model: zod.z.string().optional(),
     apiKeyEnvVar: zod.z.string().optional(),
     maxTokens: zod.z.number().int().positive().optional(),
@@ -6405,10 +6525,6 @@ async function loadModel(provider, model) {
       const { azure } = await import('@ai-sdk/azure');
       return azure(model);
     }
-    case "ollama": {
-      const { ollama } = await import('@ai-sdk/ollama');
-      return ollama(model);
-    }
     case "deepseek": {
       const { deepseek } = await import('@ai-sdk/deepseek');
       return deepseek(model);
@@ -6515,7 +6631,6 @@ var allProviders = [
   "azure",
   "google",
   "mistral",
-  "ollama",
   "deepseek",
   "openrouter",
   "xai",
@@ -6543,7 +6658,6 @@ function getMissingDependencyMessage(provider) {
     google: "npm install ai @ai-sdk/google",
     azure: "npm install ai @ai-sdk/azure",
     mistral: "npm install ai @ai-sdk/mistral",
-    ollama: "npm install ai @ai-sdk/ollama",
     deepseek: "npm install ai @ai-sdk/deepseek",
     openrouter: "npm install ai @openrouter/ai-sdk-provider",
     xai: "npm install ai @ai-sdk/xai",
@@ -6790,15 +6904,17 @@ async function runSingleIteration(evalCase, context, options) {
 function isInfrastructureError(err) {
   let name15;
   let msg;
+  let code = "";
   if (err instanceof Error) {
     name15 = err.name;
     msg = err.message.toLowerCase();
+    code = (err.code ?? "").toLowerCase();
   } else if (typeof err === "string") {
     msg = err.toLowerCase();
   } else {
     return false;
   }
-  return name15 === "AbortError" || msg.includes("econnreset") || msg.includes("etimedout") || msg.includes("econnrefused") || msg.includes("rate limit") || msg.includes("429") || msg.includes("503") || msg.includes("network");
+  return name15 === "AbortError" || msg.includes("econnreset") || msg.includes("etimedout") || msg.includes("econnrefused") || msg.includes("rate limit") || msg.includes("429") || msg.includes("503") || msg.includes("network") || code.includes("econnreset") || code.includes("etimedout") || code.includes("econnrefused");
 }
 async function runEvalCase(evalCase, context, options = {}) {
   const iterations = evalCase.iterations ?? 1;
@@ -6913,9 +7029,9 @@ async function runEvalDataset(options, context) {
     const withIterations = evalCase.mode === "llm_host" && evalCase.iterations === void 0 && defaultLlmIterations !== void 0 ? { ...evalCase, iterations: defaultLlmIterations } : evalCase;
     if (evalCase.mode === "llm_host") {
       const effectiveIterations = withIterations.iterations ?? 1;
-      if (effectiveIterations < 10) {
+      if (effectiveIterations > 1 && effectiveIterations < 10) {
         console.warn(
-          `[mcp-server-tester] Eval case "${evalCase.id}" uses llm_host mode with only ${effectiveIterations} iteration(s). The evals guide recommends >= 10 iterations. See docs/evals-guide.md for guidance on statistical reliability.`
+          `[mcp-server-tester] Eval case "${evalCase.id}": running ${effectiveIterations} iterations in llm_host mode may not be statistically reliable. Consider using 10+ iterations for accuracy measurements you can trust.`
         );
       }
     }
@@ -6963,6 +7079,16 @@ async function runEvalDataset(options, context) {
       const baseline = await loadBaseline(baselineResultsFrom);
       const baselinePassRate = baseline.total > 0 ? baseline.passed / baseline.total : 0;
       const baselineMap = buildBaselinePassMap(baseline);
+      const currentCaseIds = result.caseResults.map((cr) => cr.id);
+      const unmatchedCount = currentCaseIds.filter(
+        (id) => !baselineMap.has(id)
+      ).length;
+      const unmatchedRatio = currentCaseIds.length > 0 ? unmatchedCount / currentCaseIds.length : 0;
+      if (unmatchedRatio > 0.2) {
+        console.warn(
+          `[mcp-server-tester] Baseline comparison: ${unmatchedCount} of ${currentCaseIds.length} cases (${Math.round(unmatchedRatio * 100)}%) have no baseline entry. This may indicate the dataset structure has changed. Results for unmatched cases cannot be compared.`
+        );
+      }
       for (const cr of result.caseResults) {
         const baselinePass = baselineMap.get(cr.id);
         if (baselinePass !== void 0) {
@@ -7272,6 +7398,7 @@ exports.loadEvalDataset = loadEvalDataset;
 exports.loadEvalDatasetFromObject = loadEvalDatasetFromObject;
 exports.loadTokens = loadTokens;
 exports.loadTokensFromEnv = loadTokensFromEnv;
+exports.mcpAuthTest = test2;
 exports.normalizeToolResponse = normalizeToolResponse;
 exports.normalizeWhitespace = normalizeWhitespace;
 exports.performClientCredentialsFlow = performClientCredentialsFlow;