@gleanwork/mcp-server-tester 1.0.0-beta.0 → 1.0.0-beta.2

package/dist/index.js

@@ -1809,11 +1809,11 @@ function parseNullableDef(def, refs) {
       ]
     };
   }
-  const base2 = parseDef(def.innerType._def, {
+  const base3 = parseDef(def.innerType._def, {
     ...refs,
     currentPath: [...refs.currentPath, "anyOf", "0"]
   });
-  return base2 && { anyOf: [base2, { type: "null" }] };
+  return base3 && { anyOf: [base3, { type: "null" }] };
 }
 function parseNumberDef(def) {
   const res = {
@@ -3133,6 +3133,7 @@ var StdioConfigSchema = z.object({
   command: z.string().min(1, "command is required for stdio transport"),
   args: z.array(z.string()).optional(),
   cwd: z.string().optional(),
+  env: z.record(z.string(), z.string()).optional(),
   capabilities: MCPHostCapabilitiesSchema.optional(),
   connectTimeoutMs: z.number().positive().optional(),
   requestTimeoutMs: z.number().positive().optional(),
@@ -4379,7 +4380,7 @@ function escapeHtml(text) {
 
 // package.json
 var package_default = {
-  version: "1.0.0-beta.0"};
+  version: "1.0.0-beta.2"};
 
 // src/mcp/clientFactory.ts
 function getRetryAfterDelayMs(err) {
@@ -4451,7 +4452,14 @@ async function createMCPClientForConfig(config, options) {
       args: validatedConfig.args ?? [],
       ...validatedConfig.cwd && { cwd: validatedConfig.cwd },
       // Suppress server stderr when quiet mode is enabled
-      ...validatedConfig.quiet && { stderr: "ignore" }
+      ...validatedConfig.quiet && { stderr: "ignore" },
+      ...validatedConfig.env && {
+        env: Object.fromEntries(
+          Object.entries({ ...process.env, ...validatedConfig.env }).filter(
+            (entry) => entry[1] !== void 0
+          )
+        )
+      }
     });
     debugClient("Connecting via stdio: %O", {
       command: validatedConfig.command,
@@ -4590,7 +4598,10 @@ async function closeMCPClient(client) {
   try {
     await client.close();
   } catch (error) {
-    console.error("[MCP] Error closing client:", error);
+    debugClient(
+      "Error closing client: %s",
+      error instanceof Error ? error.message : String(error)
+    );
     throw error;
   } finally {
     const agent = agentRegistry.get(client);
@@ -5120,7 +5131,7 @@ function validateToolCalls(response, expectation) {
   ).length;
   const recall = requiredCalls.length > 0 ? calledRequiredCount / requiredCalls.length : 1;
   const allowedNames = new Set(expectation.calls.map((c) => c.name));
-  const precision = actual.length > 0 && expectation.exclusive === true ? actual.filter((c) => allowedNames.has(c.name)).length / actual.length : 1;
+  const precision = actual.length > 0 ? actual.filter((c) => allowedNames.has(c.name)).length / actual.length : 1;
   const metrics = { precision, recall };
   const order = expectation.order ?? "any";
   if (order === "strict") {
@@ -5523,9 +5534,8 @@ Validation errors: ${JSON.stringify(validation.error.issues)}`
 
 // src/judge/judgeClient.ts
 function createJudge(config = {}) {
-  const provider = config.provider ?? "claude";
+  const provider = config.provider ?? "anthropic";
   switch (provider) {
-    case "claude":
     case "anthropic":
       return createClaudeAgentJudge(config);
     case "openai":
@@ -5878,7 +5888,18 @@ function applySanitizers(value, sanitizers) {
       continue;
     }
     if (isRegexSanitizer(sanitizer)) {
-      const pattern = sanitizer.pattern instanceof RegExp ? sanitizer.pattern : new RegExp(sanitizer.pattern, "g");
+      let pattern;
+      if (sanitizer.pattern instanceof RegExp) {
+        pattern = sanitizer.pattern;
+      } else {
+        try {
+          pattern = new RegExp(sanitizer.pattern, "g");
+        } catch {
+          throw new Error(
+            `toMatchToolSnapshot: invalid regex pattern "${sanitizer.pattern}" in sanitizer`
+          );
+        }
+      }
       const replacement = sanitizer.replacement ?? "[SANITIZED]";
       result = result.replace(pattern, replacement);
       continue;
@@ -6182,6 +6203,106 @@ var test = test$1.extend({
     await use(api);
   }
 });
+
+// src/fixtures/mcpAuth.ts
+init_oauthClientProvider();
+var StaticTokenAuthProvider = class {
+  accessToken;
+  constructor(accessToken) {
+    this.accessToken = accessToken;
+  }
+  get redirectUrl() {
+    throw new Error("StaticTokenAuthProvider does not support OAuth redirects");
+  }
+  get clientMetadata() {
+    return {
+      redirect_uris: [],
+      token_endpoint_auth_method: "none",
+      grant_types: [],
+      response_types: [],
+      client_name: "@gleanwork/mcp-server-tester"
+    };
+  }
+  async clientInformation() {
+    return void 0;
+  }
+  async tokens() {
+    return {
+      access_token: this.accessToken,
+      token_type: "Bearer"
+    };
+  }
+  async saveTokens() {
+  }
+  async redirectToAuthorization() {
+    throw new Error("StaticTokenAuthProvider does not support OAuth redirects");
+  }
+  async saveCodeVerifier() {
+    throw new Error("StaticTokenAuthProvider does not support PKCE");
+  }
+  async codeVerifier() {
+    throw new Error("StaticTokenAuthProvider does not support PKCE");
+  }
+};
+var test2 = test$1.extend({
+  /**
+   * Create auth provider based on environment configuration
+   */
+  // eslint-disable-next-line no-empty-pattern
+  mcpAuthProvider: async ({}, use) => {
+    const authConfig = getAuthConfigFromEnv();
+    if (!authConfig) {
+      await use(void 0);
+      return;
+    }
+    if (authConfig.accessToken) {
+      const provider = new StaticTokenAuthProvider(authConfig.accessToken);
+      await use(provider);
+      return;
+    }
+    if (authConfig.oauth) {
+      const provider = createOAuthProvider(authConfig.oauth);
+      await use(provider);
+      return;
+    }
+    await use(void 0);
+  }
+});
+function createOAuthProvider(oauthConfig) {
+  if (!oauthConfig.authStatePath) {
+    throw new Error(
+      "OAuth configuration requires authStatePath. Use performOAuthSetup() in globalSetup to create auth state first."
+    );
+  }
+  const providerConfig = {
+    storagePath: oauthConfig.authStatePath,
+    redirectUri: oauthConfig.redirectUri ?? "http://localhost:3000/oauth/callback",
+    clientId: oauthConfig.clientId,
+    clientSecret: oauthConfig.clientSecret
+  };
+  return new PlaywrightOAuthClientProvider(providerConfig);
+}
+function getAuthConfigFromEnv() {
+  const accessToken = process.env.MCP_ACCESS_TOKEN;
+  if (accessToken) {
+    return { accessToken };
+  }
+  const oauthServerUrl = process.env.MCP_OAUTH_SERVER_URL;
+  const authStatePath = process.env.MCP_AUTH_STATE_PATH;
+  if (oauthServerUrl || authStatePath) {
+    return {
+      oauth: {
+        serverUrl: oauthServerUrl ?? "",
+        authStatePath,
+        clientId: process.env.MCP_OAUTH_CLIENT_ID,
+        clientSecret: process.env.MCP_OAUTH_CLIENT_SECRET,
+        scopes: process.env.MCP_OAUTH_SCOPES?.split(","),
+        resource: process.env.MCP_OAUTH_RESOURCE
+      }
+    };
+  }
+  return void 0;
+}
 var LLMHostConfigSchema = z.object({
   provider: z.enum([
     "openai",
@@ -6189,7 +6310,6 @@ var LLMHostConfigSchema = z.object({
     "azure",
     "google",
     "mistral",
-    "ollama",
     "deepseek",
     "openrouter",
     "xai",
@@ -6236,7 +6356,7 @@ var EvalExpectBlockSchema = z.object({
     reference: z.unknown().optional(),
     threshold: z.number().min(0).max(1).optional(),
     reps: z.number().int().min(1).optional(),
-    provider: z.enum(["claude", "anthropic", "openai", "google"]).optional(),
+    provider: z.enum(["anthropic", "openai", "google"]).optional(),
     model: z.string().optional(),
     apiKeyEnvVar: z.string().optional(),
     maxTokens: z.number().int().positive().optional(),
@@ -6378,10 +6498,6 @@ async function loadModel(provider, model) {
       const { azure } = await import('@ai-sdk/azure');
       return azure(model);
     }
-    case "ollama": {
-      const { ollama } = await import('@ai-sdk/ollama');
-      return ollama(model);
-    }
     case "deepseek": {
       const { deepseek } = await import('@ai-sdk/deepseek');
       return deepseek(model);
@@ -6488,7 +6604,6 @@ var allProviders = [
   "azure",
   "google",
   "mistral",
-  "ollama",
   "deepseek",
   "openrouter",
   "xai",
@@ -6516,7 +6631,6 @@ function getMissingDependencyMessage(provider) {
     google: "npm install ai @ai-sdk/google",
     azure: "npm install ai @ai-sdk/azure",
     mistral: "npm install ai @ai-sdk/mistral",
-    ollama: "npm install ai @ai-sdk/ollama",
     deepseek: "npm install ai @ai-sdk/deepseek",
     openrouter: "npm install ai @openrouter/ai-sdk-provider",
     xai: "npm install ai @ai-sdk/xai",
@@ -6763,15 +6877,17 @@ async function runSingleIteration(evalCase, context, options) {
 function isInfrastructureError(err) {
   let name15;
   let msg;
+  let code = "";
   if (err instanceof Error) {
     name15 = err.name;
     msg = err.message.toLowerCase();
+    code = (err.code ?? "").toLowerCase();
   } else if (typeof err === "string") {
     msg = err.toLowerCase();
   } else {
     return false;
   }
-  return name15 === "AbortError" || msg.includes("econnreset") || msg.includes("etimedout") || msg.includes("econnrefused") || msg.includes("rate limit") || msg.includes("429") || msg.includes("503") || msg.includes("network");
+  return name15 === "AbortError" || msg.includes("econnreset") || msg.includes("etimedout") || msg.includes("econnrefused") || msg.includes("rate limit") || msg.includes("429") || msg.includes("503") || msg.includes("network") || code.includes("econnreset") || code.includes("etimedout") || code.includes("econnrefused");
 }
 async function runEvalCase(evalCase, context, options = {}) {
   const iterations = evalCase.iterations ?? 1;
@@ -6886,9 +7002,9 @@ async function runEvalDataset(options, context) {
     const withIterations = evalCase.mode === "llm_host" && evalCase.iterations === void 0 && defaultLlmIterations !== void 0 ? { ...evalCase, iterations: defaultLlmIterations } : evalCase;
     if (evalCase.mode === "llm_host") {
       const effectiveIterations = withIterations.iterations ?? 1;
-      if (effectiveIterations < 10) {
+      if (effectiveIterations > 1 && effectiveIterations < 10) {
         console.warn(
-          `[mcp-server-tester] Eval case "${evalCase.id}" uses llm_host mode with only ${effectiveIterations} iteration(s). The evals guide recommends >= 10 iterations. See docs/evals-guide.md for guidance on statistical reliability.`
+          `[mcp-server-tester] Eval case "${evalCase.id}": running ${effectiveIterations} iterations in llm_host mode may not be statistically reliable. Consider using 10+ iterations for accuracy measurements you can trust.`
         );
       }
     }
@@ -6936,6 +7052,16 @@ async function runEvalDataset(options, context) {
       const baseline = await loadBaseline(baselineResultsFrom);
       const baselinePassRate = baseline.total > 0 ? baseline.passed / baseline.total : 0;
       const baselineMap = buildBaselinePassMap(baseline);
+      const currentCaseIds = result.caseResults.map((cr) => cr.id);
+      const unmatchedCount = currentCaseIds.filter(
+        (id) => !baselineMap.has(id)
+      ).length;
+      const unmatchedRatio = currentCaseIds.length > 0 ? unmatchedCount / currentCaseIds.length : 0;
+      if (unmatchedRatio > 0.2) {
+        console.warn(
+          `[mcp-server-tester] Baseline comparison: ${unmatchedCount} of ${currentCaseIds.length} cases (${Math.round(unmatchedRatio * 100)}%) have no baseline entry. This may indicate the dataset structure has changed. Results for unmatched cases cannot be compared.`
+        );
+      }
       for (const cr of result.caseResults) {
         const baselinePass = baselineMap.get(cr.id);
         if (baselinePass !== void 0) {
@@ -7212,6 +7338,6 @@ function formatCapabilities(capabilities) {
   return parts.length > 0 ? parts.join(", ") : "none declared";
 }
 
-export { BUILT_IN_RUBRICS, CLIOAuthClient, DiscoveryError, ENV_VAR_NAMES, EvalCaseSchema, EvalDatasetSchema, MCPConfigSchema, MCP_PROTOCOL_VERSION, PlaywrightOAuthClientProvider, SnapshotSanitizers, closeMCPClient, createJudge, createMCPClientForConfig, createMCPFixture, createTokenAuthHeaders, discoverAuthorizationServer, discoverProtectedResource, expect, extractText, getMissingDependencyMessage, getResponseSizeBytes, hasValidTokens, injectTokens, isBuiltInRubric, isHttpConfig, isProviderAvailable, isStdioConfig, isTokenExpired, isTokenExpiringSoon, loadBaseline, loadEvalDataset, loadEvalDatasetFromObject, loadTokens, loadTokensFromEnv, normalizeToolResponse, normalizeWhitespace, performClientCredentialsFlow, performOAuthSetup, performOAuthSetupIfNeeded, resolveRubric, runConformanceChecks, runEvalCase, runEvalDataset, runServerComparison, saveBaseline, simulateLLMHost, test, validateAccessToken, validateError, validateEvalCase, validateEvalDataset, validateJudge, validateMCPConfig, validatePattern, validateResponse, validateSchema, validateSize, validateText, validateToolCallCount, validateToolCalls };
+export { BUILT_IN_RUBRICS, CLIOAuthClient, DiscoveryError, ENV_VAR_NAMES, EvalCaseSchema, EvalDatasetSchema, MCPConfigSchema, MCP_PROTOCOL_VERSION, PlaywrightOAuthClientProvider, SnapshotSanitizers, closeMCPClient, createJudge, createMCPClientForConfig, createMCPFixture, createTokenAuthHeaders, discoverAuthorizationServer, discoverProtectedResource, expect, extractText, getMissingDependencyMessage, getResponseSizeBytes, hasValidTokens, injectTokens, isBuiltInRubric, isHttpConfig, isProviderAvailable, isStdioConfig, isTokenExpired, isTokenExpiringSoon, loadBaseline, loadEvalDataset, loadEvalDatasetFromObject, loadTokens, loadTokensFromEnv, test2 as mcpAuthTest, normalizeToolResponse, normalizeWhitespace, performClientCredentialsFlow, performOAuthSetup, performOAuthSetupIfNeeded, resolveRubric, runConformanceChecks, runEvalCase, runEvalDataset, runServerComparison, saveBaseline, simulateLLMHost, test, validateAccessToken, validateError, validateEvalCase, validateEvalDataset, validateJudge, validateMCPConfig, validatePattern, validateResponse, validateSchema, validateSize, validateText, validateToolCallCount, validateToolCalls };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map