@gleanwork/mcp-server-tester 2.0.0-beta.3 → 2.0.0-beta.4

package/dist/cli/index.js

@@ -3608,22 +3608,22 @@ async function runAnthropicComputerUseHitl(options) {
     "HITL check"
   );
 }
-async function runComputerUseDriver(query2, options, mode, label) {
+async function runComputerUseDriver(query2, options, mode, label2) {
   const env = {
     ...process.env,
     ...options.env,
     ...options.model ? { MST_COWORK_CUA_MODEL: options.model } : {}
   };
   if (options.deadlineAt <= Date.now())
-    throw new Error(`Computer Use ${label} deadline exceeded; not retrying.`);
+    throw new Error(`Computer Use ${label2} deadline exceeded; not retrying.`);
   const python = await ensureCoworkPython(env);
   const DRIVER_PATH = resolveDriverPath(env);
   const timeoutMs = options.deadlineAt - Date.now();
   if (timeoutMs <= 0)
-    throw new Error(`Computer Use ${label} deadline exceeded; not retrying.`);
+    throw new Error(`Computer Use ${label2} deadline exceeded; not retrying.`);
   const maxActions = options.maxActions ?? (mode === "hitl" ? 12 : 24);
   diagnostic(
-    `starting Computer Use ${label} (script=${DRIVER_PATH}, maxActions=${maxActions}, timeoutMs=${timeoutMs})`
+    `starting Computer Use ${label2} (script=${DRIVER_PATH}, maxActions=${maxActions}, timeoutMs=${timeoutMs})`
   );
   let stdout = "";
   try {
@@ -3654,26 +3654,26 @@ async function runComputerUseDriver(query2, options, mode, label) {
         telemetry3
       );
     }
-    diagnostic(`Computer Use ${label} failed: ${details}`);
+    diagnostic(`Computer Use ${label2} failed: ${details}`);
     throw new ComputerUseDriverError(
-      `Computer Use ${label} failed: ${details}`,
+      `Computer Use ${label2} failed: ${details}`,
       telemetry3
     );
   }
   diagnostic(
-    `Computer Use ${label} exited successfully (stdoutBytes=${stdout.length})`
+    `Computer Use ${label2} exited successfully (stdoutBytes=${stdout.length})`
   );
   const record = parseLastJsonLine(stdout);
   const expectedStatus = mode === "submit" ? "submitted" : "hitl_checked";
   if (record?.status !== expectedStatus) {
     throw new ComputerUseDriverError(
-      `Computer Use ${label} did not reach ${expectedStatus}.`,
+      `Computer Use ${label2} did not reach ${expectedStatus}.`,
       parseTelemetry(record?.telemetry, env, "partial")
     );
   }
   if (!isCount(record.action_count) || !isSafeModel(record.model, env)) {
     throw new ComputerUseDriverError(
-      `Computer Use ${label} returned invalid result fields.`,
+      `Computer Use ${label2} returned invalid result fields.`,
       parseTelemetry(record.telemetry, env, "partial")
     );
   }
@@ -3684,7 +3684,7 @@ async function runComputerUseDriver(query2, options, mode, label) {
     ...telemetry2 ? { telemetry: telemetry2 } : {}
   };
   diagnostic(
-    `Computer Use ${label} completed (actions=${common.action_count})`
+    `Computer Use ${label2} completed (actions=${common.action_count})`
   );
   if (mode === "hitl") return { status: "hitl_checked", ...common };
   const submission = asRecord(record.submission_action);
@@ -5563,12 +5563,12 @@ init_esm_shims();
 
 // src/cli/components/Spinner.tsx
 init_esm_shims();
-function Spinner({ label }) {
+function Spinner({ label: label2 }) {
   return /* @__PURE__ */ jsxs(Box, { children: [
     /* @__PURE__ */ jsx(Text, { color: "cyan", children: /* @__PURE__ */ jsx(InkSpinner, { type: "dots" }) }),
     /* @__PURE__ */ jsxs(Text, { children: [
       " ",
-      label
+      label2
     ] })
   ] });
 }
@@ -5630,7 +5630,7 @@ init_esm_shims();
 
 // package.json
 var package_default = {
-  version: "2.0.0-beta.3"};
+  version: "2.0.0-beta.4"};
 
 // src/cli/templates/index.ts
 function getPlaywrightConfigTemplate(answers) {
@@ -6362,8 +6362,56 @@ async function performClientCredentialsFlow(config) {
   };
 }
 
+// src/mcp/connectionDiagnostics.ts
+init_esm_shims();
+function classifyMCPConnectionFailure(error) {
+  if (typeof error !== "object" || error === null) return "connection_failed";
+  const details = error;
+  const status = details.response?.status ?? details.status ?? details.code;
+  if (typeof status === "number" && Number.isInteger(status) && status >= 400 && status <= 599) {
+    return `http_${status}`;
+  }
+  const message = error instanceof Error ? error.message : "";
+  const httpStatus = /\b(?:HTTP|status(?: code)?|code:)\s*[:=(]?\s*([45]\d{2})\b/i.exec(
+    message
+  )?.[1];
+  if (httpStatus) return `http_${httpStatus}`;
+  for (const code of ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND"]) {
+    if (details.code === code || message.toUpperCase().includes(code)) {
+      return code.toLowerCase();
+    }
+  }
+  if (/timed out|timeout/i.test(message)) return "timeout";
+  if (/network|socket hang up|fetch failed/i.test(message))
+    return "network_error";
+  return "connection_failed";
+}
+var MCPHttpConnectionError = class extends Error {
+  constructor(streamableError, sseError, retryable, retryAfterMs) {
+    const streamableHttpFailure = classifyMCPConnectionFailure(streamableError);
+    const sseFailure = classifyMCPConnectionFailure(sseError);
+    super(
+      `MCP connection failed: streamableHttp=${streamableHttpFailure}; sse=${sseFailure}`
+    );
+    this.retryable = retryable;
+    this.retryAfterMs = retryAfterMs;
+    this.name = "MCPHttpConnectionError";
+    this.streamableHttpFailure = streamableHttpFailure;
+    this.sseFailure = sseFailure;
+  }
+  streamableHttpFailure;
+  sseFailure;
+};
+function formatMCPConnectionFailure(error) {
+  if (error instanceof MCPHttpConnectionError) {
+    return `MCP connection failed: streamableHttp=${error.streamableHttpFailure}; sse=${error.sseFailure}`;
+  }
+  return classifyMCPConnectionFailure(error);
+}
+
 // src/mcp/clientFactory.ts
 function getRetryAfterDelayMs(err) {
+  if (err instanceof MCPHttpConnectionError) return err.retryAfterMs;
   const response = err?.response;
   const retryAfter = response?.headers?.get?.("Retry-After");
   if (retryAfter) {
@@ -6382,6 +6430,7 @@ function isTransientNetworkError(err) {
   return msg.includes("econnreset") || msg.includes("econnrefused") || msg.includes("etimedout") || msg.includes("enotfound") || msg.includes("network") || msg.includes("socket hang up") || msg.includes("fetch failed");
 }
 function isRetryableError(err) {
+  if (err instanceof MCPHttpConnectionError) return err.retryable;
   return isTransientNetworkError(err) || isRateLimitError(err);
 }
 async function retryWithBackoff(fn, maxAttempts) {
@@ -6399,7 +6448,7 @@ async function retryWithBackoff(fn, maxAttempts) {
           attempt + 1,
           maxAttempts + 1,
           delayMs,
-          err.message
+          formatMCPConnectionFailure(err)
         );
         await new Promise((resolve10) => setTimeout(resolve10, delayMs));
       } else {
@@ -6556,15 +6605,24 @@ async function createMCPClientForConfig(config, options) {
       } catch (err) {
         debugHttp(
           "streamableHttp failed (%s), falling back to SSE",
-          err.message
+          formatMCPConnectionFailure(err)
         );
         debugClient("Streamable HTTP failed, falling back to SSE transport");
         debugHttp("Attempting transport: sse");
-        const sseTransport = new SSEClientTransport(url, {
-          requestInit,
-          ...options?.authProvider ? { authProvider: options.authProvider } : {}
-        });
-        await client.connect(sseTransport, connectOptions);
+        try {
+          const sseTransport = new SSEClientTransport(url, {
+            requestInit,
+            ...options?.authProvider ? { authProvider: options.authProvider } : {}
+          });
+          await client.connect(sseTransport, connectOptions);
+        } catch (sseError) {
+          throw new MCPHttpConnectionError(
+            err,
+            sseError,
+            isRetryableError(err) || isRetryableError(sseError),
+            getRetryAfterDelayMs(err) ?? getRetryAfterDelayMs(sseError)
+          );
+        }
         debugClient("Connected via SSE");
         debugHttp("Connection established via sse");
       }
@@ -10830,6 +10888,87 @@ async function getCoworkPlatform(provider) {
   );
 }
 
+// src/evals/cowork/mcpReadiness.ts
+init_esm_shims();
+init_config();
+var PREFLIGHT_TIMEOUT_MS = 3e4;
+var CoworkMcpReadinessError = class extends Error {
+  servers;
+  constructor(servers) {
+    super(
+      `Cowork MCP preflight failed; no task was submitted. ${servers.map(
+        (server) => `${server.label}=${server.status}${server.error ? `(${server.error})` : ""}(${server.elapsedMs}ms)`
+      ).join(", ")}`
+    );
+    this.name = "CoworkMcpReadinessError";
+    this.servers = servers;
+  }
+};
+function label(server, index) {
+  return server.label ?? `server-${index + 1}`;
+}
+async function withTimeout(promise, timeoutMs) {
+  let timer;
+  try {
+    return await Promise.race([
+      promise,
+      new Promise((_, reject) => {
+        timer = setTimeout(
+          () => reject(new Error("MCP preflight timed out")),
+          timeoutMs
+        );
+      })
+    ]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+function resolveServer(server, env) {
+  if (!isHttpConfig(server)) return server;
+  const [coworkServer] = toCoworkServers([server]);
+  if (!coworkServer) throw new Error("Invalid Cowork MCP configuration.");
+  const headers = resolveCoworkMcpHeaders([coworkServer], env);
+  return {
+    ...server,
+    // Use the same validated runtime headers as Cowork setup.
+    auth: void 0,
+    headers: headers[coworkServer.label]
+  };
+}
+async function verifyCoworkMcpServers(servers, env) {
+  const results = await Promise.all(
+    servers.map(async (server, index) => {
+      const started = Date.now();
+      let client;
+      try {
+        client = await createMCPClientForConfig(resolveServer(server, env));
+        const tools = await withTimeout(
+          client.listTools(),
+          PREFLIGHT_TIMEOUT_MS
+        );
+        return {
+          label: label(server, index),
+          status: "connected",
+          toolCount: tools.tools.length,
+          elapsedMs: Date.now() - started
+        };
+      } catch (error) {
+        return {
+          label: label(server, index),
+          status: "failed",
+          elapsedMs: Date.now() - started,
+          error: formatMCPConnectionFailure(error)
+        };
+      } finally {
+        if (client) await closeMCPClient(client).catch(() => void 0);
+      }
+    })
+  );
+  if (results.some((result) => result.status !== "connected"))
+    throw new CoworkMcpReadinessError(results);
+  return results;
+}
+
 // src/evals/externalHost/builtins/anthropicClaude.ts
 init_esm_shims();
 
@@ -12387,6 +12526,14 @@ async function runBatch(requests, context, selectedPlatform) {
         env,
         model: config.model
       });
+    if (servers.length) {
+      const readiness = await verifyCoworkMcpServers(servers, env);
+      process.stderr.write(
+        `[mst:cowork] MCP preflight ready: ${readiness.map(
+          (server) => `${server.label}(${server.toolCount ?? 0} tools, ${server.elapsedMs}ms)`
+        ).join(", ")}\\n`
+      );
+    }
     for (const [index, request] of requests.entries()) {
       const caseStartedAt = Date.now();
       const computerUse = {
@@ -12888,8 +13035,8 @@ function claudeCliHost(options) {
   if (options.servers) {
     for (const key of Object.keys(mcpServers)) delete mcpServers[key];
     for (const entry of options.servers) {
-      const label = entry.label ?? "mcp-server";
-      mcpServers[label] = entry.transport === "http" ? {
+      const label2 = entry.label ?? "mcp-server";
+      mcpServers[label2] = entry.transport === "http" ? {
         type: "http",
         url: entry.serverUrl,
         headers: {
@@ -16320,9 +16467,9 @@ function buildArmDeltas(arms) {
   );
 }
 function redactServerForReport(server) {
-  const label = server.label ? { label: server.label } : {};
+  const label2 = server.label ? { label: server.label } : {};
   if (server.transport === "stdio") {
-    return { transport: "stdio", command: server.command, ...label };
+    return { transport: "stdio", command: server.command, ...label2 };
   }
   const url = new URL(server.serverUrl);
   url.username = "";
@@ -16332,7 +16479,7 @@ function redactServerForReport(server) {
   return {
     transport: "http",
     serverUrl: url.toString(),
-    ...label,
+    ...label2,
     ...server.auth?.accessTokenEnv ? { auth: { accessTokenEnv: server.auth.accessTokenEnv } } : {}
   };
 }

package/dist/fixtures/mcp.js

@@ -1874,7 +1874,7 @@ var debugHttp = createDebug(`${NAMESPACE}:http`);
 
 // package.json
 var package_default = {
-  version: "2.0.0-beta.3"};
+  version: "2.0.0-beta.4"};
 var debug = createDebug("mcp-server-tester:oauth-flow");
 async function generatePKCE() {
   const codeVerifier = oauth.generateRandomCodeVerifier();
@@ -2038,8 +2038,55 @@ async function performClientCredentialsFlow(config) {
   };
 }
 
+// src/mcp/connectionDiagnostics.ts
+function classifyMCPConnectionFailure(error) {
+  if (typeof error !== "object" || error === null) return "connection_failed";
+  const details = error;
+  const status = details.response?.status ?? details.status ?? details.code;
+  if (typeof status === "number" && Number.isInteger(status) && status >= 400 && status <= 599) {
+    return `http_${status}`;
+  }
+  const message = error instanceof Error ? error.message : "";
+  const httpStatus = /\b(?:HTTP|status(?: code)?|code:)\s*[:=(]?\s*([45]\d{2})\b/i.exec(
+    message
+  )?.[1];
+  if (httpStatus) return `http_${httpStatus}`;
+  for (const code of ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND"]) {
+    if (details.code === code || message.toUpperCase().includes(code)) {
+      return code.toLowerCase();
+    }
+  }
+  if (/timed out|timeout/i.test(message)) return "timeout";
+  if (/network|socket hang up|fetch failed/i.test(message))
+    return "network_error";
+  return "connection_failed";
+}
+var MCPHttpConnectionError = class extends Error {
+  constructor(streamableError, sseError, retryable, retryAfterMs) {
+    const streamableHttpFailure = classifyMCPConnectionFailure(streamableError);
+    const sseFailure = classifyMCPConnectionFailure(sseError);
+    super(
+      `MCP connection failed: streamableHttp=${streamableHttpFailure}; sse=${sseFailure}`
+    );
+    this.retryable = retryable;
+    this.retryAfterMs = retryAfterMs;
+    this.name = "MCPHttpConnectionError";
+    this.streamableHttpFailure = streamableHttpFailure;
+    this.sseFailure = sseFailure;
+  }
+  streamableHttpFailure;
+  sseFailure;
+};
+function formatMCPConnectionFailure(error) {
+  if (error instanceof MCPHttpConnectionError) {
+    return `MCP connection failed: streamableHttp=${error.streamableHttpFailure}; sse=${error.sseFailure}`;
+  }
+  return classifyMCPConnectionFailure(error);
+}
+
 // src/mcp/clientFactory.ts
 function getRetryAfterDelayMs(err) {
+  if (err instanceof MCPHttpConnectionError) return err.retryAfterMs;
   const response = err?.response;
   const retryAfter = response?.headers?.get?.("Retry-After");
   if (retryAfter) {
@@ -2058,6 +2105,7 @@ function isTransientNetworkError(err) {
   return msg.includes("econnreset") || msg.includes("econnrefused") || msg.includes("etimedout") || msg.includes("enotfound") || msg.includes("network") || msg.includes("socket hang up") || msg.includes("fetch failed");
 }
 function isRetryableError(err) {
+  if (err instanceof MCPHttpConnectionError) return err.retryable;
   return isTransientNetworkError(err) || isRateLimitError(err);
 }
 async function retryWithBackoff(fn, maxAttempts) {
@@ -2075,7 +2123,7 @@ async function retryWithBackoff(fn, maxAttempts) {
           attempt + 1,
           maxAttempts + 1,
           delayMs,
-          err.message
+          formatMCPConnectionFailure(err)
         );
         await new Promise((resolve) => setTimeout(resolve, delayMs));
       } else {
@@ -2232,15 +2280,24 @@ async function createMCPClientForConfig(config, options) {
       } catch (err) {
         debugHttp(
           "streamableHttp failed (%s), falling back to SSE",
-          err.message
+          formatMCPConnectionFailure(err)
         );
         debugClient("Streamable HTTP failed, falling back to SSE transport");
         debugHttp("Attempting transport: sse");
-        const sseTransport = new SSEClientTransport(url, {
-          requestInit,
-          ...options?.authProvider ? { authProvider: options.authProvider } : {}
-        });
-        await client.connect(sseTransport, connectOptions);
+        try {
+          const sseTransport = new SSEClientTransport(url, {
+            requestInit,
+            ...options?.authProvider ? { authProvider: options.authProvider } : {}
+          });
+          await client.connect(sseTransport, connectOptions);
+        } catch (sseError) {
+          throw new MCPHttpConnectionError(
+            err,
+            sseError,
+            isRetryableError(err) || isRetryableError(sseError),
+            getRetryAfterDelayMs(err) ?? getRetryAfterDelayMs(sseError)
+          );
+        }
         debugClient("Connected via SSE");
         debugHttp("Connection established via sse");
       }