@yawlabs/mcp-compliance 0.13.2 → 0.13.4

package/dist/{chunk-X5CVUDPW.js → chunk-6PF56RRO.js}

@@ -1447,6 +1447,7 @@ async function runComplianceSuite(target, options = {}) {
     }
     const nextId = createIdCounter(1e3);
     const timeout = options.timeout || 15e3;
+    const startupTimeout = options.startupTimeout ?? Math.max(timeout, 6e4);
     const retries = options.retries || 0;
     let sessionId = null;
     let negotiatedProtocolVersion = null;
@@ -1702,16 +1703,24 @@ async function runComplianceSuite(target, options = {}) {
       }
     );
     let initRes = null;
+    const initStart = Date.now();
     try {
-      initRes = await rpc("initialize", {
-        protocolVersion: SPEC_VERSION,
-        capabilities: {
-          sampling: {},
-          roots: { listChanged: true },
-          elicitation: {}
+      initRes = await mcpRequest(
+        backendUrl,
+        "initialize",
+        {
+          protocolVersion: SPEC_VERSION,
+          capabilities: {
+            sampling: {},
+            roots: { listChanged: true },
+            elicitation: {}
+          },
+          clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
         },
-        clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
-      });
+        nextId,
+        buildHeaders2(),
+        startupTimeout
+      );
       const result = initRes?.body?.result;
       if (result) {
         serverInfo.protocolVersion = result.protocolVersion || null;
@@ -1730,8 +1739,14 @@ async function runComplianceSuite(target, options = {}) {
       }
     } catch {
     }
+    const initElapsed = Date.now() - initStart;
+    if (initRes && initElapsed > timeout) {
+      warnings.push(
+        `Initialize handshake took ${initElapsed}ms \u2014 longer than --timeout ${timeout}ms. Per-test requests use --timeout, so slow servers may flake. Consider raising --timeout.`
+      );
+    }
     try {
-      await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders2(), timeout);
+      await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders2(), startupTimeout);
     } catch {
     }
     const hasTools = !!serverInfo.capabilities.tools;

package/dist/index.js

@@ -559,6 +559,7 @@ function validate(raw, source) {
   const allowed = /* @__PURE__ */ new Set([
     "target",
     "timeout",
+    "startupTimeout",
     "preflightTimeout",
     "retries",
     "only",
@@ -1803,6 +1804,7 @@ async function runComplianceSuite(target, options = {}) {
     }
     const nextId = createIdCounter(1e3);
     const timeout = options.timeout || 15e3;
+    const startupTimeout = options.startupTimeout ?? Math.max(timeout, 6e4);
     const retries = options.retries || 0;
     let sessionId = null;
     let negotiatedProtocolVersion = null;
@@ -2058,16 +2060,24 @@ async function runComplianceSuite(target, options = {}) {
       }
     );
     let initRes = null;
+    const initStart = Date.now();
     try {
-      initRes = await rpc("initialize", {
-        protocolVersion: SPEC_VERSION,
-        capabilities: {
-          sampling: {},
-          roots: { listChanged: true },
-          elicitation: {}
+      initRes = await mcpRequest(
+        backendUrl,
+        "initialize",
+        {
+          protocolVersion: SPEC_VERSION,
+          capabilities: {
+            sampling: {},
+            roots: { listChanged: true },
+            elicitation: {}
+          },
+          clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
         },
-        clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
-      });
+        nextId,
+        buildHeaders2(),
+        startupTimeout
+      );
       const result = initRes?.body?.result;
       if (result) {
         serverInfo.protocolVersion = result.protocolVersion || null;
@@ -2086,8 +2096,14 @@ async function runComplianceSuite(target, options = {}) {
       }
     } catch {
     }
+    const initElapsed = Date.now() - initStart;
+    if (initRes && initElapsed > timeout) {
+      warnings.push(
+        `Initialize handshake took ${initElapsed}ms \u2014 longer than --timeout ${timeout}ms. Per-test requests use --timeout, so slow servers may flake. Consider raising --timeout.`
+      );
+    }
     try {
-      await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders2(), timeout);
+      await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders2(), startupTimeout);
     } catch {
     }
     const hasTools = !!serverInfo.capabilities.tools;
@@ -5161,6 +5177,20 @@ function formatHtml(report) {
 </html>`;
 }
 
+// src/stdio-split.ts
+function splitStdioTarget(s) {
+  if (/["'`]/.test(s)) {
+    throw new Error(
+      "stdio command contains quote characters \u2014 pass the command and its args as separate tokens (e.g. `mcp-compliance test node dist/index.js serve`) instead of wrapping them in one quoted string."
+    );
+  }
+  const tokens = s.trim().split(/\s+/).filter(Boolean);
+  if (tokens.length === 0) {
+    throw new Error("stdio command is empty");
+  }
+  return { command: tokens[0], args: tokens.slice(1) };
+}
+
 // src/token-store.ts
 import { createHash as createHash2 } from "crypto";
 import { existsSync as existsSync3, mkdirSync, readFileSync as readFileSync3, writeFileSync } from "fs";
@@ -5290,10 +5320,17 @@ function buildTarget(positional, extraArgs, opts) {
     ...opts.envFile ? readEnvFile(opts.envFile) : {},
     ...opts.env ?? {}
   };
+  let command = positional;
+  let args = extraArgs;
+  if (extraArgs.length === 0 && /\s/.test(positional)) {
+    const split = splitStdioTarget(positional);
+    command = split.command;
+    args = split.args;
+  }
   return {
     type: "stdio",
-    command: positional,
-    args: extraArgs,
+    command,
+    args,
     env: Object.keys(env).length ? env : void 0,
     cwd: opts.cwd,
     verbose: opts.verbose
@@ -5383,8 +5420,11 @@ program.command("test").description("Run the full compliance test suite against
   {}
 ).option("--auth <token>", 'Shorthand for -H "Authorization: <token>" (HTTP only)').option("-E, --env <var>", 'Set env var for stdio command ("KEY=VALUE", repeatable)', parseEnvVar, {}).option("--env-file <path>", "Load env vars from file (KEY=VALUE per line, stdio only)").option("--cwd <dir>", "Working directory for stdio command").option(
   "--timeout <ms>",
-  "Request timeout in milliseconds (bump to 30000+ for stdio servers with slow startup)",
+  "Per-request timeout in milliseconds (applies to every test request after the initial handshake)",
   "15000"
+).option(
+  "--startup-timeout <ms>",
+  "Deadline for the initial initialize handshake (default: max(--timeout, 60000); covers cold `npx` cache fetches before a stdio server starts)"
 ).option("--no-color", "Disable colored output (also honors NO_COLOR env var)").option("--watch", "Re-run tests when files in the cwd change (stdio targets only)").option(
   "--concurrency <n>",
   "Max parallel-safe tests in flight (default 1; see docs/PERFORMANCE.md before raising)",
@@ -5446,6 +5486,7 @@ Testing ${describeTarget(transportTarget)}...
         }
         const report2 = await runComplianceSuite(transportTarget, {
           timeout: parsePositiveInt(opts.timeout, "--timeout", 1),
+          startupTimeout: opts.startupTimeout ? parsePositiveInt(opts.startupTimeout, "--startup-timeout", 1) : config?.startupTimeout,
           preflightTimeout: opts.preflightTimeout ? parsePositiveInt(opts.preflightTimeout, "--preflight-timeout", 1) : config?.preflightTimeout,
           retries: parsePositiveInt(opts.retries, "--retries"),
           concurrency: parsePositiveInt(opts.concurrency, "--concurrency", 1),

package/dist/mcp/server.js

@@ -2,7 +2,7 @@ import {
   SPEC_BASE,
   TEST_DEFINITIONS,
   runComplianceSuite
-} from "../chunk-X5CVUDPW.js";
+} from "../chunk-6PF56RRO.js";
 
 // src/mcp/server.ts
 import { existsSync, readFileSync, realpathSync } from "fs";

package/dist/runner.d.ts

@@ -180,6 +180,18 @@ interface RunOptions {
     headers?: Record<string, string>;
     /** Request timeout in milliseconds (default: 15000) */
     timeout?: number;
+    /**
+     * Deadline for the initial `initialize` handshake + `initialized`
+     * notification, in milliseconds. Kept separate from `timeout` because
+     * cold-started stdio servers — especially `npx @pkg ...` targets where
+     * npm has to resolve and fetch the package before the MCP server
+     * starts — can take tens of seconds to produce their first response,
+     * while steady-state requests complete in milliseconds. Default is
+     * `max(timeout, 60000)` so users who bump `--timeout` keep that value
+     * and users on defaults get 60s for startup. Does not apply to any
+     * per-test requests past the handshake.
+     */
+    startupTimeout?: number;
     /** Number of retries for failed tests (default: 0) */
     retries?: number;
     /** Only run tests matching these category names or test IDs */

package/dist/runner.js

@@ -10,7 +10,7 @@ import {
   previewTests,
   runComplianceSuite,
   urlHash
-} from "./chunk-X5CVUDPW.js";
+} from "./chunk-6PF56RRO.js";
 export {
   SPEC_BASE,
   SPEC_VERSION,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yawlabs/mcp-compliance",
-  "version": "0.13.2",
+  "version": "0.13.4",
   "description": "CLI tool and MCP server that tests MCP servers for spec compliance",
   "license": "MIT",
   "author": "Yaw Labs <contact@yaw.sh> (https://yaw.sh)",