@yawlabs/mcp-compliance 0.13.3 → 0.13.5

package/dist/{chunk-X5CVUDPW.js → chunk-6PF56RRO.js}

@@ -1447,6 +1447,7 @@ async function runComplianceSuite(target, options = {}) {
     }
     const nextId = createIdCounter(1e3);
     const timeout = options.timeout || 15e3;
+    const startupTimeout = options.startupTimeout ?? Math.max(timeout, 6e4);
     const retries = options.retries || 0;
     let sessionId = null;
     let negotiatedProtocolVersion = null;
@@ -1702,16 +1703,24 @@ async function runComplianceSuite(target, options = {}) {
       }
     );
     let initRes = null;
+    const initStart = Date.now();
     try {
-      initRes = await rpc("initialize", {
-        protocolVersion: SPEC_VERSION,
-        capabilities: {
-          sampling: {},
-          roots: { listChanged: true },
-          elicitation: {}
+      initRes = await mcpRequest(
+        backendUrl,
+        "initialize",
+        {
+          protocolVersion: SPEC_VERSION,
+          capabilities: {
+            sampling: {},
+            roots: { listChanged: true },
+            elicitation: {}
+          },
+          clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
         },
-        clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
-      });
+        nextId,
+        buildHeaders2(),
+        startupTimeout
+      );
       const result = initRes?.body?.result;
       if (result) {
         serverInfo.protocolVersion = result.protocolVersion || null;
@@ -1730,8 +1739,14 @@ async function runComplianceSuite(target, options = {}) {
       }
     } catch {
     }
+    const initElapsed = Date.now() - initStart;
+    if (initRes && initElapsed > timeout) {
+      warnings.push(
+        `Initialize handshake took ${initElapsed}ms \u2014 longer than --timeout ${timeout}ms. Per-test requests use --timeout, so slow servers may flake. Consider raising --timeout.`
+      );
+    }
     try {
-      await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders2(), timeout);
+      await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders2(), startupTimeout);
     } catch {
     }
     const hasTools = !!serverInfo.capabilities.tools;

package/dist/index.js

@@ -559,6 +559,7 @@ function validate(raw, source) {
   const allowed = /* @__PURE__ */ new Set([
     "target",
     "timeout",
+    "startupTimeout",
     "preflightTimeout",
     "retries",
     "only",
@@ -1803,6 +1804,7 @@ async function runComplianceSuite(target, options = {}) {
     }
     const nextId = createIdCounter(1e3);
     const timeout = options.timeout || 15e3;
+    const startupTimeout = options.startupTimeout ?? Math.max(timeout, 6e4);
     const retries = options.retries || 0;
     let sessionId = null;
     let negotiatedProtocolVersion = null;
@@ -2058,16 +2060,24 @@ async function runComplianceSuite(target, options = {}) {
       }
     );
     let initRes = null;
+    const initStart = Date.now();
     try {
-      initRes = await rpc("initialize", {
-        protocolVersion: SPEC_VERSION,
-        capabilities: {
-          sampling: {},
-          roots: { listChanged: true },
-          elicitation: {}
+      initRes = await mcpRequest(
+        backendUrl,
+        "initialize",
+        {
+          protocolVersion: SPEC_VERSION,
+          capabilities: {
+            sampling: {},
+            roots: { listChanged: true },
+            elicitation: {}
+          },
+          clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
         },
-        clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
-      });
+        nextId,
+        buildHeaders2(),
+        startupTimeout
+      );
       const result = initRes?.body?.result;
       if (result) {
         serverInfo.protocolVersion = result.protocolVersion || null;
@@ -2086,8 +2096,14 @@ async function runComplianceSuite(target, options = {}) {
       }
     } catch {
     }
+    const initElapsed = Date.now() - initStart;
+    if (initRes && initElapsed > timeout) {
+      warnings.push(
+        `Initialize handshake took ${initElapsed}ms \u2014 longer than --timeout ${timeout}ms. Per-test requests use --timeout, so slow servers may flake. Consider raising --timeout.`
+      );
+    }
     try {
-      await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders2(), timeout);
+      await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders2(), startupTimeout);
     } catch {
     }
     const hasTools = !!serverInfo.capabilities.tools;
@@ -4756,18 +4772,25 @@ var CATEGORY_LABELS = {
   security: "Security"
 };
 var CATEGORY_ORDER = ["transport", "lifecycle", "tools", "resources", "prompts", "errors", "schema", "security"];
+var GRADE_ART = {
+  A: [" \u2588\u2588\u2588\u2588\u2588\u2557 ", "\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557", "\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551", "\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551", "\u2588\u2588\u2551  \u2588\u2588\u2551", "\u255A\u2550\u255D  \u255A\u2550\u255D"],
+  B: ["\u2588\u2588\u2588\u2588\u2588\u2588\u2557 ", "\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557", "\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D", "\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557", "\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D", "\u255A\u2550\u2550\u2550\u2550\u2550\u255D "],
+  C: [" \u2588\u2588\u2588\u2588\u2588\u2588\u2557", "\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D", "\u2588\u2588\u2551     ", "\u2588\u2588\u2551     ", "\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2557", " \u255A\u2550\u2550\u2550\u2550\u2550\u255D"],
+  D: ["\u2588\u2588\u2588\u2588\u2588\u2588\u2557 ", "\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557", "\u2588\u2588\u2551  \u2588\u2588\u2551", "\u2588\u2588\u2551  \u2588\u2588\u2551", "\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D", "\u255A\u2550\u2550\u2550\u2550\u2550\u255D "],
+  F: ["\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557", "\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D", "\u2588\u2588\u2588\u2588\u2588\u2557  ", "\u2588\u2588\u2554\u2550\u2550\u255D  ", "\u2588\u2588\u2551     ", "\u255A\u2550\u255D     "]
+};
 function gradeColor(grade) {
   switch (grade) {
     case "A":
-      return chalk.green.bold(grade);
+      return (s) => chalk.green.bold(s);
     case "B":
-      return chalk.greenBright.bold(grade);
+      return (s) => chalk.greenBright.bold(s);
     case "C":
-      return chalk.yellow.bold(grade);
+      return (s) => chalk.yellow.bold(s);
     case "D":
-      return chalk.rgb(255, 165, 0).bold(grade);
+      return (s) => chalk.rgb(255, 165, 0).bold(s);
     case "F":
-      return chalk.red.bold(grade);
+      return (s) => chalk.red.bold(s);
   }
 }
 function overallColor(overall) {
@@ -4782,104 +4805,136 @@ function overallColor(overall) {
       return overall;
   }
 }
-function testLine(t) {
-  const icon = t.passed ? chalk.green("  PASS") : chalk.red("  FAIL");
-  const req = t.required ? chalk.dim(" (required)") : "";
-  const dur = chalk.dim(` ${t.durationMs}ms`);
-  let line = `${icon}  ${t.name}${req}${dur}
-${chalk.dim(`         ${t.details}`)}`;
-  if (!t.passed) {
-    const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
-    if (def?.recommendation) {
-      line += `
-${chalk.cyan(`         Fix: ${def.recommendation}`)}`;
-    }
-  }
-  return line;
+function makeBar(passed, total, width = 24) {
+  if (total === 0) return { filled: "", rest: "\u2500".repeat(width) };
+  const n = Math.max(0, Math.min(width, Math.round(passed / total * width)));
+  return { filled: "\u2588".repeat(n), rest: "\u2591".repeat(width - n) };
+}
+function barColor(passed, total) {
+  if (total === 0) return (s) => chalk.dim(s);
+  const pct2 = passed / total;
+  if (pct2 >= 1) return (s) => chalk.green(s);
+  if (pct2 >= 0.85) return (s) => chalk.greenBright(s);
+  if (pct2 >= 0.6) return (s) => chalk.yellow(s);
+  return (s) => chalk.red(s);
+}
+function padRight(s, n) {
+  return s.length >= n ? s : s + " ".repeat(n - s.length);
 }
+function padLeft(s, n) {
+  return s.length >= n ? s : " ".repeat(n - s.length) + s;
+}
+var RULE = "\u2500".repeat(62);
+var HEAVY_RULE = "\u2550".repeat(62);
 function formatTerminal(report) {
-  const lines = [];
-  lines.push("");
-  lines.push(chalk.bold("MCP Compliance Report"));
-  lines.push(chalk.dim(`Spec: ${report.specVersion}  |  Tool: v${report.toolVersion}  |  ${report.timestamp}`));
-  lines.push(chalk.dim(`URL: ${report.url}`));
+  const out = [];
+  const color = gradeColor(report.grade);
+  const art = GRADE_ART[report.grade];
+  out.push("");
+  out.push(chalk.bold("  MCP COMPLIANCE REPORT"));
+  out.push(chalk.dim(`  ${HEAVY_RULE}`));
   if (report.serverInfo.name) {
-    lines.push(
-      chalk.dim(
-        `Server: ${report.serverInfo.name} v${report.serverInfo.version || "?"} (protocol ${report.serverInfo.protocolVersion || "?"})`
-      )
-    );
+    const v = report.serverInfo.version ? ` v${report.serverInfo.version}` : "";
+    const proto = report.serverInfo.protocolVersion ? `  (protocol ${report.serverInfo.protocolVersion})` : "";
+    out.push(chalk.dim(`  Server:   ${report.serverInfo.name}${v}${proto}`));
   }
-  lines.push("");
-  lines.push(
-    `  Grade: ${gradeColor(report.grade)}  Score: ${chalk.bold(String(report.score))}%  Overall: ${overallColor(report.overall)}`
-  );
-  lines.push(
-    `  Tests: ${chalk.green(String(report.summary.passed))} passed / ${chalk.red(String(report.summary.failed))} failed / ${report.summary.total} total`
-  );
-  lines.push(`  Required: ${report.summary.requiredPassed}/${report.summary.required} passed`);
-  const grouped = {};
-  for (const t of report.tests) {
-    if (!grouped[t.category]) grouped[t.category] = [];
-    grouped[t.category].push(t);
+  out.push(chalk.dim(`  Target:   ${report.url}`));
+  out.push(chalk.dim(`  Spec:     ${report.specVersion}  \xB7  Tool v${report.toolVersion}  \xB7  ${report.timestamp}`));
+  out.push("");
+  const reqOk = report.summary.requiredPassed === report.summary.required;
+  const infoRows = [
+    "",
+    "",
+    `${chalk.bold("GRADE")}   ${color(report.grade)}       ${chalk.bold(`${report.score}%`)}`,
+    `${chalk.dim("Overall   ")}${overallColor(report.overall)}`,
+    `${chalk.dim("Tests     ")}${chalk.green(String(report.summary.passed))}${chalk.dim("/")}${report.summary.total}${report.summary.failed > 0 ? chalk.dim("  (") + chalk.red(`${report.summary.failed} failed`) + chalk.dim(")") : ""}`,
+    `${chalk.dim("Required  ")}${reqOk ? chalk.green(`${report.summary.requiredPassed}/${report.summary.required} \u2713`) : chalk.red(`${report.summary.requiredPassed}/${report.summary.required}`)}`
+  ];
+  for (let i = 0; i < 6; i++) {
+    out.push(`       ${color(art[i])}     ${infoRows[i] || ""}`);
   }
-  for (const cat of CATEGORY_ORDER) {
-    const catTests = grouped[cat];
-    if (!catTests || catTests.length === 0) continue;
-    const catStats = report.categories[cat];
+  out.push("");
+  out.push(chalk.bold("  CATEGORY BREAKDOWN"));
+  out.push(chalk.dim(`  ${RULE}`));
+  const cats = CATEGORY_ORDER.filter((c) => report.categories[c] && report.categories[c].total > 0);
+  const maxLabel = Math.max(...cats.map((c) => (CATEGORY_LABELS[c] || c).length));
+  for (const cat of cats) {
+    const stats = report.categories[cat];
     const label = CATEGORY_LABELS[cat] || cat;
-    const catColor = catStats && catStats.passed === catStats.total ? chalk.green : chalk.yellow;
-    lines.push("");
-    lines.push(catColor(`  ${label} (${catStats?.passed || 0}/${catStats?.total || 0})`));
-    for (const t of catTests) {
-      lines.push(testLine(t));
-    }
-  }
-  const caps = report.serverInfo.capabilities;
-  const declared = Object.keys(caps).filter((k) => caps[k] !== void 0);
-  if (declared.length > 0) {
-    lines.push("");
-    lines.push(chalk.dim(`  Capabilities: ${declared.join(", ")}`));
-  }
-  if (report.toolCount > 0) {
-    lines.push(
-      chalk.dim(
-        `  Tools (${report.toolCount}): ${report.toolNames.slice(0, 10).join(", ")}${report.toolCount > 10 ? "..." : ""}`
-      )
-    );
-  }
-  if (report.resourceCount > 0) {
-    lines.push(
-      chalk.dim(
-        `  Resources (${report.resourceCount}): ${report.resourceNames.slice(0, 10).join(", ")}${report.resourceCount > 10 ? "..." : ""}`
-      )
+    const { filled, rest } = makeBar(stats.passed, stats.total, 24);
+    const colorFn = barColor(stats.passed, stats.total);
+    const pct2 = stats.total === 0 ? 0 : Math.round(stats.passed / stats.total * 100);
+    const ratio = `${stats.passed}/${stats.total}`;
+    out.push(
+      `  ${padRight(label, maxLabel)}  ${colorFn(filled)}${chalk.dim(rest)}  ${padLeft(ratio, 7)}  ${padLeft(`${pct2}%`, 4)}`
     );
   }
-  if (report.promptCount > 0) {
-    lines.push(
-      chalk.dim(
-        `  Prompts (${report.promptCount}): ${report.promptNames.slice(0, 10).join(", ")}${report.promptCount > 10 ? "..." : ""}`
-      )
-    );
+  out.push("");
+  const failed = report.tests.filter((t) => !t.passed);
+  if (failed.length > 0) {
+    out.push(chalk.bold.red(`  FAILED TESTS (${failed.length})`));
+    out.push(chalk.dim(`  ${RULE}`));
+    for (const t of failed) {
+      const req = t.required ? chalk.red("required") : chalk.dim("optional");
+      out.push(
+        `  ${chalk.red("\u2717")} ${chalk.bold(t.name)}  ${chalk.dim(`[${t.id}]`)}  ${req}  ${chalk.dim(`${t.durationMs}ms`)}`
+      );
+      out.push(`      ${t.details}`);
+      const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
+      if (def?.recommendation) {
+        out.push(`      ${chalk.cyan(`\u2192 ${def.recommendation}`)}`);
+      }
+      if (t.specRef) {
+        out.push(chalk.dim(`      spec: ${t.specRef}`));
+      }
+      out.push("");
+    }
+  } else {
+    out.push(`  ${chalk.green.bold("\u2713 All tests passed")}`);
+    out.push("");
   }
   if (report.warnings.length > 0) {
-    lines.push("");
-    lines.push(chalk.yellow(`  Warnings (${report.warnings.length}):`));
+    out.push(chalk.bold.yellow(`  WARNINGS (${report.warnings.length})`));
+    out.push(chalk.dim(`  ${RULE}`));
     for (const w of report.warnings) {
-      lines.push(chalk.yellow(`    - ${w}`));
+      out.push(`  ${chalk.yellow("!")} ${w}`);
     }
+    out.push("");
+  }
+  const caps = report.serverInfo.capabilities;
+  const declared = Object.keys(caps).filter((k) => caps[k] !== void 0);
+  const hasContext = declared.length > 0 || report.toolCount > 0 || report.resourceCount > 0 || report.promptCount > 0;
+  if (hasContext) {
+    out.push(chalk.bold("  SERVER CONTEXT"));
+    out.push(chalk.dim(`  ${RULE}`));
+    if (declared.length > 0) {
+      out.push(chalk.dim(`  Capabilities:  ${declared.join(", ")}`));
+    }
+    if (report.toolCount > 0) {
+      const more = report.toolCount > 10 ? ", ..." : "";
+      out.push(chalk.dim(`  Tools (${report.toolCount}):      ${report.toolNames.slice(0, 10).join(", ")}${more}`));
+    }
+    if (report.resourceCount > 0) {
+      const more = report.resourceCount > 10 ? ", ..." : "";
+      out.push(
+        chalk.dim(`  Resources (${report.resourceCount}):  ${report.resourceNames.slice(0, 10).join(", ")}${more}`)
+      );
+    }
+    if (report.promptCount > 0) {
+      const more = report.promptCount > 10 ? ", ..." : "";
+      out.push(chalk.dim(`  Prompts (${report.promptCount}):    ${report.promptNames.slice(0, 10).join(", ")}${more}`));
+    }
+    out.push("");
   }
-  lines.push("");
   if (report.url.startsWith("stdio:")) {
-    lines.push(
-      chalk.dim("  Badge: stdio servers can't be published. Use `--output badge.svg` for a local badge image.")
+    out.push(
+      chalk.dim("  Badge:    stdio targets aren't published. Run with --output badge.svg for a local badge image.")
     );
   } else {
-    lines.push(chalk.dim("  Badge markdown:"));
-    lines.push(`  ${report.badge.markdown}`);
+    out.push(chalk.dim(`  Badge:    ${report.badge.markdown}`));
   }
-  lines.push("");
-  return lines.join("\n");
+  out.push("");
+  return out.join("\n");
 }
 function formatJson(report) {
   return JSON.stringify(report, null, 2);
@@ -5255,6 +5310,16 @@ function parsePositiveInt(value, name, min = 0) {
 function parseList(value) {
   return value.split(",").map((s) => s.trim()).filter(Boolean);
 }
+function ttyHint(format) {
+  if (!process.stdout.isTTY) return;
+  process.stderr.write(
+    chalk2.dim(
+      `  [tip] --format=${format} is machine-readable. Redirect with \`> report.${format === "sarif" ? "sarif" : format}\` or drop --format for the human report.
+
+`
+    )
+  );
+}
 function parseEnvVar(value, prev) {
   const idx = value.indexOf("=");
   if (idx === -1) throw new Error(`Invalid env var: "${value}" (expected "KEY=VALUE")`);
@@ -5404,8 +5469,11 @@ program.command("test").description("Run the full compliance test suite against
   {}
 ).option("--auth <token>", 'Shorthand for -H "Authorization: <token>" (HTTP only)').option("-E, --env <var>", 'Set env var for stdio command ("KEY=VALUE", repeatable)', parseEnvVar, {}).option("--env-file <path>", "Load env vars from file (KEY=VALUE per line, stdio only)").option("--cwd <dir>", "Working directory for stdio command").option(
   "--timeout <ms>",
-  "Request timeout in milliseconds (bump to 30000+ for stdio servers with slow startup)",
+  "Per-request timeout in milliseconds (applies to every test request after the initial handshake)",
   "15000"
+).option(
+  "--startup-timeout <ms>",
+  "Deadline for the initial initialize handshake (default: max(--timeout, 60000); covers cold `npx` cache fetches before a stdio server starts)"
 ).option("--no-color", "Disable colored output (also honors NO_COLOR env var)").option("--watch", "Re-run tests when files in the cwd change (stdio targets only)").option(
   "--concurrency <n>",
   "Max parallel-safe tests in flight (default 1; see docs/PERFORMANCE.md before raising)",
@@ -5467,6 +5535,7 @@ Testing ${describeTarget(transportTarget)}...
         }
         const report2 = await runComplianceSuite(transportTarget, {
           timeout: parsePositiveInt(opts.timeout, "--timeout", 1),
+          startupTimeout: opts.startupTimeout ? parsePositiveInt(opts.startupTimeout, "--startup-timeout", 1) : config?.startupTimeout,
           preflightTimeout: opts.preflightTimeout ? parsePositiveInt(opts.preflightTimeout, "--preflight-timeout", 1) : config?.preflightTimeout,
           retries: parsePositiveInt(opts.retries, "--retries"),
           concurrency: parsePositiveInt(opts.concurrency, "--concurrency", 1),
@@ -5483,14 +5552,17 @@ Testing ${describeTarget(transportTarget)}...
           console.log("");
         }
         if (opts.format === "json") {
+          ttyHint("json");
           console.log(formatJson(report2));
         } else if (opts.format === "sarif") {
+          ttyHint("sarif");
           console.log(formatSarif(report2));
         } else if (opts.format === "github") {
           console.log(formatGithub(report2));
         } else if (opts.format === "markdown") {
           console.log(formatMarkdown(report2));
         } else if (opts.format === "html") {
+          ttyHint("html");
           console.log(formatHtml(report2));
         } else {
           console.log(formatTerminal(report2));

package/dist/mcp/server.js

@@ -2,7 +2,7 @@ import {
   SPEC_BASE,
   TEST_DEFINITIONS,
   runComplianceSuite
-} from "../chunk-X5CVUDPW.js";
+} from "../chunk-6PF56RRO.js";
 
 // src/mcp/server.ts
 import { existsSync, readFileSync, realpathSync } from "fs";

package/dist/runner.d.ts

@@ -180,6 +180,18 @@ interface RunOptions {
     headers?: Record<string, string>;
     /** Request timeout in milliseconds (default: 15000) */
     timeout?: number;
+    /**
+     * Deadline for the initial `initialize` handshake + `initialized`
+     * notification, in milliseconds. Kept separate from `timeout` because
+     * cold-started stdio servers — especially `npx @pkg ...` targets where
+     * npm has to resolve and fetch the package before the MCP server
+     * starts — can take tens of seconds to produce their first response,
+     * while steady-state requests complete in milliseconds. Default is
+     * `max(timeout, 60000)` so users who bump `--timeout` keep that value
+     * and users on defaults get 60s for startup. Does not apply to any
+     * per-test requests past the handshake.
+     */
+    startupTimeout?: number;
     /** Number of retries for failed tests (default: 0) */
     retries?: number;
     /** Only run tests matching these category names or test IDs */

package/dist/runner.js

@@ -10,7 +10,7 @@ import {
   previewTests,
   runComplianceSuite,
   urlHash
-} from "./chunk-X5CVUDPW.js";
+} from "./chunk-6PF56RRO.js";
 export {
   SPEC_BASE,
   SPEC_VERSION,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yawlabs/mcp-compliance",
-  "version": "0.13.3",
+  "version": "0.13.5",
   "description": "CLI tool and MCP server that tests MCP servers for spec compliance",
   "license": "MIT",
   "author": "Yaw Labs <contact@yaw.sh> (https://yaw.sh)",