@yawlabs/mcp-compliance 0.7.0 → 0.8.0

package/README.md

@@ -5,7 +5,7 @@
 [![GitHub stars](https://img.shields.io/github/stars/YawLabs/mcp-compliance)](https://github.com/YawLabs/mcp-compliance/stargazers)
 [![CI](https://github.com/YawLabs/mcp-compliance/actions/workflows/ci.yml/badge.svg)](https://github.com/YawLabs/mcp-compliance/actions/workflows/ci.yml)
 
-**Test any MCP server for spec compliance.** 78-test suite covering transport, lifecycle, tools, resources, prompts, error handling, schema validation, and security against the [MCP specification](https://modelcontextprotocol.io/specification/2025-11-25). CLI, MCP server, and programmatic API.
+**Test any MCP server for spec compliance.** 81-test suite covering transport, lifecycle, tools, resources, prompts, error handling, schema validation, and security against the [MCP specification](https://modelcontextprotocol.io/specification/2025-11-25). CLI, MCP server, and programmatic API.
 
 Built and maintained by [Yaw Labs](https://yaw.sh).
 
@@ -15,7 +15,7 @@ MCP servers are multiplying fast — but most ship without compliance testing. B
 
 This tool solves that:
 
-- **78 tests across 8 categories** — transport, lifecycle, tools, resources, prompts, error handling, schema validation, and security. No gaps.
+- **81 tests across 8 categories** — transport, lifecycle, tools, resources, prompts, error handling, schema validation, and security. No gaps.
 - **Capability-driven** — tests adapt to what the server declares. If it says it supports tools, tool tests become required. No false failures for features the server doesn't claim.
 - **Graded scoring** — A-F letter grade with a weighted score (required tests 70%, optional 30%). One number to communicate compliance.
 - **CI-ready** — `--strict` mode exits with code 1 on required test failures. Drop it into any pipeline.
@@ -100,7 +100,7 @@ mcp-compliance badge https://my-server.com/mcp
 
 Outputs the markdown embed for a compliance badge hosted at [mcp.hosting](https://mcp.hosting).
 
-## What the 78 tests check
+## What the 81 tests check
 
 <details>
 <summary><strong>Transport (13 tests)</strong></summary>
@@ -122,7 +122,7 @@ Outputs the markdown embed for a compliance badge hosted at [mcp.hosting](https:
 </details>
 
 <details>
-<summary><strong>Lifecycle (15 tests)</strong></summary>
+<summary><strong>Lifecycle (17 tests)</strong></summary>
 
 - **lifecycle-init** — Initialize handshake succeeds (required)
 - **lifecycle-proto-version** — Returns valid YYYY-MM-DD protocol version (required)
@@ -139,6 +139,8 @@ Outputs the markdown embed for a compliance badge hosted at [mcp.hosting](https:
 - **lifecycle-completions** — completion/complete accepted (required if completions capability declared)
 - **lifecycle-cancellation** — Handles cancellation notifications
 - **lifecycle-progress** — Handles progress notifications gracefully
+- **lifecycle-list-changed** — Accepts listChanged notifications for declared capabilities
+- **lifecycle-progress-token** — Supports progress tokens in requests via SSE
 
 </details>
 
@@ -201,9 +203,10 @@ Outputs the markdown embed for a compliance badge hosted at [mcp.hosting](https:
 </details>
 
 <details>
-<summary><strong>Security (22 tests)</strong></summary>
+<summary><strong>Security (23 tests)</strong></summary>
 
 - **security-auth-required** — Rejects unauthenticated requests
+- **security-www-authenticate** — 401 responses include WWW-Authenticate header
 - **security-auth-malformed** — Rejects malformed auth credentials
 - **security-tls-required** — Enforces HTTPS/TLS
 - **security-session-entropy** — Session IDs are high-entropy
@@ -318,7 +321,7 @@ Restart your MCP client and approve the server when prompted.
 
 ### Tools
 
-- **mcp_compliance_test** — Run the full 78-test suite against a URL. Supports auth, custom headers, timeout, retries, and category/test filtering. Returns grade, score, and detailed results.
+- **mcp_compliance_test** — Run the full 81-test suite against a URL. Supports auth, custom headers, timeout, retries, and category/test filtering. Returns grade, score, and detailed results.
 - **mcp_compliance_badge** — Get the badge markdown/HTML for a server. Supports auth and custom headers.
 - **mcp_compliance_explain** — Explain what a specific test ID checks and why it matters.
 
@@ -345,7 +348,7 @@ const report2 = await runComplianceSuite('https://my-server.com/mcp', {
 
 The compliance testing methodology is published as an open specification:
 
-- **[MCP Compliance Testing Specification](./MCP_COMPLIANCE_SPEC.md)** — test execution model, scoring algorithm, all 78 test rules with pass/fail criteria (CC BY 4.0)
+- **[MCP Compliance Testing Specification](./MCP_COMPLIANCE_SPEC.md)** — test execution model, scoring algorithm, all 81 test rules with pass/fail criteria (CC BY 4.0)
 - **[Machine-readable rule catalog](./mcp-compliance-rules.json)** — JSON Schema-compliant catalog for programmatic consumption
 
 These are complementary to (not competing with) the [official MCP specification](https://modelcontextprotocol.io/specification/2025-11-25). The MCP spec defines what servers must do; this spec defines how to verify compliance.

package/dist/{chunk-SELO4TOW.js → chunk-DOIOJVEE.js}

@@ -174,7 +174,7 @@ var TEST_DEFINITIONS = [
     description: "Sends a request with Accept: text/event-stream and checks that SSE responses include the event: message field. Per spec, servers MUST set event: message for JSON-RPC messages in SSE streams.",
     recommendation: 'Include "event: message" before each "data:" line in your SSE responses. This is required by the MCP spec for JSON-RPC messages sent over SSE.'
   },
-  // ── Lifecycle (15 tests) ─────────────────────────────────────────
+  // ── Lifecycle (17 tests) ─────────────────────────────────────────
   {
     id: "lifecycle-init",
     name: "Initialize handshake",
@@ -310,6 +310,24 @@ var TEST_DEFINITIONS = [
     description: "Sends a notifications/progress to the server and verifies it does not error. Note: per spec, progress flows from server to client during long-running requests. This test validates the server handles unexpected notifications gracefully.",
     recommendation: "Accept unknown notifications without returning an error. The server should not crash or return a non-2xx status for notifications it does not recognize."
   },
+  {
+    id: "lifecycle-list-changed",
+    name: "Accepts listChanged notifications",
+    category: "lifecycle",
+    required: false,
+    specRef: "basic/lifecycle#capability-negotiation",
+    description: "Sends notifications/tools/list_changed, notifications/resources/list_changed, and notifications/prompts/list_changed for declared capabilities and verifies the server accepts them.",
+    recommendation: "Accept listChanged notifications gracefully. When received, re-fetch the relevant list to detect changes. These notifications signal that the client's cached list may be stale."
+  },
+  {
+    id: "lifecycle-progress-token",
+    name: "Supports progress tokens in requests",
+    category: "lifecycle",
+    required: false,
+    specRef: "basic/utilities#progress",
+    description: "Sends a tools/call request with _meta.progressToken and checks if the server sends progress notifications via SSE. Progress support is optional but recommended for long-running operations.",
+    recommendation: "When a request includes _meta.progressToken, send notifications/progress events via SSE to report progress. Include progressToken, progress (current), and optionally total fields."
+  },
   // ── Tools (4 tests) ──────────────────────────────────────────────
   {
     id: "tools-list",
@@ -567,7 +585,7 @@ var TEST_DEFINITIONS = [
     description: "Validates every resource has a valid URI (parseable as a URL) and a name field.",
     recommendation: "Ensure every resource has a valid, parseable URI and a name field. Add description and mimeType for better client integration."
   },
-  // ── Security: Auth & Transport (9 tests) ─────────────────────────
+  // ── Security: Auth & Transport (10 tests) ────────────────────────
   {
     id: "security-auth-required",
     name: "Rejects unauthenticated requests",
@@ -577,6 +595,15 @@ var TEST_DEFINITIONS = [
     description: "Sends a request without an Authorization header and verifies the server returns HTTP 401. Servers exposed over the network should require authentication.",
     recommendation: "Implement authentication on your MCP endpoint. Return HTTP 401 Unauthorized for requests without valid credentials. Use OAuth 2.1 or Bearer tokens as recommended by the MCP spec."
   },
+  {
+    id: "security-www-authenticate",
+    name: "401 responses include WWW-Authenticate header",
+    category: "security",
+    required: false,
+    specRef: "basic/authorization",
+    description: "When the server returns HTTP 401, checks for a WWW-Authenticate header indicating the required authentication scheme. Per HTTP spec (RFC 9110), servers SHOULD include this header.",
+    recommendation: `Include a WWW-Authenticate header in 401 responses to indicate the required auth scheme (e.g., 'WWW-Authenticate: Bearer realm="mcp"').`
+  },
   {
     id: "security-auth-malformed",
     name: "Rejects malformed auth credentials",
@@ -1494,6 +1521,90 @@ async function runComplianceSuite(url, options = {}) {
       };
     }
   );
+  await test(
+    "lifecycle-list-changed",
+    "Accepts listChanged notifications",
+    "lifecycle",
+    false,
+    "basic/lifecycle#capability-negotiation",
+    async () => {
+      const notifications = [
+        { method: "notifications/tools/list_changed", gate: hasTools },
+        { method: "notifications/resources/list_changed", gate: hasResources },
+        { method: "notifications/prompts/list_changed", gate: hasPrompts }
+      ];
+      const applicable = notifications.filter((n) => n.gate);
+      if (applicable.length === 0) {
+        return { passed: true, details: "No capabilities declared \u2014 listChanged notifications not applicable" };
+      }
+      const issues = [];
+      for (const { method } of applicable) {
+        try {
+          const res = await mcpNotification(backendUrl, method, void 0, buildHeaders(), timeout);
+          if (res.statusCode < 200 || res.statusCode >= 300) {
+            issues.push(`${method}: HTTP ${res.statusCode}`);
+          }
+        } catch (err) {
+          issues.push(`${method}: ${err instanceof Error ? err.message : "error"}`);
+        }
+      }
+      if (issues.length > 0) return { passed: false, details: issues.join("; ") };
+      return {
+        passed: true,
+        details: `${applicable.length} listChanged notification(s) accepted: ${applicable.map((n) => n.method).join(", ")}`
+      };
+    }
+  );
+  await test(
+    "lifecycle-progress-token",
+    "Supports progress tokens in requests",
+    "lifecycle",
+    false,
+    "basic/utilities#progress",
+    async () => {
+      if (!hasTools || toolNames.length === 0) {
+        return { passed: true, details: "No tools available for progress token test (skipped)" };
+      }
+      const progressToken = "compliance-progress-test";
+      const reqBody = JSON.stringify({
+        jsonrpc: "2.0",
+        id: nextId(),
+        method: "tools/call",
+        params: {
+          name: toolNames[0],
+          arguments: {},
+          _meta: { progressToken }
+        }
+      });
+      try {
+        const res = await request(backendUrl, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Accept: "text/event-stream",
+            ...buildHeaders()
+          },
+          body: reqBody,
+          signal: AbortSignal.timeout(timeout)
+        });
+        const text = await res.body.text();
+        const rawCtProgress = res.headers["content-type"];
+        const ct = (Array.isArray(rawCtProgress) ? rawCtProgress[0] : rawCtProgress || "").toLowerCase();
+        if (ct.includes("text/event-stream") && text.includes("notifications/progress")) {
+          return { passed: true, details: "Server sent progress notifications via SSE with progressToken" };
+        }
+        if (res.statusCode >= 200 && res.statusCode < 300) {
+          return {
+            passed: true,
+            details: "Server accepted request with progressToken (no progress events observed \u2014 optional)"
+          };
+        }
+        return { passed: true, details: `HTTP ${res.statusCode} \u2014 request with progressToken accepted` };
+      } catch {
+        return { passed: true, details: "Request with progressToken handled (no progress events observed \u2014 optional)" };
+      }
+    }
+  );
   await test(
     "transport-content-type-init",
     "Initialize response has valid content type",
@@ -2428,6 +2539,39 @@ async function runComplianceSuite(url, options = {}) {
       }
     }
   );
+  await test(
+    "security-www-authenticate",
+    "401 responses include WWW-Authenticate header",
+    "security",
+    false,
+    "basic/authorization",
+    async () => {
+      if (!hasAuth) {
+        return { passed: true, details: "Skipped: server does not require auth" };
+      }
+      const noAuthHeaders = {};
+      if (sessionId) noAuthHeaders["mcp-session-id"] = sessionId;
+      try {
+        const res = await mcpRequest(backendUrl, "ping", void 0, nextId, noAuthHeaders, timeout);
+        if (res.statusCode === 401) {
+          const wwwAuth = res.headers["www-authenticate"];
+          if (wwwAuth) {
+            return { passed: true, details: `WWW-Authenticate: ${wwwAuth}` };
+          }
+          return {
+            passed: false,
+            details: "HTTP 401 but missing WWW-Authenticate header (spec: SHOULD include to indicate required auth scheme)"
+          };
+        }
+        if (res.statusCode === 403) {
+          return { passed: true, details: "HTTP 403 (WWW-Authenticate not applicable for 403)" };
+        }
+        return { passed: true, details: `HTTP ${res.statusCode} \u2014 not a 401 response` };
+      } catch {
+        return { passed: true, details: "Connection rejected (acceptable)" };
+      }
+    }
+  );
   await test(
     "security-auth-malformed",
     "Rejects malformed auth credentials",

package/dist/index.js

@@ -189,7 +189,7 @@ var TEST_DEFINITIONS = [
     description: "Sends a request with Accept: text/event-stream and checks that SSE responses include the event: message field. Per spec, servers MUST set event: message for JSON-RPC messages in SSE streams.",
     recommendation: 'Include "event: message" before each "data:" line in your SSE responses. This is required by the MCP spec for JSON-RPC messages sent over SSE.'
   },
-  // ── Lifecycle (15 tests) ─────────────────────────────────────────
+  // ── Lifecycle (17 tests) ─────────────────────────────────────────
   {
     id: "lifecycle-init",
     name: "Initialize handshake",
@@ -325,6 +325,24 @@ var TEST_DEFINITIONS = [
     description: "Sends a notifications/progress to the server and verifies it does not error. Note: per spec, progress flows from server to client during long-running requests. This test validates the server handles unexpected notifications gracefully.",
     recommendation: "Accept unknown notifications without returning an error. The server should not crash or return a non-2xx status for notifications it does not recognize."
   },
+  {
+    id: "lifecycle-list-changed",
+    name: "Accepts listChanged notifications",
+    category: "lifecycle",
+    required: false,
+    specRef: "basic/lifecycle#capability-negotiation",
+    description: "Sends notifications/tools/list_changed, notifications/resources/list_changed, and notifications/prompts/list_changed for declared capabilities and verifies the server accepts them.",
+    recommendation: "Accept listChanged notifications gracefully. When received, re-fetch the relevant list to detect changes. These notifications signal that the client's cached list may be stale."
+  },
+  {
+    id: "lifecycle-progress-token",
+    name: "Supports progress tokens in requests",
+    category: "lifecycle",
+    required: false,
+    specRef: "basic/utilities#progress",
+    description: "Sends a tools/call request with _meta.progressToken and checks if the server sends progress notifications via SSE. Progress support is optional but recommended for long-running operations.",
+    recommendation: "When a request includes _meta.progressToken, send notifications/progress events via SSE to report progress. Include progressToken, progress (current), and optionally total fields."
+  },
   // ── Tools (4 tests) ──────────────────────────────────────────────
   {
     id: "tools-list",
@@ -582,7 +600,7 @@ var TEST_DEFINITIONS = [
     description: "Validates every resource has a valid URI (parseable as a URL) and a name field.",
     recommendation: "Ensure every resource has a valid, parseable URI and a name field. Add description and mimeType for better client integration."
   },
-  // ── Security: Auth & Transport (9 tests) ─────────────────────────
+  // ── Security: Auth & Transport (10 tests) ────────────────────────
   {
     id: "security-auth-required",
     name: "Rejects unauthenticated requests",
@@ -592,6 +610,15 @@ var TEST_DEFINITIONS = [
     description: "Sends a request without an Authorization header and verifies the server returns HTTP 401. Servers exposed over the network should require authentication.",
     recommendation: "Implement authentication on your MCP endpoint. Return HTTP 401 Unauthorized for requests without valid credentials. Use OAuth 2.1 or Bearer tokens as recommended by the MCP spec."
   },
+  {
+    id: "security-www-authenticate",
+    name: "401 responses include WWW-Authenticate header",
+    category: "security",
+    required: false,
+    specRef: "basic/authorization",
+    description: "When the server returns HTTP 401, checks for a WWW-Authenticate header indicating the required authentication scheme. Per HTTP spec (RFC 9110), servers SHOULD include this header.",
+    recommendation: `Include a WWW-Authenticate header in 401 responses to indicate the required auth scheme (e.g., 'WWW-Authenticate: Bearer realm="mcp"').`
+  },
   {
     id: "security-auth-malformed",
     name: "Rejects malformed auth credentials",
@@ -1509,6 +1536,90 @@ async function runComplianceSuite(url, options = {}) {
       };
     }
   );
+  await test(
+    "lifecycle-list-changed",
+    "Accepts listChanged notifications",
+    "lifecycle",
+    false,
+    "basic/lifecycle#capability-negotiation",
+    async () => {
+      const notifications = [
+        { method: "notifications/tools/list_changed", gate: hasTools },
+        { method: "notifications/resources/list_changed", gate: hasResources },
+        { method: "notifications/prompts/list_changed", gate: hasPrompts }
+      ];
+      const applicable = notifications.filter((n) => n.gate);
+      if (applicable.length === 0) {
+        return { passed: true, details: "No capabilities declared \u2014 listChanged notifications not applicable" };
+      }
+      const issues = [];
+      for (const { method } of applicable) {
+        try {
+          const res = await mcpNotification(backendUrl, method, void 0, buildHeaders(), timeout);
+          if (res.statusCode < 200 || res.statusCode >= 300) {
+            issues.push(`${method}: HTTP ${res.statusCode}`);
+          }
+        } catch (err) {
+          issues.push(`${method}: ${err instanceof Error ? err.message : "error"}`);
+        }
+      }
+      if (issues.length > 0) return { passed: false, details: issues.join("; ") };
+      return {
+        passed: true,
+        details: `${applicable.length} listChanged notification(s) accepted: ${applicable.map((n) => n.method).join(", ")}`
+      };
+    }
+  );
+  await test(
+    "lifecycle-progress-token",
+    "Supports progress tokens in requests",
+    "lifecycle",
+    false,
+    "basic/utilities#progress",
+    async () => {
+      if (!hasTools || toolNames.length === 0) {
+        return { passed: true, details: "No tools available for progress token test (skipped)" };
+      }
+      const progressToken = "compliance-progress-test";
+      const reqBody = JSON.stringify({
+        jsonrpc: "2.0",
+        id: nextId(),
+        method: "tools/call",
+        params: {
+          name: toolNames[0],
+          arguments: {},
+          _meta: { progressToken }
+        }
+      });
+      try {
+        const res = await request(backendUrl, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Accept: "text/event-stream",
+            ...buildHeaders()
+          },
+          body: reqBody,
+          signal: AbortSignal.timeout(timeout)
+        });
+        const text = await res.body.text();
+        const rawCtProgress = res.headers["content-type"];
+        const ct = (Array.isArray(rawCtProgress) ? rawCtProgress[0] : rawCtProgress || "").toLowerCase();
+        if (ct.includes("text/event-stream") && text.includes("notifications/progress")) {
+          return { passed: true, details: "Server sent progress notifications via SSE with progressToken" };
+        }
+        if (res.statusCode >= 200 && res.statusCode < 300) {
+          return {
+            passed: true,
+            details: "Server accepted request with progressToken (no progress events observed \u2014 optional)"
+          };
+        }
+        return { passed: true, details: `HTTP ${res.statusCode} \u2014 request with progressToken accepted` };
+      } catch {
+        return { passed: true, details: "Request with progressToken handled (no progress events observed \u2014 optional)" };
+      }
+    }
+  );
   await test(
     "transport-content-type-init",
     "Initialize response has valid content type",
@@ -2443,6 +2554,39 @@ async function runComplianceSuite(url, options = {}) {
       }
     }
   );
+  await test(
+    "security-www-authenticate",
+    "401 responses include WWW-Authenticate header",
+    "security",
+    false,
+    "basic/authorization",
+    async () => {
+      if (!hasAuth) {
+        return { passed: true, details: "Skipped: server does not require auth" };
+      }
+      const noAuthHeaders = {};
+      if (sessionId) noAuthHeaders["mcp-session-id"] = sessionId;
+      try {
+        const res = await mcpRequest(backendUrl, "ping", void 0, nextId, noAuthHeaders, timeout);
+        if (res.statusCode === 401) {
+          const wwwAuth = res.headers["www-authenticate"];
+          if (wwwAuth) {
+            return { passed: true, details: `WWW-Authenticate: ${wwwAuth}` };
+          }
+          return {
+            passed: false,
+            details: "HTTP 401 but missing WWW-Authenticate header (spec: SHOULD include to indicate required auth scheme)"
+          };
+        }
+        if (res.statusCode === 403) {
+          return { passed: true, details: "HTTP 403 (WWW-Authenticate not applicable for 403)" };
+        }
+        return { passed: true, details: `HTTP ${res.statusCode} \u2014 not a 401 response` };
+      } catch {
+        return { passed: true, details: "Connection rejected (acceptable)" };
+      }
+    }
+  );
   await test(
     "security-auth-malformed",
     "Rejects malformed auth credentials",
@@ -3258,7 +3402,7 @@ async function runComplianceSuite(url, options = {}) {
 function registerTools(server) {
   server.tool(
     "mcp_compliance_test",
-    "Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all 78 tests covering transport, lifecycle, tools, resources, prompts, errors, schema validation, and security.",
+    "Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all 81 tests covering transport, lifecycle, tools, resources, prompts, errors, schema validation, and security.",
     {
       url: z.string().url().describe("The MCP server URL to test (must be HTTP or HTTPS)"),
       auth: z.string().optional().describe('Authorization header value (e.g., "Bearer tok123")'),

package/dist/mcp/server.js

@@ -2,7 +2,7 @@ import {
   SPEC_BASE,
   TEST_DEFINITIONS,
   runComplianceSuite
-} from "../chunk-SELO4TOW.js";
+} from "../chunk-DOIOJVEE.js";
 
 // src/mcp/server.ts
 import { createRequire } from "module";
@@ -14,7 +14,7 @@ import { z } from "zod";
 function registerTools(server) {
   server.tool(
     "mcp_compliance_test",
-    "Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all 78 tests covering transport, lifecycle, tools, resources, prompts, errors, schema validation, and security.",
+    "Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all 81 tests covering transport, lifecycle, tools, resources, prompts, errors, schema validation, and security.",
     {
       url: z.string().url().describe("The MCP server URL to test (must be HTTP or HTTPS)"),
       auth: z.string().optional().describe('Authorization header value (e.g., "Bearer tok123")'),

package/dist/runner.d.ts

@@ -60,7 +60,7 @@ interface TestDefinition {
     description: string;
     recommendation: string;
 }
-/** All 78 test IDs with descriptions for the explain command */
+/** All 81 test IDs with descriptions for the explain command */
 declare const TEST_DEFINITIONS: TestDefinition[];
 
 declare function computeGrade(score: number): Grade;

package/dist/runner.js

@@ -7,7 +7,7 @@ import {
   generateBadge,
   parseSSEResponse,
   runComplianceSuite
-} from "./chunk-SELO4TOW.js";
+} from "./chunk-DOIOJVEE.js";
 export {
   SPEC_BASE,
   SPEC_VERSION,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yawlabs/mcp-compliance",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "CLI tool and MCP server that tests MCP servers for spec compliance",
   "license": "MIT",
   "author": "Yaw Labs <contact@yaw.sh> (https://yaw.sh)",