@yawlabs/mcp-compliance 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,11 +1,14 @@
1
1
  #!/usr/bin/env node
2
2
 
3
3
  // src/index.ts
4
+ import { createRequire as createRequire3 } from "module";
5
+ import chalk2 from "chalk";
6
+ import { Command } from "commander";
7
+
8
+ // src/mcp/server.ts
4
9
  import { createRequire as createRequire2 } from "module";
5
10
  import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
6
11
  import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
7
- import chalk2 from "chalk";
8
- import { Command } from "commander";
9
12
 
10
13
  // src/mcp/tools.ts
11
14
  import { z } from "zod";
@@ -15,16 +18,14 @@ import { createRequire } from "module";
15
18
  import { request } from "undici";
16
19
 
17
20
  // src/badge.ts
21
+ import { createHash } from "crypto";
22
+ function urlHash(url) {
23
+ return createHash("sha256").update(url).digest("hex").slice(0, 12);
24
+ }
18
25
  function generateBadge(url) {
19
- let parsed;
20
- try {
21
- parsed = new URL(url);
22
- } catch {
23
- parsed = new URL("https://unknown");
24
- }
25
- const encoded = encodeURIComponent(parsed.href);
26
- const imageUrl = `https://mcp.hosting/api/compliance/${encoded}/badge`;
27
- const reportUrl = `https://mcp.hosting/compliance/${encoded}`;
26
+ const hash = urlHash(url);
27
+ const imageUrl = `https://mcp.hosting/api/compliance/${hash}/badge`;
28
+ const reportUrl = `https://mcp.hosting/compliance/${hash}`;
28
29
  return {
29
30
  imageUrl,
30
31
  reportUrl,
@@ -70,14 +71,15 @@ function computeScore(tests) {
70
71
 
71
72
  // src/types.ts
72
73
  var TEST_DEFINITIONS = [
73
- // ── Transport (7 tests) ──────────────────────────────────────────
74
+ // ── Transport (10 tests) ─────────────────────────────────────────
74
75
  {
75
76
  id: "transport-post",
76
77
  name: "HTTP POST accepted",
77
78
  category: "transport",
78
79
  required: true,
79
80
  specRef: "basic/transports#streamable-http",
80
- description: "Verifies the server accepts HTTP POST requests and returns a 2xx status code. This is the fundamental transport requirement for Streamable HTTP MCP servers."
81
+ description: "Verifies the server accepts HTTP POST requests and returns a 2xx status code. This is the fundamental transport requirement for Streamable HTTP MCP servers.",
82
+ recommendation: "Ensure your server listens for POST requests on the MCP endpoint. If you see 401/403, pass --auth with a valid token. Check that the URL is correct and the server is running."
81
83
  },
82
84
  {
83
85
  id: "transport-content-type",
@@ -85,7 +87,8 @@ var TEST_DEFINITIONS = [
85
87
  category: "transport",
86
88
  required: true,
87
89
  specRef: "basic/transports#streamable-http",
88
- description: "Checks that the server responds with Content-Type application/json or text/event-stream. MCP servers must use one of these two content types."
90
+ description: "Checks that the server responds with Content-Type application/json or text/event-stream. MCP servers must use one of these two content types.",
91
+ recommendation: 'Set the Content-Type response header to "application/json" for synchronous responses or "text/event-stream" for streaming. Do not use text/html or other types.'
89
92
  },
90
93
  {
91
94
  id: "transport-notification-202",
@@ -93,7 +96,8 @@ var TEST_DEFINITIONS = [
93
96
  category: "transport",
94
97
  required: false,
95
98
  specRef: "basic/transports#streamable-http",
96
- description: "Verifies that sending a JSON-RPC notification (no id field) returns HTTP 202 Accepted with no body. Per spec, servers MUST return 202 for notifications."
99
+ description: "Verifies that sending a JSON-RPC notification (no id field) returns HTTP 202 Accepted with no body. Per spec, servers MUST return 202 for notifications.",
100
+ recommendation: "Detect JSON-RPC messages without an id field and return HTTP 202 with an empty body. Do not attempt to send a JSON-RPC response for notifications."
97
101
  },
98
102
  {
99
103
  id: "transport-session-id",
@@ -101,7 +105,8 @@ var TEST_DEFINITIONS = [
101
105
  category: "transport",
102
106
  required: false,
103
107
  specRef: "basic/transports#streamable-http",
104
- description: "Tests that the server returns HTTP 400 when MCP-Session-Id header is missing on requests after initialization (when the server issued a session ID)."
108
+ description: "Tests that the server returns HTTP 400 when MCP-Session-Id header is missing on requests after initialization (when the server issued a session ID).",
109
+ recommendation: "If your server issues an MCP-Session-Id header in the initialize response, reject subsequent requests that omit this header with HTTP 400."
105
110
  },
106
111
  {
107
112
  id: "transport-get",
@@ -109,7 +114,8 @@ var TEST_DEFINITIONS = [
109
114
  category: "transport",
110
115
  required: false,
111
116
  specRef: "basic/transports#streamable-http",
112
- description: "Tests the GET endpoint for server-initiated messages. Server should return text/event-stream or 405 Method Not Allowed."
117
+ description: "Tests the GET endpoint for server-initiated messages. Server should return text/event-stream or 405 Method Not Allowed.",
118
+ recommendation: "If your server supports server-initiated messages, handle GET with text/event-stream. Otherwise, return 405 Method Not Allowed."
113
119
  },
114
120
  {
115
121
  id: "transport-delete",
@@ -117,7 +123,8 @@ var TEST_DEFINITIONS = [
117
123
  category: "transport",
118
124
  required: false,
119
125
  specRef: "basic/transports#streamable-http",
120
- description: "Tests the DELETE endpoint for session termination. Server should accept the request or return 405 Method Not Allowed."
126
+ description: "Tests the DELETE endpoint for session termination. Server should accept the request or return 405 Method Not Allowed.",
127
+ recommendation: "Handle DELETE requests for session cleanup, or return 405 if session termination is not supported. Do not return 500."
121
128
  },
122
129
  {
123
130
  id: "transport-batch-reject",
@@ -125,16 +132,45 @@ var TEST_DEFINITIONS = [
125
132
  category: "transport",
126
133
  required: true,
127
134
  specRef: "basic/transports#streamable-http",
128
- description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests."
135
+ description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests.",
136
+ recommendation: "Check if the parsed JSON body is an array. If so, return a JSON-RPC error or HTTP 400. Do not process batch requests \u2014 MCP explicitly forbids them."
137
+ },
138
+ {
139
+ id: "transport-content-type-init",
140
+ name: "Initialize response has valid content type",
141
+ category: "transport",
142
+ required: false,
143
+ specRef: "basic/transports#streamable-http",
144
+ description: "Validates that the initialize response uses application/json or text/event-stream content type. Some servers return other types for the handshake.",
145
+ recommendation: 'Ensure the initialize response uses Content-Type "application/json" or "text/event-stream". Do not return text/html or other types for JSON-RPC responses.'
129
146
  },
130
- // ── Lifecycle (10 tests) ─────────────────────────────────────────
147
+ {
148
+ id: "transport-get-stream",
149
+ name: "GET with session returns SSE or 405",
150
+ category: "transport",
151
+ required: false,
152
+ specRef: "basic/transports#streamable-http",
153
+ description: "Tests the GET endpoint with an active session ID for server-initiated messages. After initialization, the server should either return an SSE stream or 405.",
154
+ recommendation: "If your server supports server-initiated messages, return text/event-stream on GET with a valid session ID. Otherwise, return 405 Method Not Allowed."
155
+ },
156
+ {
157
+ id: "transport-concurrent",
158
+ name: "Handles concurrent requests",
159
+ category: "transport",
160
+ required: false,
161
+ specRef: "basic/transports#streamable-http",
162
+ description: "Sends multiple JSON-RPC requests in parallel and verifies the server responds to all with correct matching IDs. Tests that the server can handle concurrent connections.",
163
+ recommendation: "Ensure your server can handle multiple simultaneous requests. Each response must include the correct id matching the request. Use async handlers or connection pooling."
164
+ },
165
+ // ── Lifecycle (12 tests) ─────────────────────────────────────────
131
166
  {
132
167
  id: "lifecycle-init",
133
168
  name: "Initialize handshake",
134
169
  category: "lifecycle",
135
170
  required: true,
136
171
  specRef: "basic/lifecycle#initialization",
137
- description: "Tests the initialize handshake by sending an initialize request with client capabilities. The server must return a result with protocolVersion."
172
+ description: "Tests the initialize handshake by sending an initialize request with client capabilities. The server must return a result with protocolVersion.",
173
+ recommendation: 'Implement the "initialize" method handler. Return a result object with at least protocolVersion, capabilities, and serverInfo fields.'
138
174
  },
139
175
  {
140
176
  id: "lifecycle-proto-version",
@@ -142,7 +178,8 @@ var TEST_DEFINITIONS = [
142
178
  category: "lifecycle",
143
179
  required: true,
144
180
  specRef: "basic/lifecycle#version-negotiation",
145
- description: "Validates that the protocolVersion returned by the server matches the YYYY-MM-DD date format required by the spec."
181
+ description: "Validates that the protocolVersion returned by the server matches the YYYY-MM-DD date format required by the spec.",
182
+ recommendation: `Return protocolVersion as a YYYY-MM-DD string (e.g., "2025-11-25"). The server should negotiate based on the client's requested version.`
146
183
  },
147
184
  {
148
185
  id: "lifecycle-server-info",
@@ -150,7 +187,8 @@ var TEST_DEFINITIONS = [
150
187
  category: "lifecycle",
151
188
  required: false,
152
189
  specRef: "basic/lifecycle#initialization",
153
- description: "Checks that the server includes a serverInfo object with at least a name field in its initialize response. While recommended, this is not strictly required."
190
+ description: "Checks that the server includes a serverInfo object with at least a name field in its initialize response. While recommended, this is not strictly required.",
191
+ recommendation: 'Add a serverInfo object to your initialize response: { name: "your-server", version: "1.0.0" }. This helps clients identify your server.'
154
192
  },
155
193
  {
156
194
  id: "lifecycle-capabilities",
@@ -158,7 +196,8 @@ var TEST_DEFINITIONS = [
158
196
  category: "lifecycle",
159
197
  required: true,
160
198
  specRef: "basic/lifecycle#capability-negotiation",
161
- description: "Verifies the server returns a capabilities object in its initialize response. An empty object is valid (no optional features declared)."
199
+ description: "Verifies the server returns a capabilities object in its initialize response. An empty object is valid (no optional features declared).",
200
+ recommendation: "Include a capabilities object in your initialize response. Declare the features your server supports (tools, resources, prompts, logging, etc.). An empty object {} is valid."
162
201
  },
163
202
  {
164
203
  id: "lifecycle-jsonrpc",
@@ -166,7 +205,8 @@ var TEST_DEFINITIONS = [
166
205
  category: "lifecycle",
167
206
  required: true,
168
207
  specRef: "basic",
169
- description: 'Validates that the initialize response is a proper JSON-RPC 2.0 message with jsonrpc="2.0", an id field, and either a result or error field.'
208
+ description: 'Validates that the initialize response is a proper JSON-RPC 2.0 message with jsonrpc="2.0", an id field, and either a result or error field.',
209
+ recommendation: 'Ensure every response includes jsonrpc: "2.0", the matching id from the request, and either a result or error field. Never omit the jsonrpc field.'
170
210
  },
171
211
  {
172
212
  id: "lifecycle-ping",
@@ -174,7 +214,8 @@ var TEST_DEFINITIONS = [
174
214
  category: "lifecycle",
175
215
  required: true,
176
216
  specRef: "basic/utilities#ping",
177
- description: "Tests that the server responds to the ping method with an empty result object. This is a required utility method."
217
+ description: "Tests that the server responds to the ping method with an empty result object. This is a required utility method.",
218
+ recommendation: 'Implement a "ping" method handler that returns an empty result object {}. This is required by the MCP spec for keepalive and connectivity checking.'
178
219
  },
179
220
  {
180
221
  id: "lifecycle-instructions",
@@ -182,7 +223,8 @@ var TEST_DEFINITIONS = [
182
223
  category: "lifecycle",
183
224
  required: false,
184
225
  specRef: "basic/lifecycle#initialization",
185
- description: "If the server includes an instructions field in the initialize response, validates it is a string. Instructions provide guidance for how the client should interact with the server."
226
+ description: "If the server includes an instructions field in the initialize response, validates it is a string. Instructions provide guidance for how the client should interact with the server.",
227
+ recommendation: "If you include an instructions field in the initialize response, ensure it is a string. Remove the field or fix the type if it is not a string."
186
228
  },
187
229
  {
188
230
  id: "lifecycle-id-match",
@@ -190,7 +232,8 @@ var TEST_DEFINITIONS = [
190
232
  category: "lifecycle",
191
233
  required: true,
192
234
  specRef: "basic",
193
- description: "Verifies that the JSON-RPC response id matches the request id sent by the client. This is a fundamental JSON-RPC 2.0 requirement."
235
+ description: "Verifies that the JSON-RPC response id matches the request id sent by the client. This is a fundamental JSON-RPC 2.0 requirement.",
236
+ recommendation: "Copy the id field from the request into the response. This is a core JSON-RPC 2.0 requirement. Check that your framework does not modify or discard the request ID."
194
237
  },
195
238
  {
196
239
  id: "lifecycle-logging",
@@ -198,7 +241,8 @@ var TEST_DEFINITIONS = [
198
241
  category: "lifecycle",
199
242
  required: false,
200
243
  specRef: "server/utilities#logging",
201
- description: "If the server declares logging capability, tests that logging/setLevel method is accepted with a valid log level."
244
+ description: "If the server declares logging capability, tests that logging/setLevel method is accepted with a valid log level.",
245
+ recommendation: 'If you declare logging in capabilities, implement the "logging/setLevel" handler. Accept standard log levels: debug, info, notice, warning, error, critical, alert, emergency.'
202
246
  },
203
247
  {
204
248
  id: "lifecycle-completions",
@@ -206,7 +250,26 @@ var TEST_DEFINITIONS = [
206
250
  category: "lifecycle",
207
251
  required: false,
208
252
  specRef: "server/utilities#completion",
209
- description: "If the server declares completions capability, tests that the completion/complete method is accepted."
253
+ description: "If the server declares completions capability, tests that the completion/complete method is accepted.",
254
+ recommendation: 'If you declare completions in capabilities, implement the "completion/complete" handler. Return a completion object with a values array, even if empty.'
255
+ },
256
+ {
257
+ id: "lifecycle-cancellation",
258
+ name: "Handles cancellation notifications",
259
+ category: "lifecycle",
260
+ required: false,
261
+ specRef: "basic/utilities#cancellation",
262
+ description: "Tests that the server accepts notifications/cancelled without error. Servers should gracefully handle cancellation of unknown or completed requests.",
263
+ recommendation: "Accept notifications/cancelled and stop any in-progress work for the referenced requestId. If the request is unknown or already complete, silently ignore the cancellation."
264
+ },
265
+ {
266
+ id: "lifecycle-progress",
267
+ name: "Accepts progress notifications",
268
+ category: "lifecycle",
269
+ required: false,
270
+ specRef: "basic/utilities#progress",
271
+ description: "Tests that the server accepts notifications/progress without error. Servers should handle progress notifications for request tracking.",
272
+ recommendation: "Accept notifications/progress with progressToken, progress, and optional total fields. Ignore notifications for unknown progress tokens."
210
273
  },
211
274
  // ── Tools (4 tests) ──────────────────────────────────────────────
212
275
  {
@@ -215,7 +278,8 @@ var TEST_DEFINITIONS = [
215
278
  category: "tools",
216
279
  required: false,
217
280
  specRef: "server/tools#listing-tools",
218
- description: "Calls tools/list and validates it returns an array of tool definitions. Required if the server declares tools capability."
281
+ description: "Calls tools/list and validates it returns an array of tool definitions. Required if the server declares tools capability.",
282
+ recommendation: "Implement the tools/list handler to return { tools: [...] } with an array of tool definition objects. Each tool needs at least a name and inputSchema."
219
283
  },
220
284
  {
221
285
  id: "tools-call",
@@ -223,7 +287,8 @@ var TEST_DEFINITIONS = [
223
287
  category: "tools",
224
288
  required: false,
225
289
  specRef: "server/tools#calling-tools",
226
- description: "Calls the first tool with empty arguments and verifies the response format. Accepts both successful results and InvalidParams errors."
290
+ description: "Calls the first tool with empty arguments and verifies the response format. Accepts both successful results and InvalidParams errors.",
291
+ recommendation: "Ensure tools/call returns { content: [...] } with an array of content objects, each having a type field. Return isError: true for tool execution errors."
227
292
  },
228
293
  {
229
294
  id: "tools-pagination",
@@ -231,7 +296,8 @@ var TEST_DEFINITIONS = [
231
296
  category: "tools",
232
297
  required: false,
233
298
  specRef: "server/tools#listing-tools",
234
- description: "Tests cursor-based pagination on tools/list. Validates nextCursor is a string if present and that fetching the next page returns a valid response."
299
+ description: "Tests cursor-based pagination on tools/list. Validates nextCursor is a string if present and that fetching the next page returns a valid response.",
300
+ recommendation: "If your server has many tools, include a nextCursor string in the response. Ensure passing this cursor back in a subsequent request returns the next page."
235
301
  },
236
302
  {
237
303
  id: "tools-content-types",
@@ -239,7 +305,8 @@ var TEST_DEFINITIONS = [
239
305
  category: "tools",
240
306
  required: false,
241
307
  specRef: "server/tools#calling-tools",
242
- description: "Validates that content items returned by tools/call have a recognized type field (text, image, audio, resource, resource_link)."
308
+ description: "Validates that content items returned by tools/call have a recognized type field (text, image, audio, resource, resource_link).",
309
+ recommendation: 'Every content item returned by tools/call must have a type field set to one of: "text", "image", "audio", "resource", or "resource_link". Check for typos or missing type fields.'
243
310
  },
244
311
  // ── Resources (5 tests) ──────────────────────────────────────────
245
312
  {
@@ -248,7 +315,8 @@ var TEST_DEFINITIONS = [
248
315
  category: "resources",
249
316
  required: false,
250
317
  specRef: "server/resources#listing-resources",
251
- description: "Calls resources/list and validates it returns an array. Required if the server declares resources capability."
318
+ description: "Calls resources/list and validates it returns an array. Required if the server declares resources capability.",
319
+ recommendation: "Implement resources/list to return { resources: [...] } with an array of resource objects. Each resource needs at least a uri and name."
252
320
  },
253
321
  {
254
322
  id: "resources-read",
@@ -256,7 +324,8 @@ var TEST_DEFINITIONS = [
256
324
  category: "resources",
257
325
  required: false,
258
326
  specRef: "server/resources#reading-resources",
259
- description: "Reads the first resource and validates the response contains a contents array with proper uri and text/blob fields."
327
+ description: "Reads the first resource and validates the response contains a contents array with proper uri and text/blob fields.",
328
+ recommendation: "Implement resources/read to return { contents: [...] } where each item has a uri and either a text or blob field. Ensure the uri matches the requested resource."
260
329
  },
261
330
  {
262
331
  id: "resources-templates",
@@ -264,7 +333,8 @@ var TEST_DEFINITIONS = [
264
333
  category: "resources",
265
334
  required: false,
266
335
  specRef: "server/resources#resource-templates",
267
- description: "Tests the resource templates endpoint. Accepts Method not found (-32601) since templates are optional."
336
+ description: "Tests the resource templates endpoint. Accepts Method not found (-32601) since templates are optional.",
337
+ recommendation: "If your server supports resource templates, implement resources/templates/list returning { resourceTemplates: [...] }. Otherwise, return error code -32601."
268
338
  },
269
339
  {
270
340
  id: "resources-pagination",
@@ -272,7 +342,8 @@ var TEST_DEFINITIONS = [
272
342
  category: "resources",
273
343
  required: false,
274
344
  specRef: "server/resources#listing-resources",
275
- description: "Tests cursor-based pagination on resources/list. Validates nextCursor is a string if present and that fetching the next page works."
345
+ description: "Tests cursor-based pagination on resources/list. Validates nextCursor is a string if present and that fetching the next page works.",
346
+ recommendation: "If you return nextCursor in resources/list, ensure it is a string and that passing it back as cursor in the next request returns valid results."
276
347
  },
277
348
  {
278
349
  id: "resources-subscribe",
@@ -280,7 +351,8 @@ var TEST_DEFINITIONS = [
280
351
  category: "resources",
281
352
  required: false,
282
353
  specRef: "server/resources#subscriptions",
283
- description: "If the server declares resources.subscribe capability, tests that resources/subscribe and resources/unsubscribe methods are accepted."
354
+ description: "If the server declares resources.subscribe capability, tests that resources/subscribe and resources/unsubscribe methods are accepted.",
355
+ recommendation: "If you declare resources.subscribe capability, implement both resources/subscribe and resources/unsubscribe handlers. Both should accept a uri parameter."
284
356
  },
285
357
  // ── Prompts (3 tests) ────────────────────────────────────────────
286
358
  {
@@ -289,7 +361,8 @@ var TEST_DEFINITIONS = [
289
361
  category: "prompts",
290
362
  required: false,
291
363
  specRef: "server/prompts#listing-prompts",
292
- description: "Calls prompts/list and validates it returns an array. Required if the server declares prompts capability."
364
+ description: "Calls prompts/list and validates it returns an array. Required if the server declares prompts capability.",
365
+ recommendation: "Implement prompts/list to return { prompts: [...] } with an array of prompt objects. Each prompt needs at least a name field."
293
366
  },
294
367
  {
295
368
  id: "prompts-get",
@@ -297,7 +370,8 @@ var TEST_DEFINITIONS = [
297
370
  category: "prompts",
298
371
  required: false,
299
372
  specRef: "server/prompts#getting-a-prompt",
300
- description: "Gets the first prompt and validates the response contains a messages array with proper role and content fields."
373
+ description: "Gets the first prompt and validates the response contains a messages array with proper role and content fields.",
374
+ recommendation: 'Implement prompts/get to return { messages: [...] } where each message has a role ("user" or "assistant") and a content field.'
301
375
  },
302
376
  {
303
377
  id: "prompts-pagination",
@@ -305,7 +379,8 @@ var TEST_DEFINITIONS = [
305
379
  category: "prompts",
306
380
  required: false,
307
381
  specRef: "server/prompts#listing-prompts",
308
- description: "Tests cursor-based pagination on prompts/list. Validates nextCursor is a string if present and that fetching the next page works."
382
+ description: "Tests cursor-based pagination on prompts/list. Validates nextCursor is a string if present and that fetching the next page works.",
383
+ recommendation: "If you return nextCursor in prompts/list, ensure it is a string and that passing it back as cursor in the next request returns valid results."
309
384
  },
310
385
  // ── Error Handling (8 tests) ─────────────────────────────────────
311
386
  {
@@ -314,7 +389,8 @@ var TEST_DEFINITIONS = [
314
389
  category: "errors",
315
390
  required: true,
316
391
  specRef: "basic",
317
- description: "Sends an unknown method and verifies the server returns a JSON-RPC error. The spec requires error code -32601 (Method not found)."
392
+ description: "Sends an unknown method and verifies the server returns a JSON-RPC error. The spec requires error code -32601 (Method not found).",
393
+ recommendation: "Return a JSON-RPC error with code -32601 (Method not found) for any unrecognized method name. Do not silently ignore unknown methods."
318
394
  },
319
395
  {
320
396
  id: "error-method-code",
@@ -322,7 +398,8 @@ var TEST_DEFINITIONS = [
322
398
  category: "errors",
323
399
  required: false,
324
400
  specRef: "basic",
325
- description: "Checks the error code is specifically -32601 (Method not found) for unknown methods, as required by JSON-RPC 2.0."
401
+ description: "Checks the error code is specifically -32601 (Method not found) for unknown methods, as required by JSON-RPC 2.0.",
402
+ recommendation: "Use exactly error code -32601 for unknown methods. Do not use generic error codes like -32000. This is required by JSON-RPC 2.0."
326
403
  },
327
404
  {
328
405
  id: "error-invalid-jsonrpc",
@@ -330,7 +407,8 @@ var TEST_DEFINITIONS = [
330
407
  category: "errors",
331
408
  required: true,
332
409
  specRef: "basic",
333
- description: "Sends a malformed JSON-RPC message (missing required fields) and verifies the server returns an error or 4xx status."
410
+ description: "Sends a malformed JSON-RPC message (missing required fields) and verifies the server returns an error or 4xx status.",
411
+ recommendation: "Validate incoming JSON-RPC messages for required fields (jsonrpc, method). Return error code -32600 (Invalid Request) or HTTP 400 for malformed messages."
334
412
  },
335
413
  {
336
414
  id: "error-invalid-json",
@@ -338,7 +416,8 @@ var TEST_DEFINITIONS = [
338
416
  category: "errors",
339
417
  required: false,
340
418
  specRef: "basic",
341
- description: "Sends invalid JSON and verifies the server returns a parse error (-32700) or 4xx status code."
419
+ description: "Sends invalid JSON and verifies the server returns a parse error (-32700) or 4xx status code.",
420
+ recommendation: "Catch JSON parse errors and return error code -32700 (Parse error) with a descriptive message. Do not return 500 for malformed input."
342
421
  },
343
422
  {
344
423
  id: "error-missing-params",
@@ -346,7 +425,8 @@ var TEST_DEFINITIONS = [
346
425
  category: "errors",
347
426
  required: false,
348
427
  specRef: "server/tools#error-handling",
349
- description: "Calls tools/call with an empty params object (missing required name field) and verifies an error is returned."
428
+ description: "Calls tools/call with an empty params object (missing required name field) and verifies an error is returned.",
429
+ recommendation: "Validate tools/call params and return error code -32602 (Invalid params) when the required name field is missing."
350
430
  },
351
431
  {
352
432
  id: "error-parse-code",
@@ -354,7 +434,8 @@ var TEST_DEFINITIONS = [
354
434
  category: "errors",
355
435
  required: false,
356
436
  specRef: "basic",
357
- description: "Checks that the server returns the specific JSON-RPC error code -32700 (Parse error) when receiving invalid JSON, as required by the JSON-RPC 2.0 specification."
437
+ description: "Checks that the server returns the specific JSON-RPC error code -32700 (Parse error) when receiving invalid JSON, as required by the JSON-RPC 2.0 specification.",
438
+ recommendation: "Return exactly error code -32700 for JSON parse failures. Most JSON-RPC frameworks handle this automatically \u2014 check yours does not override the code."
358
439
  },
359
440
  {
360
441
  id: "error-invalid-request-code",
@@ -362,7 +443,8 @@ var TEST_DEFINITIONS = [
362
443
  category: "errors",
363
444
  required: false,
364
445
  specRef: "basic",
365
- description: "Checks that the server returns the specific JSON-RPC error code -32600 (Invalid Request) for malformed JSON-RPC messages missing required fields."
446
+ description: "Checks that the server returns the specific JSON-RPC error code -32600 (Invalid Request) for malformed JSON-RPC messages missing required fields.",
447
+ recommendation: "Return exactly error code -32600 for structurally invalid JSON-RPC messages (e.g., missing method field). Check your JSON-RPC middleware configuration."
366
448
  },
367
449
  {
368
450
  id: "tools-call-unknown",
@@ -370,7 +452,8 @@ var TEST_DEFINITIONS = [
370
452
  category: "errors",
371
453
  required: false,
372
454
  specRef: "server/tools#error-handling",
373
- description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response."
455
+ description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response.",
456
+ recommendation: "Return a JSON-RPC error or set isError: true when tools/call receives an unrecognized tool name. Do not return an empty success response."
374
457
  },
375
458
  // ── Schema Validation (6 tests) ──────────────────────────────────
376
459
  {
@@ -379,7 +462,8 @@ var TEST_DEFINITIONS = [
379
462
  category: "schema",
380
463
  required: false,
381
464
  specRef: "server/tools#data-types",
382
- description: 'Validates every tool has a valid name (1-128 chars, alphanumeric/underscore/hyphen/dot) and a required inputSchema of type "object".'
465
+ description: 'Validates every tool has a valid name (1-128 chars, alphanumeric/underscore/hyphen/dot) and a required inputSchema of type "object".',
466
+ recommendation: 'Ensure every tool has a name (1-128 chars, [A-Za-z0-9_.-]) and an inputSchema with type: "object". Add descriptions to tools for better AI assistant integration.'
383
467
  },
384
468
  {
385
469
  id: "tools-annotations",
@@ -387,7 +471,8 @@ var TEST_DEFINITIONS = [
387
471
  category: "schema",
388
472
  required: false,
389
473
  specRef: "server/tools#annotations",
390
- description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans; title should be a string."
474
+ description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans; title should be a string.",
475
+ recommendation: "If you include annotations on tools, ensure readOnlyHint, destructiveHint, idempotentHint, and openWorldHint are booleans. Title must be a string."
391
476
  },
392
477
  {
393
478
  id: "tools-title-field",
@@ -395,7 +480,8 @@ var TEST_DEFINITIONS = [
395
480
  category: "schema",
396
481
  required: false,
397
482
  specRef: "server/tools#data-types",
398
- description: "Checks if tools include the optional title field for human-readable display names. Added in spec version 2025-11-25."
483
+ description: "Checks if tools include the optional title field for human-readable display names. Added in spec version 2025-11-25.",
484
+ recommendation: "Add a title field (human-readable string) to each tool definition. This helps MCP clients display your tools in a user-friendly way."
399
485
  },
400
486
  {
401
487
  id: "tools-output-schema",
@@ -403,7 +489,8 @@ var TEST_DEFINITIONS = [
403
489
  category: "schema",
404
490
  required: false,
405
491
  specRef: "server/tools#structured-content",
406
- description: 'If tools declare an outputSchema, validates it is a valid JSON Schema object with type "object". Used for structured output validation.'
492
+ description: 'If tools declare an outputSchema, validates it is a valid JSON Schema object with type "object". Used for structured output validation.',
493
+ recommendation: 'If you declare outputSchema on a tool, ensure it is a valid JSON Schema object with type: "object". Remove outputSchema if you do not need structured output.'
407
494
  },
408
495
  {
409
496
  id: "prompts-schema",
@@ -411,7 +498,8 @@ var TEST_DEFINITIONS = [
411
498
  category: "schema",
412
499
  required: false,
413
500
  specRef: "server/prompts#data-types",
414
- description: "Validates every prompt has a name and that any arguments array contains items with name fields."
501
+ description: "Validates every prompt has a name and that any arguments array contains items with name fields.",
502
+ recommendation: "Ensure every prompt has a name field. If the prompt has arguments, each argument object must include a name field."
415
503
  },
416
504
  {
417
505
  id: "resources-schema",
@@ -419,7 +507,8 @@ var TEST_DEFINITIONS = [
419
507
  category: "schema",
420
508
  required: false,
421
509
  specRef: "server/resources#data-types",
422
- description: "Validates every resource has a valid URI (parseable as a URL) and a name field."
510
+ description: "Validates every resource has a valid URI (parseable as a URL) and a name field.",
511
+ recommendation: "Ensure every resource has a valid, parseable URI and a name field. Add description and mimeType for better client integration."
423
512
  }
424
513
  ];
425
514
 
@@ -523,19 +612,39 @@ async function mcpNotification(backendUrl, method, params, extraHeaders, timeout
523
612
  return { statusCode: res.statusCode, headers: responseHeaders };
524
613
  }
525
614
  async function runComplianceSuite(url, options = {}) {
526
- let parsed;
527
615
  try {
528
- parsed = new URL(url);
616
+ const parsed = new URL(url);
529
617
  if (!["http:", "https:"].includes(parsed.protocol)) {
530
618
  throw new Error("Only HTTP and HTTPS URLs are supported");
531
619
  }
532
620
  } catch (e) {
533
- if (e.message.includes("Only HTTP")) throw e;
621
+ if (e instanceof Error && e.message.includes("Only HTTP")) throw e;
534
622
  throw new Error(`Invalid URL: ${url}`);
535
623
  }
536
624
  const backendUrl = url;
625
+ let serverReachable = true;
626
+ try {
627
+ const preflight = await request(backendUrl, {
628
+ method: "POST",
629
+ headers: {
630
+ "Content-Type": "application/json",
631
+ Accept: "application/json, text/event-stream",
632
+ ...options.headers
633
+ },
634
+ body: JSON.stringify({ jsonrpc: "2.0", id: 0, method: "ping" }),
635
+ signal: AbortSignal.timeout(Math.min(options.timeout || 15e3, 1e4))
636
+ });
637
+ await preflight.body.text();
638
+ } catch {
639
+ serverReachable = false;
640
+ }
537
641
  const tests = [];
538
642
  const warnings = [];
643
+ if (!serverReachable) {
644
+ warnings.push(
645
+ `Server at ${url} is unreachable \u2014 all tests will fail. Check the URL and ensure the server is running.`
646
+ );
647
+ }
539
648
  const nextId = createIdCounter(1e3);
540
649
  const timeout = options.timeout || 15e3;
541
650
  const retries = options.retries || 0;
@@ -580,7 +689,8 @@ async function runComplianceSuite(url, options = {}) {
580
689
  if (lastResult.passed) break;
581
690
  if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
582
691
  } catch (err) {
583
- lastResult = { passed: false, details: `Error: ${err.message}` };
692
+ const message = err instanceof Error ? err.message : String(err);
693
+ lastResult = { passed: false, details: `Error: ${message}` };
584
694
  if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
585
695
  }
586
696
  }
@@ -609,10 +719,23 @@ async function runComplianceSuite(url, options = {}) {
609
719
  body: JSON.stringify({ jsonrpc: "2.0", id: 99901, method: "ping" }),
610
720
  signal: AbortSignal.timeout(timeout)
611
721
  });
612
- await res.body.text();
613
- const passed = res.statusCode >= 200 && res.statusCode < 300;
614
- const note = res.statusCode === 401 || res.statusCode === 403 ? " (auth required)" : "";
615
- return { passed, details: `HTTP ${res.statusCode}${note}` };
722
+ const text = await res.body.text();
723
+ if (res.statusCode >= 200 && res.statusCode < 300) {
724
+ return { passed: true, details: `HTTP ${res.statusCode}` };
725
+ }
726
+ if (res.statusCode === 401 || res.statusCode === 403) {
727
+ return { passed: false, details: `HTTP ${res.statusCode} (auth required \u2014 pass --auth)` };
728
+ }
729
+ if (res.statusCode === 400) {
730
+ try {
731
+ const body = JSON.parse(text);
732
+ if (body?.error || body?.jsonrpc) {
733
+ return { passed: true, details: "HTTP 400 with JSON-RPC response (server requires initialization first)" };
734
+ }
735
+ } catch {
736
+ }
737
+ }
738
+ return { passed: false, details: `HTTP ${res.statusCode}` };
616
739
  }
617
740
  );
618
741
  await test(
@@ -642,18 +765,29 @@ async function runComplianceSuite(url, options = {}) {
642
765
  false,
643
766
  "basic/transports#streamable-http",
644
767
  async () => {
768
+ const getHeaders = { Accept: "text/event-stream", ...buildHeaders() };
645
769
  const res = await request(backendUrl, {
646
770
  method: "GET",
647
- headers: { Accept: "text/event-stream", ...userHeaders },
771
+ headers: getHeaders,
648
772
  signal: AbortSignal.timeout(timeout)
649
773
  });
650
- await res.body.text();
774
+ const body = await res.body.text();
651
775
  const ct = (res.headers["content-type"] || "").toLowerCase();
652
776
  if (res.statusCode === 405) {
653
777
  return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
654
778
  }
655
779
  if (ct.includes("text/event-stream")) {
656
- return { passed: true, details: "Returns text/event-stream for SSE" };
780
+ if (body.trim().length > 0) {
781
+ const hasDataFields = body.includes("data:");
782
+ const hasEventFields = body.includes("event:");
783
+ if (!hasDataFields && !hasEventFields) {
784
+ return {
785
+ passed: false,
786
+ details: "Content-Type is text/event-stream but body has no SSE data: or event: fields"
787
+ };
788
+ }
789
+ }
790
+ return { passed: true, details: "Returns text/event-stream with valid SSE format" };
657
791
  }
658
792
  if (res.statusCode >= 200 && res.statusCode < 300) {
659
793
  return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
@@ -661,31 +795,6 @@ async function runComplianceSuite(url, options = {}) {
661
795
  return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
662
796
  }
663
797
  );
664
- await test(
665
- "transport-delete",
666
- "DELETE accepted or returns 405",
667
- "transport",
668
- false,
669
- "basic/transports#streamable-http",
670
- async () => {
671
- const res = await request(backendUrl, {
672
- method: "DELETE",
673
- headers: { ...userHeaders },
674
- signal: AbortSignal.timeout(timeout)
675
- });
676
- await res.body.text();
677
- if (res.statusCode === 405) {
678
- return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
679
- }
680
- if (res.statusCode >= 200 && res.statusCode < 300) {
681
- return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
682
- }
683
- if (res.statusCode === 400 || res.statusCode === 404) {
684
- return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
685
- }
686
- return { passed: false, details: `HTTP ${res.statusCode}` };
687
- }
688
- );
689
798
  await test(
690
799
  "transport-batch-reject",
691
800
  "Rejects JSON-RPC batch requests",
@@ -723,7 +832,7 @@ async function runComplianceSuite(url, options = {}) {
723
832
  try {
724
833
  initRes = await rpc("initialize", {
725
834
  protocolVersion: SPEC_VERSION,
726
- capabilities: { roots: { listChanged: true }, sampling: {} },
835
+ capabilities: {},
727
836
  clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
728
837
  });
729
838
  const result = initRes?.body?.result;
@@ -736,7 +845,7 @@ async function runComplianceSuite(url, options = {}) {
736
845
  if (sid) sessionId = sid;
737
846
  if (result.protocolVersion) negotiatedProtocolVersion = result.protocolVersion;
738
847
  }
739
- } catch (err) {
848
+ } catch {
740
849
  }
741
850
  try {
742
851
  await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders(), timeout);
@@ -762,13 +871,13 @@ async function runComplianceSuite(url, options = {}) {
762
871
  true,
763
872
  "basic/lifecycle#version-negotiation",
764
873
  async () => {
765
- const version2 = initRes?.body?.result?.protocolVersion;
766
- if (!version2) return { passed: false, details: "No protocolVersion" };
767
- const valid = /^\d{4}-\d{2}-\d{2}$/.test(version2);
768
- if (valid && version2 !== SPEC_VERSION) {
769
- warnings.push(`Server negotiated protocol version ${version2} (latest is ${SPEC_VERSION})`);
874
+ const version3 = initRes?.body?.result?.protocolVersion;
875
+ if (!version3) return { passed: false, details: "No protocolVersion" };
876
+ const valid = /^\d{4}-\d{2}-\d{2}$/.test(version3);
877
+ if (valid && version3 !== SPEC_VERSION) {
878
+ warnings.push(`Server negotiated protocol version ${version3} (latest is ${SPEC_VERSION})`);
770
879
  }
771
- return { passed: valid, details: `Version: ${version2}` };
880
+ return { passed: valid, details: `Version: ${version3}` };
772
881
  }
773
882
  );
774
883
  await test(
@@ -855,7 +964,17 @@ async function runComplianceSuite(url, options = {}) {
855
964
  if (res.body?.error) {
856
965
  return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
857
966
  }
858
- return { passed: true, details: "logging/setLevel accepted" };
967
+ const invalidRes = await rpc("logging/setLevel", { level: "__invalid_level__" });
968
+ const validatesInput = !!invalidRes.body?.error;
969
+ const validLevels = ["debug", "warning", "error"];
970
+ const accepted = [];
971
+ for (const level of validLevels) {
972
+ const r = await rpc("logging/setLevel", { level });
973
+ if (!r.body?.error) accepted.push(level);
974
+ }
975
+ const details = validatesInput ? `logging/setLevel accepted (validates levels, ${accepted.length + 1} levels accepted)` : "logging/setLevel accepted (warning: server does not reject invalid log levels)";
976
+ if (!validatesInput) warnings.push("Server accepts invalid log levels without error");
977
+ return { passed: true, details };
859
978
  }
860
979
  );
861
980
  const hasCompletions = !!serverInfo.capabilities.completions;
@@ -884,6 +1003,59 @@ async function runComplianceSuite(url, options = {}) {
884
1003
  return { passed: true, details: "completion/complete accepted" };
885
1004
  }
886
1005
  );
1006
+ await test(
1007
+ "lifecycle-cancellation",
1008
+ "Handles cancellation notifications",
1009
+ "lifecycle",
1010
+ false,
1011
+ "basic/utilities#cancellation",
1012
+ async () => {
1013
+ const res = await mcpNotification(
1014
+ backendUrl,
1015
+ "notifications/cancelled",
1016
+ { requestId: 99999, reason: "compliance test" },
1017
+ buildHeaders(),
1018
+ timeout
1019
+ );
1020
+ if (res.statusCode >= 200 && res.statusCode < 300) {
1021
+ return { passed: true, details: `HTTP ${res.statusCode} (cancellation accepted)` };
1022
+ }
1023
+ return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should accept cancellation notifications` };
1024
+ }
1025
+ );
1026
+ await test(
1027
+ "lifecycle-progress",
1028
+ "Accepts progress notifications",
1029
+ "lifecycle",
1030
+ false,
1031
+ "basic/utilities#progress",
1032
+ async () => {
1033
+ const res = await mcpNotification(
1034
+ backendUrl,
1035
+ "notifications/progress",
1036
+ { progressToken: "compliance-test-token", progress: 50, total: 100 },
1037
+ buildHeaders(),
1038
+ timeout
1039
+ );
1040
+ if (res.statusCode >= 200 && res.statusCode < 300) {
1041
+ return { passed: true, details: `HTTP ${res.statusCode} (progress notification accepted)` };
1042
+ }
1043
+ return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should accept progress notifications` };
1044
+ }
1045
+ );
1046
+ await test(
1047
+ "transport-content-type-init",
1048
+ "Initialize response has valid content type",
1049
+ "transport",
1050
+ false,
1051
+ "basic/transports#streamable-http",
1052
+ async () => {
1053
+ if (!initRes) return { passed: false, details: "No init response to check" };
1054
+ const ct = (initRes.headers["content-type"] || "").toLowerCase();
1055
+ const valid = ct.includes("application/json") || ct.includes("text/event-stream");
1056
+ return { passed: valid, details: `Content-Type: ${ct || "missing"}` };
1057
+ }
1058
+ );
887
1059
  await test(
888
1060
  "transport-notification-202",
889
1061
  "Notification returns 202 Accepted",
@@ -941,6 +1113,88 @@ async function runComplianceSuite(url, options = {}) {
941
1113
  return { passed: false, details: `HTTP ${res.statusCode}` };
942
1114
  }
943
1115
  );
1116
+ await test(
1117
+ "transport-get-stream",
1118
+ "GET with session returns SSE or 405",
1119
+ "transport",
1120
+ false,
1121
+ "basic/transports#streamable-http",
1122
+ async () => {
1123
+ if (!sessionId) {
1124
+ return { passed: true, details: "No session ID \u2014 server-initiated messages not applicable" };
1125
+ }
1126
+ const res = await request(backendUrl, {
1127
+ method: "GET",
1128
+ headers: { Accept: "text/event-stream", ...buildHeaders() },
1129
+ signal: AbortSignal.timeout(Math.min(timeout, 3e3))
1130
+ });
1131
+ const body = await res.body.text();
1132
+ const ct = (res.headers["content-type"] || "").toLowerCase();
1133
+ if (res.statusCode === 405) {
1134
+ return { passed: true, details: "HTTP 405 (server does not support server-initiated messages)" };
1135
+ }
1136
+ if (ct.includes("text/event-stream")) {
1137
+ if (body.trim().length > 0) {
1138
+ const hasSSEFields = body.includes("data:") || body.includes("event:");
1139
+ if (!hasSSEFields) {
1140
+ return { passed: false, details: "Content-Type is text/event-stream but body has no SSE fields" };
1141
+ }
1142
+ }
1143
+ return { passed: true, details: "GET with session returns SSE stream for server-initiated messages" };
1144
+ }
1145
+ if (res.statusCode >= 200 && res.statusCode < 300) {
1146
+ return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
1147
+ }
1148
+ return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
1149
+ }
1150
+ );
1151
+ await test(
1152
+ "transport-concurrent",
1153
+ "Handles concurrent requests",
1154
+ "transport",
1155
+ false,
1156
+ "basic/transports#streamable-http",
1157
+ async () => {
1158
+ const ids = [createIdCounter(99930)(), createIdCounter(99931)(), createIdCounter(99932)()];
1159
+ const promises = ids.map(
1160
+ (id) => request(backendUrl, {
1161
+ method: "POST",
1162
+ headers: {
1163
+ "Content-Type": "application/json",
1164
+ Accept: "application/json, text/event-stream",
1165
+ ...buildHeaders()
1166
+ },
1167
+ body: JSON.stringify({ jsonrpc: "2.0", id, method: "ping" }),
1168
+ signal: AbortSignal.timeout(timeout)
1169
+ }).then(async (res) => {
1170
+ const text = await res.body.text();
1171
+ const ct = (res.headers["content-type"] || "").toLowerCase();
1172
+ let body;
1173
+ if (ct.includes("text/event-stream")) {
1174
+ body = parseSSEResponse(text);
1175
+ }
1176
+ if (!body) {
1177
+ try {
1178
+ body = JSON.parse(text);
1179
+ } catch {
1180
+ }
1181
+ }
1182
+ return { statusCode: res.statusCode, body, requestId: id };
1183
+ })
1184
+ );
1185
+ const results = await Promise.all(promises);
1186
+ const issues = [];
1187
+ for (const r of results) {
1188
+ if (r.statusCode < 200 || r.statusCode >= 300) {
1189
+ issues.push(`Request id=${r.requestId}: HTTP ${r.statusCode}`);
1190
+ } else if (r.body?.id !== r.requestId) {
1191
+ issues.push(`Request id=${r.requestId}: response id=${r.body?.id} (mismatch)`);
1192
+ }
1193
+ }
1194
+ if (issues.length > 0) return { passed: false, details: issues.join("; ") };
1195
+ return { passed: true, details: `${results.length} concurrent requests handled correctly` };
1196
+ }
1197
+ );
944
1198
  const hasTools = !!serverInfo.capabilities.tools;
945
1199
  let cachedToolsList = null;
946
1200
  await test(
@@ -962,6 +1216,7 @@ async function runComplianceSuite(url, options = {}) {
962
1216
  };
963
1217
  }
964
1218
  );
1219
+ const toolsListOk = cachedToolsList !== null;
965
1220
  await test(
966
1221
  "tools-schema",
967
1222
  "All tools have name and inputSchema",
@@ -969,7 +1224,8 @@ async function runComplianceSuite(url, options = {}) {
969
1224
  hasTools,
970
1225
  "server/tools#data-types",
971
1226
  async () => {
972
- const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
1227
+ if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
1228
+ const tools = cachedToolsList ?? [];
973
1229
  const issues = [];
974
1230
  for (const tool of tools) {
975
1231
  if (!tool.name) {
@@ -1001,7 +1257,8 @@ async function runComplianceSuite(url, options = {}) {
1001
1257
  false,
1002
1258
  "server/tools#annotations",
1003
1259
  async () => {
1004
- const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
1260
+ if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
1261
+ const tools = cachedToolsList ?? [];
1005
1262
  if (tools.length === 0) return { passed: true, details: "No tools to validate" };
1006
1263
  const issues = [];
1007
1264
  let annotatedCount = 0;
@@ -1031,7 +1288,8 @@ async function runComplianceSuite(url, options = {}) {
1031
1288
  }
1032
1289
  );
1033
1290
  await test("tools-title-field", "Tools include title field", "schema", false, "server/tools#data-types", async () => {
1034
- const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
1291
+ if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
1292
+ const tools = cachedToolsList ?? [];
1035
1293
  if (tools.length === 0) return { passed: true, details: "No tools to validate" };
1036
1294
  const withTitle = tools.filter((t) => typeof t.title === "string");
1037
1295
  const issues = [];
@@ -1053,7 +1311,8 @@ async function runComplianceSuite(url, options = {}) {
1053
1311
  false,
1054
1312
  "server/tools#structured-content",
1055
1313
  async () => {
1056
- const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
1314
+ if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
1315
+ const tools = cachedToolsList ?? [];
1057
1316
  if (tools.length === 0) return { passed: true, details: "No tools to validate" };
1058
1317
  const issues = [];
1059
1318
  let withSchema = 0;
@@ -1094,14 +1353,14 @@ async function runComplianceSuite(url, options = {}) {
1094
1353
  return { passed: true, details: `Protocol error: code ${code} \u2014 ${error.message}` };
1095
1354
  }
1096
1355
  if (result?.content && Array.isArray(result.content)) {
1356
+ if (result.isError) {
1357
+ return { passed: true, details: "Tool returned execution error with content (valid)" };
1358
+ }
1097
1359
  const badItems = result.content.filter((c) => !c.type);
1098
1360
  if (badItems.length > 0)
1099
1361
  return { passed: false, details: `${badItems.length} content item(s) missing 'type' field` };
1100
1362
  return { passed: true, details: `Returned ${result.content.length} content item(s)` };
1101
1363
  }
1102
- if (result?.isError && result?.content && Array.isArray(result.content)) {
1103
- return { passed: true, details: "Tool returned execution error with content (valid)" };
1104
- }
1105
1364
  return { passed: false, details: "Response missing content array" };
1106
1365
  }
1107
1366
  );
@@ -1203,6 +1462,7 @@ async function runComplianceSuite(url, options = {}) {
1203
1462
  return { passed: true, details: `${resourceCount} resource(s)` };
1204
1463
  }
1205
1464
  );
1465
+ const resourcesListOk = cachedResourcesList !== null;
1206
1466
  await test(
1207
1467
  "resources-schema",
1208
1468
  "Resources have uri and name",
@@ -1210,7 +1470,8 @@ async function runComplianceSuite(url, options = {}) {
1210
1470
  true,
1211
1471
  "server/resources#data-types",
1212
1472
  async () => {
1213
- const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
1473
+ if (!resourcesListOk) return { passed: false, details: "Skipped: resources/list failed" };
1474
+ const resources = cachedResourcesList ?? [];
1214
1475
  const issues = [];
1215
1476
  for (const r of resources) {
1216
1477
  if (!r.uri) issues.push("Resource missing uri");
@@ -1239,7 +1500,7 @@ async function runComplianceSuite(url, options = {}) {
1239
1500
  false,
1240
1501
  "server/resources#reading-resources",
1241
1502
  async () => {
1242
- const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
1503
+ const resources = cachedResourcesList ?? [];
1243
1504
  const firstUri = resources[0]?.uri;
1244
1505
  if (!firstUri) return { passed: false, details: "No resource URI to test" };
1245
1506
  const readRes = await rpc("resources/read", { uri: firstUri });
@@ -1272,8 +1533,15 @@ async function runComplianceSuite(url, options = {}) {
1272
1533
  if (!Array.isArray(templates)) return { passed: false, details: "No resourceTemplates array" };
1273
1534
  const issues = [];
1274
1535
  for (const t of templates) {
1275
- if (!t.uriTemplate) issues.push("Template missing uriTemplate");
1536
+ if (!t.uriTemplate) {
1537
+ issues.push("Template missing uriTemplate");
1538
+ } else if (typeof t.uriTemplate !== "string") {
1539
+ issues.push(`uriTemplate should be a string, got ${typeof t.uriTemplate}`);
1540
+ } else if (!t.uriTemplate.includes("{") || !t.uriTemplate.includes("}")) {
1541
+ warnings.push(`Template "${t.name || t.uriTemplate}" has no URI template parameters (e.g., {id})`);
1542
+ }
1276
1543
  if (!t.name) issues.push(`${t.uriTemplate || "?"}: missing name`);
1544
+ if (!t.description) warnings.push(`Template "${t.name || t.uriTemplate || "?"}" missing description`);
1277
1545
  }
1278
1546
  if (issues.length > 0) return { passed: false, details: issues.join("; ") };
1279
1547
  return { passed: true, details: `${templates.length} resource template(s)` };
@@ -1315,7 +1583,7 @@ async function runComplianceSuite(url, options = {}) {
1315
1583
  true,
1316
1584
  "server/resources#subscriptions",
1317
1585
  async () => {
1318
- const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
1586
+ const resources = cachedResourcesList ?? [];
1319
1587
  const firstUri = resources[0]?.uri;
1320
1588
  if (!firstUri) return { passed: false, details: "No resource URI for subscribe test" };
1321
1589
  const subRes = await rpc("resources/subscribe", { uri: firstUri });
@@ -1359,8 +1627,10 @@ async function runComplianceSuite(url, options = {}) {
1359
1627
  };
1360
1628
  }
1361
1629
  );
1630
+ const promptsListOk = cachedPromptsList !== null;
1362
1631
  await test("prompts-schema", "Prompts have name field", "schema", true, "server/prompts#data-types", async () => {
1363
- const prompts = cachedPromptsList ?? (await rpc("prompts/list")).body?.result?.prompts ?? [];
1632
+ if (!promptsListOk) return { passed: false, details: "Skipped: prompts/list failed" };
1633
+ const prompts = cachedPromptsList ?? [];
1364
1634
  const issues = [];
1365
1635
  for (const p of prompts) {
1366
1636
  if (!p.name) issues.push("Prompt missing name");
@@ -1556,6 +1826,60 @@ async function runComplianceSuite(url, options = {}) {
1556
1826
  }
1557
1827
  return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error code -32600` };
1558
1828
  });
1829
+ await test(
1830
+ "transport-delete",
1831
+ "DELETE accepted or returns 405",
1832
+ "transport",
1833
+ false,
1834
+ "basic/transports#streamable-http",
1835
+ async () => {
1836
+ const deleteHeaders = { ...buildHeaders() };
1837
+ const res = await request(backendUrl, {
1838
+ method: "DELETE",
1839
+ headers: deleteHeaders,
1840
+ signal: AbortSignal.timeout(timeout)
1841
+ });
1842
+ await res.body.text();
1843
+ if (res.statusCode === 405) {
1844
+ return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
1845
+ }
1846
+ if (res.statusCode >= 200 && res.statusCode < 300) {
1847
+ if (sessionId) {
1848
+ try {
1849
+ const verifyRes = await mcpRequest(
1850
+ backendUrl,
1851
+ "ping",
1852
+ void 0,
1853
+ createIdCounter(99920),
1854
+ deleteHeaders,
1855
+ timeout
1856
+ );
1857
+ if (verifyRes.statusCode === 400 || verifyRes.statusCode === 404 || verifyRes.statusCode === 409) {
1858
+ return {
1859
+ passed: true,
1860
+ details: `HTTP ${res.statusCode} (session terminated, post-delete request correctly rejected with ${verifyRes.statusCode})`
1861
+ };
1862
+ }
1863
+ } catch {
1864
+ return {
1865
+ passed: true,
1866
+ details: `HTTP ${res.statusCode} (session terminated, post-delete request rejected)`
1867
+ };
1868
+ }
1869
+ }
1870
+ return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
1871
+ }
1872
+ if (res.statusCode === 400 || res.statusCode === 404) {
1873
+ return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
1874
+ }
1875
+ return { passed: false, details: `HTTP ${res.statusCode}` };
1876
+ }
1877
+ );
1878
+ const MAX_WARNINGS = 50;
1879
+ if (warnings.length > MAX_WARNINGS) {
1880
+ const truncated = warnings.length - MAX_WARNINGS;
1881
+ warnings.splice(MAX_WARNINGS, truncated, `... and ${truncated} more warning(s) suppressed`);
1882
+ }
1559
1883
  const { score, grade, overall, summary, categories } = computeScore(tests);
1560
1884
  const badge = generateBadge(url);
1561
1885
  return {
@@ -1585,7 +1909,7 @@ async function runComplianceSuite(url, options = {}) {
1585
1909
  function registerTools(server) {
1586
1910
  server.tool(
1587
1911
  "mcp_compliance_test",
1588
- "Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all 43 tests covering transport, lifecycle, tools, resources, prompts, errors, and schema validation.",
1912
+ "Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all 48 tests covering transport, lifecycle, tools, resources, prompts, errors, and schema validation.",
1589
1913
  {
1590
1914
  url: z.string().url().describe("The MCP server URL to test (must be HTTP or HTTPS)"),
1591
1915
  auth: z.string().optional().describe('Authorization header value (e.g., "Bearer tok123")'),
@@ -1641,8 +1965,9 @@ ${JSON.stringify(report, null, 2)}` }
1641
1965
  ]
1642
1966
  };
1643
1967
  } catch (err) {
1968
+ const message = err instanceof Error ? err.message : String(err);
1644
1969
  return {
1645
- content: [{ type: "text", text: `Error running compliance test: ${err.message}` }],
1970
+ content: [{ type: "text", text: `Error running compliance test: ${message}` }],
1646
1971
  isError: true
1647
1972
  };
1648
1973
  }
@@ -1690,8 +2015,9 @@ ${JSON.stringify(report, null, 2)}` }
1690
2015
  ]
1691
2016
  };
1692
2017
  } catch (err) {
2018
+ const message = err instanceof Error ? err.message : String(err);
1693
2019
  return {
1694
- content: [{ type: "text", text: `Error: ${err.message}` }],
2020
+ content: [{ type: "text", text: `Error: ${message}` }],
1695
2021
  isError: true
1696
2022
  };
1697
2023
  }
@@ -1735,9 +2061,11 @@ ${TEST_DEFINITIONS.map((t) => t.id).join(", ")}`
1735
2061
  `Name: ${def.name}`,
1736
2062
  `Category: ${def.category}`,
1737
2063
  `Required: ${def.required ? "Yes" : "No"}`,
1738
- `Spec reference: https://modelcontextprotocol.io/specification/2025-11-25/${def.specRef}`,
2064
+ `Spec reference: ${SPEC_BASE}/${def.specRef}`,
2065
+ "",
2066
+ def.description,
1739
2067
  "",
1740
- def.description
2068
+ `Fix: ${def.recommendation}`
1741
2069
  ].join("\n")
1742
2070
  }
1743
2071
  ]
@@ -1746,6 +2074,27 @@ ${TEST_DEFINITIONS.map((t) => t.id).join(", ")}`
1746
2074
  );
1747
2075
  }
1748
2076
 
2077
+ // src/mcp/server.ts
2078
+ var require2 = createRequire2(import.meta.url);
2079
+ var { version } = require2("../../package.json");
2080
+ function createComplianceServer() {
2081
+ const server = new McpServer({ name: "mcp-compliance", version });
2082
+ registerTools(server);
2083
+ return server;
2084
+ }
2085
+ async function startServer() {
2086
+ const server = createComplianceServer();
2087
+ const transport = new StdioServerTransport();
2088
+ await server.connect(transport);
2089
+ }
2090
+ var isDirectRun = process.argv[1]?.endsWith("mcp/server.js") || process.argv[1]?.endsWith("mcp\\server.js");
2091
+ if (isDirectRun) {
2092
+ startServer().catch((err) => {
2093
+ console.error("MCP server error:", err);
2094
+ process.exit(1);
2095
+ });
2096
+ }
2097
+
1749
2098
  // src/reporter.ts
1750
2099
  import chalk from "chalk";
1751
2100
  var CATEGORY_LABELS = {
@@ -1788,8 +2137,16 @@ function testLine(t) {
1788
2137
  const icon = t.passed ? chalk.green(" PASS") : chalk.red(" FAIL");
1789
2138
  const req = t.required ? chalk.dim(" (required)") : "";
1790
2139
  const dur = chalk.dim(` ${t.durationMs}ms`);
1791
- return `${icon} ${t.name}${req}${dur}
2140
+ let line = `${icon} ${t.name}${req}${dur}
1792
2141
  ${chalk.dim(` ${t.details}`)}`;
2142
+ if (!t.passed) {
2143
+ const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
2144
+ if (def?.recommendation) {
2145
+ line += `
2146
+ ${chalk.cyan(` Fix: ${def.recommendation}`)}`;
2147
+ }
2148
+ }
2149
+ return line;
1793
2150
  }
1794
2151
  function formatTerminal(report) {
1795
2152
  const lines = [];
@@ -1872,10 +2229,85 @@ function formatTerminal(report) {
1872
2229
  function formatJson(report) {
1873
2230
  return JSON.stringify(report, null, 2);
1874
2231
  }
2232
+ function formatSarif(report) {
2233
+ const rules = report.tests.map((t) => {
2234
+ const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
2235
+ return {
2236
+ id: t.id,
2237
+ name: t.name,
2238
+ shortDescription: { text: t.name },
2239
+ fullDescription: { text: def?.description || t.details },
2240
+ helpUri: t.specRef || `${SPEC_BASE}/basic`,
2241
+ properties: {
2242
+ category: t.category,
2243
+ required: t.required
2244
+ }
2245
+ };
2246
+ });
2247
+ const results = report.tests.filter((t) => !t.passed).map((t) => {
2248
+ const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
2249
+ return {
2250
+ ruleId: t.id,
2251
+ level: t.required ? "error" : "warning",
2252
+ message: {
2253
+ text: def?.recommendation ? `${t.details}. Fix: ${def.recommendation}` : t.details
2254
+ },
2255
+ locations: [
2256
+ {
2257
+ physicalLocation: {
2258
+ artifactLocation: {
2259
+ uri: report.url,
2260
+ uriBaseId: "MCP_SERVER"
2261
+ }
2262
+ }
2263
+ }
2264
+ ],
2265
+ properties: {
2266
+ category: t.category,
2267
+ durationMs: t.durationMs
2268
+ }
2269
+ };
2270
+ });
2271
+ const sarif = {
2272
+ $schema: "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json",
2273
+ version: "2.1.0",
2274
+ runs: [
2275
+ {
2276
+ tool: {
2277
+ driver: {
2278
+ name: "mcp-compliance",
2279
+ version: report.toolVersion,
2280
+ informationUri: "https://github.com/YawLabs/mcp-compliance",
2281
+ rules
2282
+ }
2283
+ },
2284
+ results,
2285
+ invocations: [
2286
+ {
2287
+ executionSuccessful: report.overall !== "fail",
2288
+ properties: {
2289
+ grade: report.grade,
2290
+ score: report.score,
2291
+ overall: report.overall,
2292
+ specVersion: report.specVersion,
2293
+ serverUrl: report.url,
2294
+ serverName: report.serverInfo.name,
2295
+ serverVersion: report.serverInfo.version,
2296
+ protocolVersion: report.serverInfo.protocolVersion,
2297
+ testsPassed: report.summary.passed,
2298
+ testsTotal: report.summary.total
2299
+ }
2300
+ }
2301
+ ]
2302
+ }
2303
+ ]
2304
+ };
2305
+ return JSON.stringify(sarif, null, 2);
2306
+ }
1875
2307
 
1876
2308
  // src/index.ts
1877
- var require2 = createRequire2(import.meta.url);
1878
- var { version } = require2("../package.json");
2309
+ var require3 = createRequire3(import.meta.url);
2310
+ var { version: version2 } = require3("../package.json");
1879
2311
  function parseHeaderArg(value, prev) {
1880
2312
  const idx = value.indexOf(":");
1881
2313
  if (idx === -1) {
@@ -1886,12 +2318,27 @@ function parseHeaderArg(value, prev) {
1886
2318
  prev[key] = val;
1887
2319
  return prev;
1888
2320
  }
2321
+ function parsePositiveInt(value, name, min = 0) {
2322
+ const n = Number.parseInt(value, 10);
2323
+ if (Number.isNaN(n) || n < min) {
2324
+ throw new Error(`${name} must be an integer >= ${min}, got "${value}"`);
2325
+ }
2326
+ return n;
2327
+ }
1889
2328
  function parseList(value) {
1890
2329
  return value.split(",").map((s) => s.trim()).filter(Boolean);
1891
2330
  }
1892
2331
  var program = new Command();
1893
- program.name("mcp-compliance").description("Test MCP servers for spec compliance").version(version);
1894
- program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal or json", "terminal").option("--strict", "Exit with code 1 on any required test failure (for CI)").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").option("--retries <n>", "Number of retries for failed tests", "0").option("--only <items>", "Only run tests matching these categories or test IDs (comma-separated)", parseList).option("--skip <items>", "Skip tests matching these categories or test IDs (comma-separated)", parseList).option("--verbose", "Print each test result as it runs").action(
2332
+ program.name("mcp-compliance").description("Test MCP servers for spec compliance").version(version2);
2333
+ program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal, json, or sarif", "terminal").option("--strict", "Exit with code 1 on any required test failure (for CI)").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").option("--retries <n>", "Number of retries for failed tests", "0").option(
2334
+ "--only <items>",
2335
+ 'Only run matching categories or test IDs, comma-separated (e.g., "transport,lifecycle" or "transport-post,lifecycle-init")',
2336
+ parseList
2337
+ ).option(
2338
+ "--skip <items>",
2339
+ 'Skip matching categories or test IDs, comma-separated (e.g., "schema" or "tools-pagination")',
2340
+ parseList
2341
+ ).option("--verbose", "Print each test result as it runs").action(
1895
2342
  async (url, opts) => {
1896
2343
  try {
1897
2344
  const headers = { ...opts.header };
@@ -1903,8 +2350,8 @@ Testing ${url}...
1903
2350
  }
1904
2351
  const report = await runComplianceSuite(url, {
1905
2352
  headers,
1906
- timeout: Number.parseInt(opts.timeout, 10) || 15e3,
1907
- retries: Number.parseInt(opts.retries, 10) || 0,
2353
+ timeout: parsePositiveInt(opts.timeout, "--timeout", 1),
2354
+ retries: parsePositiveInt(opts.retries, "--retries"),
1908
2355
  only: opts.only,
1909
2356
  skip: opts.skip,
1910
2357
  onProgress: opts.verbose ? (testId, passed, details) => {
@@ -1917,6 +2364,8 @@ Testing ${url}...
1917
2364
  }
1918
2365
  if (opts.format === "json") {
1919
2366
  console.log(formatJson(report));
2367
+ } else if (opts.format === "sarif") {
2368
+ console.log(formatSarif(report));
1920
2369
  } else {
1921
2370
  console.log(formatTerminal(report));
1922
2371
  }
@@ -1924,11 +2373,12 @@ Testing ${url}...
1924
2373
  process.exit(1);
1925
2374
  }
1926
2375
  } catch (err) {
1927
- if (opts.format === "json") {
1928
- console.error(JSON.stringify({ error: err.message }));
2376
+ const message = err instanceof Error ? err.message : String(err);
2377
+ if (opts.format === "json" || opts.format === "sarif") {
2378
+ console.error(JSON.stringify({ error: message }));
1929
2379
  } else {
1930
2380
  console.error(chalk2.red(`
1931
- Error: ${err.message}
2381
+ Error: ${message}
1932
2382
  `));
1933
2383
  }
1934
2384
  process.exit(1);
@@ -1944,23 +2394,21 @@ Testing ${url}...
1944
2394
  `));
1945
2395
  const report = await runComplianceSuite(url, {
1946
2396
  headers,
1947
- timeout: Number.parseInt(opts.timeout, 10) || 15e3
2397
+ timeout: parsePositiveInt(opts.timeout, "--timeout", 1)
1948
2398
  });
1949
2399
  console.log(`Grade: ${report.grade} (${report.score}%)
1950
2400
  `);
1951
2401
  console.log(report.badge.markdown);
1952
2402
  console.log("");
1953
2403
  } catch (err) {
2404
+ const message = err instanceof Error ? err.message : String(err);
1954
2405
  console.error(chalk2.red(`
1955
- Error: ${err.message}
2406
+ Error: ${message}
1956
2407
  `));
1957
2408
  process.exit(1);
1958
2409
  }
1959
2410
  });
1960
2411
  program.command("mcp").description("Start the MCP compliance server (stdio transport)").action(async () => {
1961
- const server = new McpServer({ name: "mcp-compliance", version });
1962
- registerTools(server);
1963
- const transport = new StdioServerTransport();
1964
- await server.connect(transport);
2412
+ await startServer();
1965
2413
  });
1966
2414
  program.parse();