@yawlabs/mcp-compliance 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +78 -13
- package/dist/{chunk-U66YZGE5.js → chunk-Z7VLPYIO.js} +429 -105
- package/dist/index.js +582 -134
- package/dist/mcp/server.d.ts +11 -1
- package/dist/mcp/server.js +33 -17
- package/dist/runner.d.ts +13 -2
- package/dist/runner.js +7 -1
- package/package.json +13 -5
|
@@ -3,16 +3,14 @@ import { createRequire } from "module";
|
|
|
3
3
|
import { request } from "undici";
|
|
4
4
|
|
|
5
5
|
// src/badge.ts
|
|
6
|
+
import { createHash } from "crypto";
|
|
7
|
+
function urlHash(url) {
|
|
8
|
+
return createHash("sha256").update(url).digest("hex").slice(0, 12);
|
|
9
|
+
}
|
|
6
10
|
function generateBadge(url) {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
} catch {
|
|
11
|
-
parsed = new URL("https://unknown");
|
|
12
|
-
}
|
|
13
|
-
const encoded = encodeURIComponent(parsed.href);
|
|
14
|
-
const imageUrl = `https://mcp.hosting/api/compliance/${encoded}/badge`;
|
|
15
|
-
const reportUrl = `https://mcp.hosting/compliance/${encoded}`;
|
|
11
|
+
const hash = urlHash(url);
|
|
12
|
+
const imageUrl = `https://mcp.hosting/api/compliance/${hash}/badge`;
|
|
13
|
+
const reportUrl = `https://mcp.hosting/compliance/${hash}`;
|
|
16
14
|
return {
|
|
17
15
|
imageUrl,
|
|
18
16
|
reportUrl,
|
|
@@ -58,14 +56,15 @@ function computeScore(tests) {
|
|
|
58
56
|
|
|
59
57
|
// src/types.ts
|
|
60
58
|
var TEST_DEFINITIONS = [
|
|
61
|
-
// ── Transport (
|
|
59
|
+
// ── Transport (10 tests) ─────────────────────────────────────────
|
|
62
60
|
{
|
|
63
61
|
id: "transport-post",
|
|
64
62
|
name: "HTTP POST accepted",
|
|
65
63
|
category: "transport",
|
|
66
64
|
required: true,
|
|
67
65
|
specRef: "basic/transports#streamable-http",
|
|
68
|
-
description: "Verifies the server accepts HTTP POST requests and returns a 2xx status code. This is the fundamental transport requirement for Streamable HTTP MCP servers."
|
|
66
|
+
description: "Verifies the server accepts HTTP POST requests and returns a 2xx status code. This is the fundamental transport requirement for Streamable HTTP MCP servers.",
|
|
67
|
+
recommendation: "Ensure your server listens for POST requests on the MCP endpoint. If you see 401/403, pass --auth with a valid token. Check that the URL is correct and the server is running."
|
|
69
68
|
},
|
|
70
69
|
{
|
|
71
70
|
id: "transport-content-type",
|
|
@@ -73,7 +72,8 @@ var TEST_DEFINITIONS = [
|
|
|
73
72
|
category: "transport",
|
|
74
73
|
required: true,
|
|
75
74
|
specRef: "basic/transports#streamable-http",
|
|
76
|
-
description: "Checks that the server responds with Content-Type application/json or text/event-stream. MCP servers must use one of these two content types."
|
|
75
|
+
description: "Checks that the server responds with Content-Type application/json or text/event-stream. MCP servers must use one of these two content types.",
|
|
76
|
+
recommendation: 'Set the Content-Type response header to "application/json" for synchronous responses or "text/event-stream" for streaming. Do not use text/html or other types.'
|
|
77
77
|
},
|
|
78
78
|
{
|
|
79
79
|
id: "transport-notification-202",
|
|
@@ -81,7 +81,8 @@ var TEST_DEFINITIONS = [
|
|
|
81
81
|
category: "transport",
|
|
82
82
|
required: false,
|
|
83
83
|
specRef: "basic/transports#streamable-http",
|
|
84
|
-
description: "Verifies that sending a JSON-RPC notification (no id field) returns HTTP 202 Accepted with no body. Per spec, servers MUST return 202 for notifications."
|
|
84
|
+
description: "Verifies that sending a JSON-RPC notification (no id field) returns HTTP 202 Accepted with no body. Per spec, servers MUST return 202 for notifications.",
|
|
85
|
+
recommendation: "Detect JSON-RPC messages without an id field and return HTTP 202 with an empty body. Do not attempt to send a JSON-RPC response for notifications."
|
|
85
86
|
},
|
|
86
87
|
{
|
|
87
88
|
id: "transport-session-id",
|
|
@@ -89,7 +90,8 @@ var TEST_DEFINITIONS = [
|
|
|
89
90
|
category: "transport",
|
|
90
91
|
required: false,
|
|
91
92
|
specRef: "basic/transports#streamable-http",
|
|
92
|
-
description: "Tests that the server returns HTTP 400 when MCP-Session-Id header is missing on requests after initialization (when the server issued a session ID)."
|
|
93
|
+
description: "Tests that the server returns HTTP 400 when MCP-Session-Id header is missing on requests after initialization (when the server issued a session ID).",
|
|
94
|
+
recommendation: "If your server issues an MCP-Session-Id header in the initialize response, reject subsequent requests that omit this header with HTTP 400."
|
|
93
95
|
},
|
|
94
96
|
{
|
|
95
97
|
id: "transport-get",
|
|
@@ -97,7 +99,8 @@ var TEST_DEFINITIONS = [
|
|
|
97
99
|
category: "transport",
|
|
98
100
|
required: false,
|
|
99
101
|
specRef: "basic/transports#streamable-http",
|
|
100
|
-
description: "Tests the GET endpoint for server-initiated messages. Server should return text/event-stream or 405 Method Not Allowed."
|
|
102
|
+
description: "Tests the GET endpoint for server-initiated messages. Server should return text/event-stream or 405 Method Not Allowed.",
|
|
103
|
+
recommendation: "If your server supports server-initiated messages, handle GET with text/event-stream. Otherwise, return 405 Method Not Allowed."
|
|
101
104
|
},
|
|
102
105
|
{
|
|
103
106
|
id: "transport-delete",
|
|
@@ -105,7 +108,8 @@ var TEST_DEFINITIONS = [
|
|
|
105
108
|
category: "transport",
|
|
106
109
|
required: false,
|
|
107
110
|
specRef: "basic/transports#streamable-http",
|
|
108
|
-
description: "Tests the DELETE endpoint for session termination. Server should accept the request or return 405 Method Not Allowed."
|
|
111
|
+
description: "Tests the DELETE endpoint for session termination. Server should accept the request or return 405 Method Not Allowed.",
|
|
112
|
+
recommendation: "Handle DELETE requests for session cleanup, or return 405 if session termination is not supported. Do not return 500."
|
|
109
113
|
},
|
|
110
114
|
{
|
|
111
115
|
id: "transport-batch-reject",
|
|
@@ -113,16 +117,45 @@ var TEST_DEFINITIONS = [
|
|
|
113
117
|
category: "transport",
|
|
114
118
|
required: true,
|
|
115
119
|
specRef: "basic/transports#streamable-http",
|
|
116
|
-
description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests."
|
|
120
|
+
description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests.",
|
|
121
|
+
recommendation: "Check if the parsed JSON body is an array. If so, return a JSON-RPC error or HTTP 400. Do not process batch requests \u2014 MCP explicitly forbids them."
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
id: "transport-content-type-init",
|
|
125
|
+
name: "Initialize response has valid content type",
|
|
126
|
+
category: "transport",
|
|
127
|
+
required: false,
|
|
128
|
+
specRef: "basic/transports#streamable-http",
|
|
129
|
+
description: "Validates that the initialize response uses application/json or text/event-stream content type. Some servers return other types for the handshake.",
|
|
130
|
+
recommendation: 'Ensure the initialize response uses Content-Type "application/json" or "text/event-stream". Do not return text/html or other types for JSON-RPC responses.'
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
id: "transport-get-stream",
|
|
134
|
+
name: "GET with session returns SSE or 405",
|
|
135
|
+
category: "transport",
|
|
136
|
+
required: false,
|
|
137
|
+
specRef: "basic/transports#streamable-http",
|
|
138
|
+
description: "Tests the GET endpoint with an active session ID for server-initiated messages. After initialization, the server should either return an SSE stream or 405.",
|
|
139
|
+
recommendation: "If your server supports server-initiated messages, return text/event-stream on GET with a valid session ID. Otherwise, return 405 Method Not Allowed."
|
|
117
140
|
},
|
|
118
|
-
|
|
141
|
+
{
|
|
142
|
+
id: "transport-concurrent",
|
|
143
|
+
name: "Handles concurrent requests",
|
|
144
|
+
category: "transport",
|
|
145
|
+
required: false,
|
|
146
|
+
specRef: "basic/transports#streamable-http",
|
|
147
|
+
description: "Sends multiple JSON-RPC requests in parallel and verifies the server responds to all with correct matching IDs. Tests that the server can handle concurrent connections.",
|
|
148
|
+
recommendation: "Ensure your server can handle multiple simultaneous requests. Each response must include the correct id matching the request. Use async handlers or connection pooling."
|
|
149
|
+
},
|
|
150
|
+
// ── Lifecycle (12 tests) ─────────────────────────────────────────
|
|
119
151
|
{
|
|
120
152
|
id: "lifecycle-init",
|
|
121
153
|
name: "Initialize handshake",
|
|
122
154
|
category: "lifecycle",
|
|
123
155
|
required: true,
|
|
124
156
|
specRef: "basic/lifecycle#initialization",
|
|
125
|
-
description: "Tests the initialize handshake by sending an initialize request with client capabilities. The server must return a result with protocolVersion."
|
|
157
|
+
description: "Tests the initialize handshake by sending an initialize request with client capabilities. The server must return a result with protocolVersion.",
|
|
158
|
+
recommendation: 'Implement the "initialize" method handler. Return a result object with at least protocolVersion, capabilities, and serverInfo fields.'
|
|
126
159
|
},
|
|
127
160
|
{
|
|
128
161
|
id: "lifecycle-proto-version",
|
|
@@ -130,7 +163,8 @@ var TEST_DEFINITIONS = [
|
|
|
130
163
|
category: "lifecycle",
|
|
131
164
|
required: true,
|
|
132
165
|
specRef: "basic/lifecycle#version-negotiation",
|
|
133
|
-
description: "Validates that the protocolVersion returned by the server matches the YYYY-MM-DD date format required by the spec."
|
|
166
|
+
description: "Validates that the protocolVersion returned by the server matches the YYYY-MM-DD date format required by the spec.",
|
|
167
|
+
recommendation: `Return protocolVersion as a YYYY-MM-DD string (e.g., "2025-11-25"). The server should negotiate based on the client's requested version.`
|
|
134
168
|
},
|
|
135
169
|
{
|
|
136
170
|
id: "lifecycle-server-info",
|
|
@@ -138,7 +172,8 @@ var TEST_DEFINITIONS = [
|
|
|
138
172
|
category: "lifecycle",
|
|
139
173
|
required: false,
|
|
140
174
|
specRef: "basic/lifecycle#initialization",
|
|
141
|
-
description: "Checks that the server includes a serverInfo object with at least a name field in its initialize response. While recommended, this is not strictly required."
|
|
175
|
+
description: "Checks that the server includes a serverInfo object with at least a name field in its initialize response. While recommended, this is not strictly required.",
|
|
176
|
+
recommendation: 'Add a serverInfo object to your initialize response: { name: "your-server", version: "1.0.0" }. This helps clients identify your server.'
|
|
142
177
|
},
|
|
143
178
|
{
|
|
144
179
|
id: "lifecycle-capabilities",
|
|
@@ -146,7 +181,8 @@ var TEST_DEFINITIONS = [
|
|
|
146
181
|
category: "lifecycle",
|
|
147
182
|
required: true,
|
|
148
183
|
specRef: "basic/lifecycle#capability-negotiation",
|
|
149
|
-
description: "Verifies the server returns a capabilities object in its initialize response. An empty object is valid (no optional features declared)."
|
|
184
|
+
description: "Verifies the server returns a capabilities object in its initialize response. An empty object is valid (no optional features declared).",
|
|
185
|
+
recommendation: "Include a capabilities object in your initialize response. Declare the features your server supports (tools, resources, prompts, logging, etc.). An empty object {} is valid."
|
|
150
186
|
},
|
|
151
187
|
{
|
|
152
188
|
id: "lifecycle-jsonrpc",
|
|
@@ -154,7 +190,8 @@ var TEST_DEFINITIONS = [
|
|
|
154
190
|
category: "lifecycle",
|
|
155
191
|
required: true,
|
|
156
192
|
specRef: "basic",
|
|
157
|
-
description: 'Validates that the initialize response is a proper JSON-RPC 2.0 message with jsonrpc="2.0", an id field, and either a result or error field.'
|
|
193
|
+
description: 'Validates that the initialize response is a proper JSON-RPC 2.0 message with jsonrpc="2.0", an id field, and either a result or error field.',
|
|
194
|
+
recommendation: 'Ensure every response includes jsonrpc: "2.0", the matching id from the request, and either a result or error field. Never omit the jsonrpc field.'
|
|
158
195
|
},
|
|
159
196
|
{
|
|
160
197
|
id: "lifecycle-ping",
|
|
@@ -162,7 +199,8 @@ var TEST_DEFINITIONS = [
|
|
|
162
199
|
category: "lifecycle",
|
|
163
200
|
required: true,
|
|
164
201
|
specRef: "basic/utilities#ping",
|
|
165
|
-
description: "Tests that the server responds to the ping method with an empty result object. This is a required utility method."
|
|
202
|
+
description: "Tests that the server responds to the ping method with an empty result object. This is a required utility method.",
|
|
203
|
+
recommendation: 'Implement a "ping" method handler that returns an empty result object {}. This is required by the MCP spec for keepalive and connectivity checking.'
|
|
166
204
|
},
|
|
167
205
|
{
|
|
168
206
|
id: "lifecycle-instructions",
|
|
@@ -170,7 +208,8 @@ var TEST_DEFINITIONS = [
|
|
|
170
208
|
category: "lifecycle",
|
|
171
209
|
required: false,
|
|
172
210
|
specRef: "basic/lifecycle#initialization",
|
|
173
|
-
description: "If the server includes an instructions field in the initialize response, validates it is a string. Instructions provide guidance for how the client should interact with the server."
|
|
211
|
+
description: "If the server includes an instructions field in the initialize response, validates it is a string. Instructions provide guidance for how the client should interact with the server.",
|
|
212
|
+
recommendation: "If you include an instructions field in the initialize response, ensure it is a string. Remove the field or fix the type if it is not a string."
|
|
174
213
|
},
|
|
175
214
|
{
|
|
176
215
|
id: "lifecycle-id-match",
|
|
@@ -178,7 +217,8 @@ var TEST_DEFINITIONS = [
|
|
|
178
217
|
category: "lifecycle",
|
|
179
218
|
required: true,
|
|
180
219
|
specRef: "basic",
|
|
181
|
-
description: "Verifies that the JSON-RPC response id matches the request id sent by the client. This is a fundamental JSON-RPC 2.0 requirement."
|
|
220
|
+
description: "Verifies that the JSON-RPC response id matches the request id sent by the client. This is a fundamental JSON-RPC 2.0 requirement.",
|
|
221
|
+
recommendation: "Copy the id field from the request into the response. This is a core JSON-RPC 2.0 requirement. Check that your framework does not modify or discard the request ID."
|
|
182
222
|
},
|
|
183
223
|
{
|
|
184
224
|
id: "lifecycle-logging",
|
|
@@ -186,7 +226,8 @@ var TEST_DEFINITIONS = [
|
|
|
186
226
|
category: "lifecycle",
|
|
187
227
|
required: false,
|
|
188
228
|
specRef: "server/utilities#logging",
|
|
189
|
-
description: "If the server declares logging capability, tests that logging/setLevel method is accepted with a valid log level."
|
|
229
|
+
description: "If the server declares logging capability, tests that logging/setLevel method is accepted with a valid log level.",
|
|
230
|
+
recommendation: 'If you declare logging in capabilities, implement the "logging/setLevel" handler. Accept standard log levels: debug, info, notice, warning, error, critical, alert, emergency.'
|
|
190
231
|
},
|
|
191
232
|
{
|
|
192
233
|
id: "lifecycle-completions",
|
|
@@ -194,7 +235,26 @@ var TEST_DEFINITIONS = [
|
|
|
194
235
|
category: "lifecycle",
|
|
195
236
|
required: false,
|
|
196
237
|
specRef: "server/utilities#completion",
|
|
197
|
-
description: "If the server declares completions capability, tests that the completion/complete method is accepted."
|
|
238
|
+
description: "If the server declares completions capability, tests that the completion/complete method is accepted.",
|
|
239
|
+
recommendation: 'If you declare completions in capabilities, implement the "completion/complete" handler. Return a completion object with a values array, even if empty.'
|
|
240
|
+
},
|
|
241
|
+
{
|
|
242
|
+
id: "lifecycle-cancellation",
|
|
243
|
+
name: "Handles cancellation notifications",
|
|
244
|
+
category: "lifecycle",
|
|
245
|
+
required: false,
|
|
246
|
+
specRef: "basic/utilities#cancellation",
|
|
247
|
+
description: "Tests that the server accepts notifications/cancelled without error. Servers should gracefully handle cancellation of unknown or completed requests.",
|
|
248
|
+
recommendation: "Accept notifications/cancelled and stop any in-progress work for the referenced requestId. If the request is unknown or already complete, silently ignore the cancellation."
|
|
249
|
+
},
|
|
250
|
+
{
|
|
251
|
+
id: "lifecycle-progress",
|
|
252
|
+
name: "Accepts progress notifications",
|
|
253
|
+
category: "lifecycle",
|
|
254
|
+
required: false,
|
|
255
|
+
specRef: "basic/utilities#progress",
|
|
256
|
+
description: "Tests that the server accepts notifications/progress without error. Servers should handle progress notifications for request tracking.",
|
|
257
|
+
recommendation: "Accept notifications/progress with progressToken, progress, and optional total fields. Ignore notifications for unknown progress tokens."
|
|
198
258
|
},
|
|
199
259
|
// ── Tools (4 tests) ──────────────────────────────────────────────
|
|
200
260
|
{
|
|
@@ -203,7 +263,8 @@ var TEST_DEFINITIONS = [
|
|
|
203
263
|
category: "tools",
|
|
204
264
|
required: false,
|
|
205
265
|
specRef: "server/tools#listing-tools",
|
|
206
|
-
description: "Calls tools/list and validates it returns an array of tool definitions. Required if the server declares tools capability."
|
|
266
|
+
description: "Calls tools/list and validates it returns an array of tool definitions. Required if the server declares tools capability.",
|
|
267
|
+
recommendation: "Implement the tools/list handler to return { tools: [...] } with an array of tool definition objects. Each tool needs at least a name and inputSchema."
|
|
207
268
|
},
|
|
208
269
|
{
|
|
209
270
|
id: "tools-call",
|
|
@@ -211,7 +272,8 @@ var TEST_DEFINITIONS = [
|
|
|
211
272
|
category: "tools",
|
|
212
273
|
required: false,
|
|
213
274
|
specRef: "server/tools#calling-tools",
|
|
214
|
-
description: "Calls the first tool with empty arguments and verifies the response format. Accepts both successful results and InvalidParams errors."
|
|
275
|
+
description: "Calls the first tool with empty arguments and verifies the response format. Accepts both successful results and InvalidParams errors.",
|
|
276
|
+
recommendation: "Ensure tools/call returns { content: [...] } with an array of content objects, each having a type field. Return isError: true for tool execution errors."
|
|
215
277
|
},
|
|
216
278
|
{
|
|
217
279
|
id: "tools-pagination",
|
|
@@ -219,7 +281,8 @@ var TEST_DEFINITIONS = [
|
|
|
219
281
|
category: "tools",
|
|
220
282
|
required: false,
|
|
221
283
|
specRef: "server/tools#listing-tools",
|
|
222
|
-
description: "Tests cursor-based pagination on tools/list. Validates nextCursor is a string if present and that fetching the next page returns a valid response."
|
|
284
|
+
description: "Tests cursor-based pagination on tools/list. Validates nextCursor is a string if present and that fetching the next page returns a valid response.",
|
|
285
|
+
recommendation: "If your server has many tools, include a nextCursor string in the response. Ensure passing this cursor back in a subsequent request returns the next page."
|
|
223
286
|
},
|
|
224
287
|
{
|
|
225
288
|
id: "tools-content-types",
|
|
@@ -227,7 +290,8 @@ var TEST_DEFINITIONS = [
|
|
|
227
290
|
category: "tools",
|
|
228
291
|
required: false,
|
|
229
292
|
specRef: "server/tools#calling-tools",
|
|
230
|
-
description: "Validates that content items returned by tools/call have a recognized type field (text, image, audio, resource, resource_link)."
|
|
293
|
+
description: "Validates that content items returned by tools/call have a recognized type field (text, image, audio, resource, resource_link).",
|
|
294
|
+
recommendation: 'Every content item returned by tools/call must have a type field set to one of: "text", "image", "audio", "resource", or "resource_link". Check for typos or missing type fields.'
|
|
231
295
|
},
|
|
232
296
|
// ── Resources (5 tests) ──────────────────────────────────────────
|
|
233
297
|
{
|
|
@@ -236,7 +300,8 @@ var TEST_DEFINITIONS = [
|
|
|
236
300
|
category: "resources",
|
|
237
301
|
required: false,
|
|
238
302
|
specRef: "server/resources#listing-resources",
|
|
239
|
-
description: "Calls resources/list and validates it returns an array. Required if the server declares resources capability."
|
|
303
|
+
description: "Calls resources/list and validates it returns an array. Required if the server declares resources capability.",
|
|
304
|
+
recommendation: "Implement resources/list to return { resources: [...] } with an array of resource objects. Each resource needs at least a uri and name."
|
|
240
305
|
},
|
|
241
306
|
{
|
|
242
307
|
id: "resources-read",
|
|
@@ -244,7 +309,8 @@ var TEST_DEFINITIONS = [
|
|
|
244
309
|
category: "resources",
|
|
245
310
|
required: false,
|
|
246
311
|
specRef: "server/resources#reading-resources",
|
|
247
|
-
description: "Reads the first resource and validates the response contains a contents array with proper uri and text/blob fields."
|
|
312
|
+
description: "Reads the first resource and validates the response contains a contents array with proper uri and text/blob fields.",
|
|
313
|
+
recommendation: "Implement resources/read to return { contents: [...] } where each item has a uri and either a text or blob field. Ensure the uri matches the requested resource."
|
|
248
314
|
},
|
|
249
315
|
{
|
|
250
316
|
id: "resources-templates",
|
|
@@ -252,7 +318,8 @@ var TEST_DEFINITIONS = [
|
|
|
252
318
|
category: "resources",
|
|
253
319
|
required: false,
|
|
254
320
|
specRef: "server/resources#resource-templates",
|
|
255
|
-
description: "Tests the resource templates endpoint. Accepts Method not found (-32601) since templates are optional."
|
|
321
|
+
description: "Tests the resource templates endpoint. Accepts Method not found (-32601) since templates are optional.",
|
|
322
|
+
recommendation: "If your server supports resource templates, implement resources/templates/list returning { resourceTemplates: [...] }. Otherwise, return error code -32601."
|
|
256
323
|
},
|
|
257
324
|
{
|
|
258
325
|
id: "resources-pagination",
|
|
@@ -260,7 +327,8 @@ var TEST_DEFINITIONS = [
|
|
|
260
327
|
category: "resources",
|
|
261
328
|
required: false,
|
|
262
329
|
specRef: "server/resources#listing-resources",
|
|
263
|
-
description: "Tests cursor-based pagination on resources/list. Validates nextCursor is a string if present and that fetching the next page works."
|
|
330
|
+
description: "Tests cursor-based pagination on resources/list. Validates nextCursor is a string if present and that fetching the next page works.",
|
|
331
|
+
recommendation: "If you return nextCursor in resources/list, ensure it is a string and that passing it back as cursor in the next request returns valid results."
|
|
264
332
|
},
|
|
265
333
|
{
|
|
266
334
|
id: "resources-subscribe",
|
|
@@ -268,7 +336,8 @@ var TEST_DEFINITIONS = [
|
|
|
268
336
|
category: "resources",
|
|
269
337
|
required: false,
|
|
270
338
|
specRef: "server/resources#subscriptions",
|
|
271
|
-
description: "If the server declares resources.subscribe capability, tests that resources/subscribe and resources/unsubscribe methods are accepted."
|
|
339
|
+
description: "If the server declares resources.subscribe capability, tests that resources/subscribe and resources/unsubscribe methods are accepted.",
|
|
340
|
+
recommendation: "If you declare resources.subscribe capability, implement both resources/subscribe and resources/unsubscribe handlers. Both should accept a uri parameter."
|
|
272
341
|
},
|
|
273
342
|
// ── Prompts (3 tests) ────────────────────────────────────────────
|
|
274
343
|
{
|
|
@@ -277,7 +346,8 @@ var TEST_DEFINITIONS = [
|
|
|
277
346
|
category: "prompts",
|
|
278
347
|
required: false,
|
|
279
348
|
specRef: "server/prompts#listing-prompts",
|
|
280
|
-
description: "Calls prompts/list and validates it returns an array. Required if the server declares prompts capability."
|
|
349
|
+
description: "Calls prompts/list and validates it returns an array. Required if the server declares prompts capability.",
|
|
350
|
+
recommendation: "Implement prompts/list to return { prompts: [...] } with an array of prompt objects. Each prompt needs at least a name field."
|
|
281
351
|
},
|
|
282
352
|
{
|
|
283
353
|
id: "prompts-get",
|
|
@@ -285,7 +355,8 @@ var TEST_DEFINITIONS = [
|
|
|
285
355
|
category: "prompts",
|
|
286
356
|
required: false,
|
|
287
357
|
specRef: "server/prompts#getting-a-prompt",
|
|
288
|
-
description: "Gets the first prompt and validates the response contains a messages array with proper role and content fields."
|
|
358
|
+
description: "Gets the first prompt and validates the response contains a messages array with proper role and content fields.",
|
|
359
|
+
recommendation: 'Implement prompts/get to return { messages: [...] } where each message has a role ("user" or "assistant") and a content field.'
|
|
289
360
|
},
|
|
290
361
|
{
|
|
291
362
|
id: "prompts-pagination",
|
|
@@ -293,7 +364,8 @@ var TEST_DEFINITIONS = [
|
|
|
293
364
|
category: "prompts",
|
|
294
365
|
required: false,
|
|
295
366
|
specRef: "server/prompts#listing-prompts",
|
|
296
|
-
description: "Tests cursor-based pagination on prompts/list. Validates nextCursor is a string if present and that fetching the next page works."
|
|
367
|
+
description: "Tests cursor-based pagination on prompts/list. Validates nextCursor is a string if present and that fetching the next page works.",
|
|
368
|
+
recommendation: "If you return nextCursor in prompts/list, ensure it is a string and that passing it back as cursor in the next request returns valid results."
|
|
297
369
|
},
|
|
298
370
|
// ── Error Handling (8 tests) ─────────────────────────────────────
|
|
299
371
|
{
|
|
@@ -302,7 +374,8 @@ var TEST_DEFINITIONS = [
|
|
|
302
374
|
category: "errors",
|
|
303
375
|
required: true,
|
|
304
376
|
specRef: "basic",
|
|
305
|
-
description: "Sends an unknown method and verifies the server returns a JSON-RPC error. The spec requires error code -32601 (Method not found)."
|
|
377
|
+
description: "Sends an unknown method and verifies the server returns a JSON-RPC error. The spec requires error code -32601 (Method not found).",
|
|
378
|
+
recommendation: "Return a JSON-RPC error with code -32601 (Method not found) for any unrecognized method name. Do not silently ignore unknown methods."
|
|
306
379
|
},
|
|
307
380
|
{
|
|
308
381
|
id: "error-method-code",
|
|
@@ -310,7 +383,8 @@ var TEST_DEFINITIONS = [
|
|
|
310
383
|
category: "errors",
|
|
311
384
|
required: false,
|
|
312
385
|
specRef: "basic",
|
|
313
|
-
description: "Checks the error code is specifically -32601 (Method not found) for unknown methods, as required by JSON-RPC 2.0."
|
|
386
|
+
description: "Checks the error code is specifically -32601 (Method not found) for unknown methods, as required by JSON-RPC 2.0.",
|
|
387
|
+
recommendation: "Use exactly error code -32601 for unknown methods. Do not use generic error codes like -32000. This is required by JSON-RPC 2.0."
|
|
314
388
|
},
|
|
315
389
|
{
|
|
316
390
|
id: "error-invalid-jsonrpc",
|
|
@@ -318,7 +392,8 @@ var TEST_DEFINITIONS = [
|
|
|
318
392
|
category: "errors",
|
|
319
393
|
required: true,
|
|
320
394
|
specRef: "basic",
|
|
321
|
-
description: "Sends a malformed JSON-RPC message (missing required fields) and verifies the server returns an error or 4xx status."
|
|
395
|
+
description: "Sends a malformed JSON-RPC message (missing required fields) and verifies the server returns an error or 4xx status.",
|
|
396
|
+
recommendation: "Validate incoming JSON-RPC messages for required fields (jsonrpc, method). Return error code -32600 (Invalid Request) or HTTP 400 for malformed messages."
|
|
322
397
|
},
|
|
323
398
|
{
|
|
324
399
|
id: "error-invalid-json",
|
|
@@ -326,7 +401,8 @@ var TEST_DEFINITIONS = [
|
|
|
326
401
|
category: "errors",
|
|
327
402
|
required: false,
|
|
328
403
|
specRef: "basic",
|
|
329
|
-
description: "Sends invalid JSON and verifies the server returns a parse error (-32700) or 4xx status code."
|
|
404
|
+
description: "Sends invalid JSON and verifies the server returns a parse error (-32700) or 4xx status code.",
|
|
405
|
+
recommendation: "Catch JSON parse errors and return error code -32700 (Parse error) with a descriptive message. Do not return 500 for malformed input."
|
|
330
406
|
},
|
|
331
407
|
{
|
|
332
408
|
id: "error-missing-params",
|
|
@@ -334,7 +410,8 @@ var TEST_DEFINITIONS = [
|
|
|
334
410
|
category: "errors",
|
|
335
411
|
required: false,
|
|
336
412
|
specRef: "server/tools#error-handling",
|
|
337
|
-
description: "Calls tools/call with an empty params object (missing required name field) and verifies an error is returned."
|
|
413
|
+
description: "Calls tools/call with an empty params object (missing required name field) and verifies an error is returned.",
|
|
414
|
+
recommendation: "Validate tools/call params and return error code -32602 (Invalid params) when the required name field is missing."
|
|
338
415
|
},
|
|
339
416
|
{
|
|
340
417
|
id: "error-parse-code",
|
|
@@ -342,7 +419,8 @@ var TEST_DEFINITIONS = [
|
|
|
342
419
|
category: "errors",
|
|
343
420
|
required: false,
|
|
344
421
|
specRef: "basic",
|
|
345
|
-
description: "Checks that the server returns the specific JSON-RPC error code -32700 (Parse error) when receiving invalid JSON, as required by the JSON-RPC 2.0 specification."
|
|
422
|
+
description: "Checks that the server returns the specific JSON-RPC error code -32700 (Parse error) when receiving invalid JSON, as required by the JSON-RPC 2.0 specification.",
|
|
423
|
+
recommendation: "Return exactly error code -32700 for JSON parse failures. Most JSON-RPC frameworks handle this automatically \u2014 check yours does not override the code."
|
|
346
424
|
},
|
|
347
425
|
{
|
|
348
426
|
id: "error-invalid-request-code",
|
|
@@ -350,7 +428,8 @@ var TEST_DEFINITIONS = [
|
|
|
350
428
|
category: "errors",
|
|
351
429
|
required: false,
|
|
352
430
|
specRef: "basic",
|
|
353
|
-
description: "Checks that the server returns the specific JSON-RPC error code -32600 (Invalid Request) for malformed JSON-RPC messages missing required fields."
|
|
431
|
+
description: "Checks that the server returns the specific JSON-RPC error code -32600 (Invalid Request) for malformed JSON-RPC messages missing required fields.",
|
|
432
|
+
recommendation: "Return exactly error code -32600 for structurally invalid JSON-RPC messages (e.g., missing method field). Check your JSON-RPC middleware configuration."
|
|
354
433
|
},
|
|
355
434
|
{
|
|
356
435
|
id: "tools-call-unknown",
|
|
@@ -358,7 +437,8 @@ var TEST_DEFINITIONS = [
|
|
|
358
437
|
category: "errors",
|
|
359
438
|
required: false,
|
|
360
439
|
specRef: "server/tools#error-handling",
|
|
361
|
-
description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response."
|
|
440
|
+
description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response.",
|
|
441
|
+
recommendation: "Return a JSON-RPC error or set isError: true when tools/call receives an unrecognized tool name. Do not return an empty success response."
|
|
362
442
|
},
|
|
363
443
|
// ── Schema Validation (6 tests) ──────────────────────────────────
|
|
364
444
|
{
|
|
@@ -367,7 +447,8 @@ var TEST_DEFINITIONS = [
|
|
|
367
447
|
category: "schema",
|
|
368
448
|
required: false,
|
|
369
449
|
specRef: "server/tools#data-types",
|
|
370
|
-
description: 'Validates every tool has a valid name (1-128 chars, alphanumeric/underscore/hyphen/dot) and a required inputSchema of type "object".'
|
|
450
|
+
description: 'Validates every tool has a valid name (1-128 chars, alphanumeric/underscore/hyphen/dot) and a required inputSchema of type "object".',
|
|
451
|
+
recommendation: 'Ensure every tool has a name (1-128 chars, [A-Za-z0-9_.-]) and an inputSchema with type: "object". Add descriptions to tools for better AI assistant integration.'
|
|
371
452
|
},
|
|
372
453
|
{
|
|
373
454
|
id: "tools-annotations",
|
|
@@ -375,7 +456,8 @@ var TEST_DEFINITIONS = [
|
|
|
375
456
|
category: "schema",
|
|
376
457
|
required: false,
|
|
377
458
|
specRef: "server/tools#annotations",
|
|
378
|
-
description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans; title should be a string."
|
|
459
|
+
description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans; title should be a string.",
|
|
460
|
+
recommendation: "If you include annotations on tools, ensure readOnlyHint, destructiveHint, idempotentHint, and openWorldHint are booleans. Title must be a string."
|
|
379
461
|
},
|
|
380
462
|
{
|
|
381
463
|
id: "tools-title-field",
|
|
@@ -383,7 +465,8 @@ var TEST_DEFINITIONS = [
|
|
|
383
465
|
category: "schema",
|
|
384
466
|
required: false,
|
|
385
467
|
specRef: "server/tools#data-types",
|
|
386
|
-
description: "Checks if tools include the optional title field for human-readable display names. Added in spec version 2025-11-25."
|
|
468
|
+
description: "Checks if tools include the optional title field for human-readable display names. Added in spec version 2025-11-25.",
|
|
469
|
+
recommendation: "Add a title field (human-readable string) to each tool definition. This helps MCP clients display your tools in a user-friendly way."
|
|
387
470
|
},
|
|
388
471
|
{
|
|
389
472
|
id: "tools-output-schema",
|
|
@@ -391,7 +474,8 @@ var TEST_DEFINITIONS = [
|
|
|
391
474
|
category: "schema",
|
|
392
475
|
required: false,
|
|
393
476
|
specRef: "server/tools#structured-content",
|
|
394
|
-
description: 'If tools declare an outputSchema, validates it is a valid JSON Schema object with type "object". Used for structured output validation.'
|
|
477
|
+
description: 'If tools declare an outputSchema, validates it is a valid JSON Schema object with type "object". Used for structured output validation.',
|
|
478
|
+
recommendation: 'If you declare outputSchema on a tool, ensure it is a valid JSON Schema object with type: "object". Remove outputSchema if you do not need structured output.'
|
|
395
479
|
},
|
|
396
480
|
{
|
|
397
481
|
id: "prompts-schema",
|
|
@@ -399,7 +483,8 @@ var TEST_DEFINITIONS = [
|
|
|
399
483
|
category: "schema",
|
|
400
484
|
required: false,
|
|
401
485
|
specRef: "server/prompts#data-types",
|
|
402
|
-
description: "Validates every prompt has a name and that any arguments array contains items with name fields."
|
|
486
|
+
description: "Validates every prompt has a name and that any arguments array contains items with name fields.",
|
|
487
|
+
recommendation: "Ensure every prompt has a name field. If the prompt has arguments, each argument object must include a name field."
|
|
403
488
|
},
|
|
404
489
|
{
|
|
405
490
|
id: "resources-schema",
|
|
@@ -407,7 +492,8 @@ var TEST_DEFINITIONS = [
|
|
|
407
492
|
category: "schema",
|
|
408
493
|
required: false,
|
|
409
494
|
specRef: "server/resources#data-types",
|
|
410
|
-
description: "Validates every resource has a valid URI (parseable as a URL) and a name field."
|
|
495
|
+
description: "Validates every resource has a valid URI (parseable as a URL) and a name field.",
|
|
496
|
+
recommendation: "Ensure every resource has a valid, parseable URI and a name field. Add description and mimeType for better client integration."
|
|
411
497
|
}
|
|
412
498
|
];
|
|
413
499
|
|
|
@@ -511,19 +597,39 @@ async function mcpNotification(backendUrl, method, params, extraHeaders, timeout
|
|
|
511
597
|
return { statusCode: res.statusCode, headers: responseHeaders };
|
|
512
598
|
}
|
|
513
599
|
async function runComplianceSuite(url, options = {}) {
|
|
514
|
-
let parsed;
|
|
515
600
|
try {
|
|
516
|
-
parsed = new URL(url);
|
|
601
|
+
const parsed = new URL(url);
|
|
517
602
|
if (!["http:", "https:"].includes(parsed.protocol)) {
|
|
518
603
|
throw new Error("Only HTTP and HTTPS URLs are supported");
|
|
519
604
|
}
|
|
520
605
|
} catch (e) {
|
|
521
|
-
if (e.message.includes("Only HTTP")) throw e;
|
|
606
|
+
if (e instanceof Error && e.message.includes("Only HTTP")) throw e;
|
|
522
607
|
throw new Error(`Invalid URL: ${url}`);
|
|
523
608
|
}
|
|
524
609
|
const backendUrl = url;
|
|
610
|
+
let serverReachable = true;
|
|
611
|
+
try {
|
|
612
|
+
const preflight = await request(backendUrl, {
|
|
613
|
+
method: "POST",
|
|
614
|
+
headers: {
|
|
615
|
+
"Content-Type": "application/json",
|
|
616
|
+
Accept: "application/json, text/event-stream",
|
|
617
|
+
...options.headers
|
|
618
|
+
},
|
|
619
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 0, method: "ping" }),
|
|
620
|
+
signal: AbortSignal.timeout(Math.min(options.timeout || 15e3, 1e4))
|
|
621
|
+
});
|
|
622
|
+
await preflight.body.text();
|
|
623
|
+
} catch {
|
|
624
|
+
serverReachable = false;
|
|
625
|
+
}
|
|
525
626
|
const tests = [];
|
|
526
627
|
const warnings = [];
|
|
628
|
+
if (!serverReachable) {
|
|
629
|
+
warnings.push(
|
|
630
|
+
`Server at ${url} is unreachable \u2014 all tests will fail. Check the URL and ensure the server is running.`
|
|
631
|
+
);
|
|
632
|
+
}
|
|
527
633
|
const nextId = createIdCounter(1e3);
|
|
528
634
|
const timeout = options.timeout || 15e3;
|
|
529
635
|
const retries = options.retries || 0;
|
|
@@ -568,7 +674,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
568
674
|
if (lastResult.passed) break;
|
|
569
675
|
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
|
|
570
676
|
} catch (err) {
|
|
571
|
-
|
|
677
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
678
|
+
lastResult = { passed: false, details: `Error: ${message}` };
|
|
572
679
|
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
|
|
573
680
|
}
|
|
574
681
|
}
|
|
@@ -597,10 +704,23 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
597
704
|
body: JSON.stringify({ jsonrpc: "2.0", id: 99901, method: "ping" }),
|
|
598
705
|
signal: AbortSignal.timeout(timeout)
|
|
599
706
|
});
|
|
600
|
-
await res.body.text();
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
707
|
+
const text = await res.body.text();
|
|
708
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
709
|
+
return { passed: true, details: `HTTP ${res.statusCode}` };
|
|
710
|
+
}
|
|
711
|
+
if (res.statusCode === 401 || res.statusCode === 403) {
|
|
712
|
+
return { passed: false, details: `HTTP ${res.statusCode} (auth required \u2014 pass --auth)` };
|
|
713
|
+
}
|
|
714
|
+
if (res.statusCode === 400) {
|
|
715
|
+
try {
|
|
716
|
+
const body = JSON.parse(text);
|
|
717
|
+
if (body?.error || body?.jsonrpc) {
|
|
718
|
+
return { passed: true, details: "HTTP 400 with JSON-RPC response (server requires initialization first)" };
|
|
719
|
+
}
|
|
720
|
+
} catch {
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
604
724
|
}
|
|
605
725
|
);
|
|
606
726
|
await test(
|
|
@@ -630,18 +750,29 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
630
750
|
false,
|
|
631
751
|
"basic/transports#streamable-http",
|
|
632
752
|
async () => {
|
|
753
|
+
const getHeaders = { Accept: "text/event-stream", ...buildHeaders() };
|
|
633
754
|
const res = await request(backendUrl, {
|
|
634
755
|
method: "GET",
|
|
635
|
-
headers:
|
|
756
|
+
headers: getHeaders,
|
|
636
757
|
signal: AbortSignal.timeout(timeout)
|
|
637
758
|
});
|
|
638
|
-
await res.body.text();
|
|
759
|
+
const body = await res.body.text();
|
|
639
760
|
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
640
761
|
if (res.statusCode === 405) {
|
|
641
762
|
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
642
763
|
}
|
|
643
764
|
if (ct.includes("text/event-stream")) {
|
|
644
|
-
|
|
765
|
+
if (body.trim().length > 0) {
|
|
766
|
+
const hasDataFields = body.includes("data:");
|
|
767
|
+
const hasEventFields = body.includes("event:");
|
|
768
|
+
if (!hasDataFields && !hasEventFields) {
|
|
769
|
+
return {
|
|
770
|
+
passed: false,
|
|
771
|
+
details: "Content-Type is text/event-stream but body has no SSE data: or event: fields"
|
|
772
|
+
};
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
return { passed: true, details: "Returns text/event-stream with valid SSE format" };
|
|
645
776
|
}
|
|
646
777
|
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
647
778
|
return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
|
|
@@ -649,31 +780,6 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
649
780
|
return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
|
|
650
781
|
}
|
|
651
782
|
);
|
|
652
|
-
await test(
|
|
653
|
-
"transport-delete",
|
|
654
|
-
"DELETE accepted or returns 405",
|
|
655
|
-
"transport",
|
|
656
|
-
false,
|
|
657
|
-
"basic/transports#streamable-http",
|
|
658
|
-
async () => {
|
|
659
|
-
const res = await request(backendUrl, {
|
|
660
|
-
method: "DELETE",
|
|
661
|
-
headers: { ...userHeaders },
|
|
662
|
-
signal: AbortSignal.timeout(timeout)
|
|
663
|
-
});
|
|
664
|
-
await res.body.text();
|
|
665
|
-
if (res.statusCode === 405) {
|
|
666
|
-
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
667
|
-
}
|
|
668
|
-
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
669
|
-
return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
|
|
670
|
-
}
|
|
671
|
-
if (res.statusCode === 400 || res.statusCode === 404) {
|
|
672
|
-
return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
|
|
673
|
-
}
|
|
674
|
-
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
675
|
-
}
|
|
676
|
-
);
|
|
677
783
|
await test(
|
|
678
784
|
"transport-batch-reject",
|
|
679
785
|
"Rejects JSON-RPC batch requests",
|
|
@@ -711,7 +817,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
711
817
|
try {
|
|
712
818
|
initRes = await rpc("initialize", {
|
|
713
819
|
protocolVersion: SPEC_VERSION,
|
|
714
|
-
capabilities: {
|
|
820
|
+
capabilities: {},
|
|
715
821
|
clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
|
|
716
822
|
});
|
|
717
823
|
const result = initRes?.body?.result;
|
|
@@ -724,7 +830,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
724
830
|
if (sid) sessionId = sid;
|
|
725
831
|
if (result.protocolVersion) negotiatedProtocolVersion = result.protocolVersion;
|
|
726
832
|
}
|
|
727
|
-
} catch
|
|
833
|
+
} catch {
|
|
728
834
|
}
|
|
729
835
|
try {
|
|
730
836
|
await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders(), timeout);
|
|
@@ -843,7 +949,17 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
843
949
|
if (res.body?.error) {
|
|
844
950
|
return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
|
|
845
951
|
}
|
|
846
|
-
|
|
952
|
+
const invalidRes = await rpc("logging/setLevel", { level: "__invalid_level__" });
|
|
953
|
+
const validatesInput = !!invalidRes.body?.error;
|
|
954
|
+
const validLevels = ["debug", "warning", "error"];
|
|
955
|
+
const accepted = [];
|
|
956
|
+
for (const level of validLevels) {
|
|
957
|
+
const r = await rpc("logging/setLevel", { level });
|
|
958
|
+
if (!r.body?.error) accepted.push(level);
|
|
959
|
+
}
|
|
960
|
+
const details = validatesInput ? `logging/setLevel accepted (validates levels, ${accepted.length + 1} levels accepted)` : "logging/setLevel accepted (warning: server does not reject invalid log levels)";
|
|
961
|
+
if (!validatesInput) warnings.push("Server accepts invalid log levels without error");
|
|
962
|
+
return { passed: true, details };
|
|
847
963
|
}
|
|
848
964
|
);
|
|
849
965
|
const hasCompletions = !!serverInfo.capabilities.completions;
|
|
@@ -872,6 +988,59 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
872
988
|
return { passed: true, details: "completion/complete accepted" };
|
|
873
989
|
}
|
|
874
990
|
);
|
|
991
|
+
await test(
|
|
992
|
+
"lifecycle-cancellation",
|
|
993
|
+
"Handles cancellation notifications",
|
|
994
|
+
"lifecycle",
|
|
995
|
+
false,
|
|
996
|
+
"basic/utilities#cancellation",
|
|
997
|
+
async () => {
|
|
998
|
+
const res = await mcpNotification(
|
|
999
|
+
backendUrl,
|
|
1000
|
+
"notifications/cancelled",
|
|
1001
|
+
{ requestId: 99999, reason: "compliance test" },
|
|
1002
|
+
buildHeaders(),
|
|
1003
|
+
timeout
|
|
1004
|
+
);
|
|
1005
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1006
|
+
return { passed: true, details: `HTTP ${res.statusCode} (cancellation accepted)` };
|
|
1007
|
+
}
|
|
1008
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should accept cancellation notifications` };
|
|
1009
|
+
}
|
|
1010
|
+
);
|
|
1011
|
+
await test(
|
|
1012
|
+
"lifecycle-progress",
|
|
1013
|
+
"Accepts progress notifications",
|
|
1014
|
+
"lifecycle",
|
|
1015
|
+
false,
|
|
1016
|
+
"basic/utilities#progress",
|
|
1017
|
+
async () => {
|
|
1018
|
+
const res = await mcpNotification(
|
|
1019
|
+
backendUrl,
|
|
1020
|
+
"notifications/progress",
|
|
1021
|
+
{ progressToken: "compliance-test-token", progress: 50, total: 100 },
|
|
1022
|
+
buildHeaders(),
|
|
1023
|
+
timeout
|
|
1024
|
+
);
|
|
1025
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1026
|
+
return { passed: true, details: `HTTP ${res.statusCode} (progress notification accepted)` };
|
|
1027
|
+
}
|
|
1028
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should accept progress notifications` };
|
|
1029
|
+
}
|
|
1030
|
+
);
|
|
1031
|
+
await test(
|
|
1032
|
+
"transport-content-type-init",
|
|
1033
|
+
"Initialize response has valid content type",
|
|
1034
|
+
"transport",
|
|
1035
|
+
false,
|
|
1036
|
+
"basic/transports#streamable-http",
|
|
1037
|
+
async () => {
|
|
1038
|
+
if (!initRes) return { passed: false, details: "No init response to check" };
|
|
1039
|
+
const ct = (initRes.headers["content-type"] || "").toLowerCase();
|
|
1040
|
+
const valid = ct.includes("application/json") || ct.includes("text/event-stream");
|
|
1041
|
+
return { passed: valid, details: `Content-Type: ${ct || "missing"}` };
|
|
1042
|
+
}
|
|
1043
|
+
);
|
|
875
1044
|
await test(
|
|
876
1045
|
"transport-notification-202",
|
|
877
1046
|
"Notification returns 202 Accepted",
|
|
@@ -929,6 +1098,88 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
929
1098
|
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
930
1099
|
}
|
|
931
1100
|
);
|
|
1101
|
+
await test(
|
|
1102
|
+
"transport-get-stream",
|
|
1103
|
+
"GET with session returns SSE or 405",
|
|
1104
|
+
"transport",
|
|
1105
|
+
false,
|
|
1106
|
+
"basic/transports#streamable-http",
|
|
1107
|
+
async () => {
|
|
1108
|
+
if (!sessionId) {
|
|
1109
|
+
return { passed: true, details: "No session ID \u2014 server-initiated messages not applicable" };
|
|
1110
|
+
}
|
|
1111
|
+
const res = await request(backendUrl, {
|
|
1112
|
+
method: "GET",
|
|
1113
|
+
headers: { Accept: "text/event-stream", ...buildHeaders() },
|
|
1114
|
+
signal: AbortSignal.timeout(Math.min(timeout, 3e3))
|
|
1115
|
+
});
|
|
1116
|
+
const body = await res.body.text();
|
|
1117
|
+
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
1118
|
+
if (res.statusCode === 405) {
|
|
1119
|
+
return { passed: true, details: "HTTP 405 (server does not support server-initiated messages)" };
|
|
1120
|
+
}
|
|
1121
|
+
if (ct.includes("text/event-stream")) {
|
|
1122
|
+
if (body.trim().length > 0) {
|
|
1123
|
+
const hasSSEFields = body.includes("data:") || body.includes("event:");
|
|
1124
|
+
if (!hasSSEFields) {
|
|
1125
|
+
return { passed: false, details: "Content-Type is text/event-stream but body has no SSE fields" };
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
return { passed: true, details: "GET with session returns SSE stream for server-initiated messages" };
|
|
1129
|
+
}
|
|
1130
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1131
|
+
return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
|
|
1132
|
+
}
|
|
1133
|
+
return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
|
|
1134
|
+
}
|
|
1135
|
+
);
|
|
1136
|
+
await test(
|
|
1137
|
+
"transport-concurrent",
|
|
1138
|
+
"Handles concurrent requests",
|
|
1139
|
+
"transport",
|
|
1140
|
+
false,
|
|
1141
|
+
"basic/transports#streamable-http",
|
|
1142
|
+
async () => {
|
|
1143
|
+
const ids = [createIdCounter(99930)(), createIdCounter(99931)(), createIdCounter(99932)()];
|
|
1144
|
+
const promises = ids.map(
|
|
1145
|
+
(id) => request(backendUrl, {
|
|
1146
|
+
method: "POST",
|
|
1147
|
+
headers: {
|
|
1148
|
+
"Content-Type": "application/json",
|
|
1149
|
+
Accept: "application/json, text/event-stream",
|
|
1150
|
+
...buildHeaders()
|
|
1151
|
+
},
|
|
1152
|
+
body: JSON.stringify({ jsonrpc: "2.0", id, method: "ping" }),
|
|
1153
|
+
signal: AbortSignal.timeout(timeout)
|
|
1154
|
+
}).then(async (res) => {
|
|
1155
|
+
const text = await res.body.text();
|
|
1156
|
+
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
1157
|
+
let body;
|
|
1158
|
+
if (ct.includes("text/event-stream")) {
|
|
1159
|
+
body = parseSSEResponse(text);
|
|
1160
|
+
}
|
|
1161
|
+
if (!body) {
|
|
1162
|
+
try {
|
|
1163
|
+
body = JSON.parse(text);
|
|
1164
|
+
} catch {
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
return { statusCode: res.statusCode, body, requestId: id };
|
|
1168
|
+
})
|
|
1169
|
+
);
|
|
1170
|
+
const results = await Promise.all(promises);
|
|
1171
|
+
const issues = [];
|
|
1172
|
+
for (const r of results) {
|
|
1173
|
+
if (r.statusCode < 200 || r.statusCode >= 300) {
|
|
1174
|
+
issues.push(`Request id=${r.requestId}: HTTP ${r.statusCode}`);
|
|
1175
|
+
} else if (r.body?.id !== r.requestId) {
|
|
1176
|
+
issues.push(`Request id=${r.requestId}: response id=${r.body?.id} (mismatch)`);
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1180
|
+
return { passed: true, details: `${results.length} concurrent requests handled correctly` };
|
|
1181
|
+
}
|
|
1182
|
+
);
|
|
932
1183
|
const hasTools = !!serverInfo.capabilities.tools;
|
|
933
1184
|
let cachedToolsList = null;
|
|
934
1185
|
await test(
|
|
@@ -950,6 +1201,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
950
1201
|
};
|
|
951
1202
|
}
|
|
952
1203
|
);
|
|
1204
|
+
const toolsListOk = cachedToolsList !== null;
|
|
953
1205
|
await test(
|
|
954
1206
|
"tools-schema",
|
|
955
1207
|
"All tools have name and inputSchema",
|
|
@@ -957,7 +1209,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
957
1209
|
hasTools,
|
|
958
1210
|
"server/tools#data-types",
|
|
959
1211
|
async () => {
|
|
960
|
-
|
|
1212
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1213
|
+
const tools = cachedToolsList ?? [];
|
|
961
1214
|
const issues = [];
|
|
962
1215
|
for (const tool of tools) {
|
|
963
1216
|
if (!tool.name) {
|
|
@@ -989,7 +1242,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
989
1242
|
false,
|
|
990
1243
|
"server/tools#annotations",
|
|
991
1244
|
async () => {
|
|
992
|
-
|
|
1245
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1246
|
+
const tools = cachedToolsList ?? [];
|
|
993
1247
|
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
994
1248
|
const issues = [];
|
|
995
1249
|
let annotatedCount = 0;
|
|
@@ -1019,7 +1273,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1019
1273
|
}
|
|
1020
1274
|
);
|
|
1021
1275
|
await test("tools-title-field", "Tools include title field", "schema", false, "server/tools#data-types", async () => {
|
|
1022
|
-
|
|
1276
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1277
|
+
const tools = cachedToolsList ?? [];
|
|
1023
1278
|
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1024
1279
|
const withTitle = tools.filter((t) => typeof t.title === "string");
|
|
1025
1280
|
const issues = [];
|
|
@@ -1041,7 +1296,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1041
1296
|
false,
|
|
1042
1297
|
"server/tools#structured-content",
|
|
1043
1298
|
async () => {
|
|
1044
|
-
|
|
1299
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1300
|
+
const tools = cachedToolsList ?? [];
|
|
1045
1301
|
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1046
1302
|
const issues = [];
|
|
1047
1303
|
let withSchema = 0;
|
|
@@ -1082,14 +1338,14 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1082
1338
|
return { passed: true, details: `Protocol error: code ${code} \u2014 ${error.message}` };
|
|
1083
1339
|
}
|
|
1084
1340
|
if (result?.content && Array.isArray(result.content)) {
|
|
1341
|
+
if (result.isError) {
|
|
1342
|
+
return { passed: true, details: "Tool returned execution error with content (valid)" };
|
|
1343
|
+
}
|
|
1085
1344
|
const badItems = result.content.filter((c) => !c.type);
|
|
1086
1345
|
if (badItems.length > 0)
|
|
1087
1346
|
return { passed: false, details: `${badItems.length} content item(s) missing 'type' field` };
|
|
1088
1347
|
return { passed: true, details: `Returned ${result.content.length} content item(s)` };
|
|
1089
1348
|
}
|
|
1090
|
-
if (result?.isError && result?.content && Array.isArray(result.content)) {
|
|
1091
|
-
return { passed: true, details: "Tool returned execution error with content (valid)" };
|
|
1092
|
-
}
|
|
1093
1349
|
return { passed: false, details: "Response missing content array" };
|
|
1094
1350
|
}
|
|
1095
1351
|
);
|
|
@@ -1191,6 +1447,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1191
1447
|
return { passed: true, details: `${resourceCount} resource(s)` };
|
|
1192
1448
|
}
|
|
1193
1449
|
);
|
|
1450
|
+
const resourcesListOk = cachedResourcesList !== null;
|
|
1194
1451
|
await test(
|
|
1195
1452
|
"resources-schema",
|
|
1196
1453
|
"Resources have uri and name",
|
|
@@ -1198,7 +1455,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1198
1455
|
true,
|
|
1199
1456
|
"server/resources#data-types",
|
|
1200
1457
|
async () => {
|
|
1201
|
-
|
|
1458
|
+
if (!resourcesListOk) return { passed: false, details: "Skipped: resources/list failed" };
|
|
1459
|
+
const resources = cachedResourcesList ?? [];
|
|
1202
1460
|
const issues = [];
|
|
1203
1461
|
for (const r of resources) {
|
|
1204
1462
|
if (!r.uri) issues.push("Resource missing uri");
|
|
@@ -1227,7 +1485,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1227
1485
|
false,
|
|
1228
1486
|
"server/resources#reading-resources",
|
|
1229
1487
|
async () => {
|
|
1230
|
-
const resources = cachedResourcesList ??
|
|
1488
|
+
const resources = cachedResourcesList ?? [];
|
|
1231
1489
|
const firstUri = resources[0]?.uri;
|
|
1232
1490
|
if (!firstUri) return { passed: false, details: "No resource URI to test" };
|
|
1233
1491
|
const readRes = await rpc("resources/read", { uri: firstUri });
|
|
@@ -1260,8 +1518,15 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1260
1518
|
if (!Array.isArray(templates)) return { passed: false, details: "No resourceTemplates array" };
|
|
1261
1519
|
const issues = [];
|
|
1262
1520
|
for (const t of templates) {
|
|
1263
|
-
if (!t.uriTemplate)
|
|
1521
|
+
if (!t.uriTemplate) {
|
|
1522
|
+
issues.push("Template missing uriTemplate");
|
|
1523
|
+
} else if (typeof t.uriTemplate !== "string") {
|
|
1524
|
+
issues.push(`uriTemplate should be a string, got ${typeof t.uriTemplate}`);
|
|
1525
|
+
} else if (!t.uriTemplate.includes("{") || !t.uriTemplate.includes("}")) {
|
|
1526
|
+
warnings.push(`Template "${t.name || t.uriTemplate}" has no URI template parameters (e.g., {id})`);
|
|
1527
|
+
}
|
|
1264
1528
|
if (!t.name) issues.push(`${t.uriTemplate || "?"}: missing name`);
|
|
1529
|
+
if (!t.description) warnings.push(`Template "${t.name || t.uriTemplate || "?"}" missing description`);
|
|
1265
1530
|
}
|
|
1266
1531
|
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1267
1532
|
return { passed: true, details: `${templates.length} resource template(s)` };
|
|
@@ -1303,7 +1568,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1303
1568
|
true,
|
|
1304
1569
|
"server/resources#subscriptions",
|
|
1305
1570
|
async () => {
|
|
1306
|
-
const resources = cachedResourcesList ??
|
|
1571
|
+
const resources = cachedResourcesList ?? [];
|
|
1307
1572
|
const firstUri = resources[0]?.uri;
|
|
1308
1573
|
if (!firstUri) return { passed: false, details: "No resource URI for subscribe test" };
|
|
1309
1574
|
const subRes = await rpc("resources/subscribe", { uri: firstUri });
|
|
@@ -1347,8 +1612,10 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1347
1612
|
};
|
|
1348
1613
|
}
|
|
1349
1614
|
);
|
|
1615
|
+
const promptsListOk = cachedPromptsList !== null;
|
|
1350
1616
|
await test("prompts-schema", "Prompts have name field", "schema", true, "server/prompts#data-types", async () => {
|
|
1351
|
-
|
|
1617
|
+
if (!promptsListOk) return { passed: false, details: "Skipped: prompts/list failed" };
|
|
1618
|
+
const prompts = cachedPromptsList ?? [];
|
|
1352
1619
|
const issues = [];
|
|
1353
1620
|
for (const p of prompts) {
|
|
1354
1621
|
if (!p.name) issues.push("Prompt missing name");
|
|
@@ -1544,6 +1811,60 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1544
1811
|
}
|
|
1545
1812
|
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error code -32600` };
|
|
1546
1813
|
});
|
|
1814
|
+
await test(
|
|
1815
|
+
"transport-delete",
|
|
1816
|
+
"DELETE accepted or returns 405",
|
|
1817
|
+
"transport",
|
|
1818
|
+
false,
|
|
1819
|
+
"basic/transports#streamable-http",
|
|
1820
|
+
async () => {
|
|
1821
|
+
const deleteHeaders = { ...buildHeaders() };
|
|
1822
|
+
const res = await request(backendUrl, {
|
|
1823
|
+
method: "DELETE",
|
|
1824
|
+
headers: deleteHeaders,
|
|
1825
|
+
signal: AbortSignal.timeout(timeout)
|
|
1826
|
+
});
|
|
1827
|
+
await res.body.text();
|
|
1828
|
+
if (res.statusCode === 405) {
|
|
1829
|
+
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
1830
|
+
}
|
|
1831
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1832
|
+
if (sessionId) {
|
|
1833
|
+
try {
|
|
1834
|
+
const verifyRes = await mcpRequest(
|
|
1835
|
+
backendUrl,
|
|
1836
|
+
"ping",
|
|
1837
|
+
void 0,
|
|
1838
|
+
createIdCounter(99920),
|
|
1839
|
+
deleteHeaders,
|
|
1840
|
+
timeout
|
|
1841
|
+
);
|
|
1842
|
+
if (verifyRes.statusCode === 400 || verifyRes.statusCode === 404 || verifyRes.statusCode === 409) {
|
|
1843
|
+
return {
|
|
1844
|
+
passed: true,
|
|
1845
|
+
details: `HTTP ${res.statusCode} (session terminated, post-delete request correctly rejected with ${verifyRes.statusCode})`
|
|
1846
|
+
};
|
|
1847
|
+
}
|
|
1848
|
+
} catch {
|
|
1849
|
+
return {
|
|
1850
|
+
passed: true,
|
|
1851
|
+
details: `HTTP ${res.statusCode} (session terminated, post-delete request rejected)`
|
|
1852
|
+
};
|
|
1853
|
+
}
|
|
1854
|
+
}
|
|
1855
|
+
return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
|
|
1856
|
+
}
|
|
1857
|
+
if (res.statusCode === 400 || res.statusCode === 404) {
|
|
1858
|
+
return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
|
|
1859
|
+
}
|
|
1860
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
1861
|
+
}
|
|
1862
|
+
);
|
|
1863
|
+
const MAX_WARNINGS = 50;
|
|
1864
|
+
if (warnings.length > MAX_WARNINGS) {
|
|
1865
|
+
const truncated = warnings.length - MAX_WARNINGS;
|
|
1866
|
+
warnings.splice(MAX_WARNINGS, truncated, `... and ${truncated} more warning(s) suppressed`);
|
|
1867
|
+
}
|
|
1547
1868
|
const { score, grade, overall, summary, categories } = computeScore(tests);
|
|
1548
1869
|
const badge = generateBadge(url);
|
|
1549
1870
|
return {
|
|
@@ -1574,5 +1895,8 @@ export {
|
|
|
1574
1895
|
computeGrade,
|
|
1575
1896
|
computeScore,
|
|
1576
1897
|
TEST_DEFINITIONS,
|
|
1898
|
+
SPEC_VERSION,
|
|
1899
|
+
SPEC_BASE,
|
|
1900
|
+
parseSSEResponse,
|
|
1577
1901
|
runComplianceSuite
|
|
1578
1902
|
};
|