@yawlabs/mcp-compliance 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +78 -13
- package/dist/{chunk-U66YZGE5.js → chunk-Z7VLPYIO.js} +429 -105
- package/dist/index.js +582 -134
- package/dist/mcp/server.d.ts +11 -1
- package/dist/mcp/server.js +33 -17
- package/dist/runner.d.ts +13 -2
- package/dist/runner.js +7 -1
- package/package.json +13 -5
package/dist/index.js
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
+
import { createRequire as createRequire3 } from "module";
|
|
5
|
+
import chalk2 from "chalk";
|
|
6
|
+
import { Command } from "commander";
|
|
7
|
+
|
|
8
|
+
// src/mcp/server.ts
|
|
4
9
|
import { createRequire as createRequire2 } from "module";
|
|
5
10
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
6
11
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
7
|
-
import chalk2 from "chalk";
|
|
8
|
-
import { Command } from "commander";
|
|
9
12
|
|
|
10
13
|
// src/mcp/tools.ts
|
|
11
14
|
import { z } from "zod";
|
|
@@ -15,16 +18,14 @@ import { createRequire } from "module";
|
|
|
15
18
|
import { request } from "undici";
|
|
16
19
|
|
|
17
20
|
// src/badge.ts
|
|
21
|
+
import { createHash } from "crypto";
|
|
22
|
+
function urlHash(url) {
|
|
23
|
+
return createHash("sha256").update(url).digest("hex").slice(0, 12);
|
|
24
|
+
}
|
|
18
25
|
function generateBadge(url) {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
} catch {
|
|
23
|
-
parsed = new URL("https://unknown");
|
|
24
|
-
}
|
|
25
|
-
const encoded = encodeURIComponent(parsed.href);
|
|
26
|
-
const imageUrl = `https://mcp.hosting/api/compliance/${encoded}/badge`;
|
|
27
|
-
const reportUrl = `https://mcp.hosting/compliance/${encoded}`;
|
|
26
|
+
const hash = urlHash(url);
|
|
27
|
+
const imageUrl = `https://mcp.hosting/api/compliance/${hash}/badge`;
|
|
28
|
+
const reportUrl = `https://mcp.hosting/compliance/${hash}`;
|
|
28
29
|
return {
|
|
29
30
|
imageUrl,
|
|
30
31
|
reportUrl,
|
|
@@ -70,14 +71,15 @@ function computeScore(tests) {
|
|
|
70
71
|
|
|
71
72
|
// src/types.ts
|
|
72
73
|
var TEST_DEFINITIONS = [
|
|
73
|
-
// ── Transport (
|
|
74
|
+
// ── Transport (10 tests) ─────────────────────────────────────────
|
|
74
75
|
{
|
|
75
76
|
id: "transport-post",
|
|
76
77
|
name: "HTTP POST accepted",
|
|
77
78
|
category: "transport",
|
|
78
79
|
required: true,
|
|
79
80
|
specRef: "basic/transports#streamable-http",
|
|
80
|
-
description: "Verifies the server accepts HTTP POST requests and returns a 2xx status code. This is the fundamental transport requirement for Streamable HTTP MCP servers."
|
|
81
|
+
description: "Verifies the server accepts HTTP POST requests and returns a 2xx status code. This is the fundamental transport requirement for Streamable HTTP MCP servers.",
|
|
82
|
+
recommendation: "Ensure your server listens for POST requests on the MCP endpoint. If you see 401/403, pass --auth with a valid token. Check that the URL is correct and the server is running."
|
|
81
83
|
},
|
|
82
84
|
{
|
|
83
85
|
id: "transport-content-type",
|
|
@@ -85,7 +87,8 @@ var TEST_DEFINITIONS = [
|
|
|
85
87
|
category: "transport",
|
|
86
88
|
required: true,
|
|
87
89
|
specRef: "basic/transports#streamable-http",
|
|
88
|
-
description: "Checks that the server responds with Content-Type application/json or text/event-stream. MCP servers must use one of these two content types."
|
|
90
|
+
description: "Checks that the server responds with Content-Type application/json or text/event-stream. MCP servers must use one of these two content types.",
|
|
91
|
+
recommendation: 'Set the Content-Type response header to "application/json" for synchronous responses or "text/event-stream" for streaming. Do not use text/html or other types.'
|
|
89
92
|
},
|
|
90
93
|
{
|
|
91
94
|
id: "transport-notification-202",
|
|
@@ -93,7 +96,8 @@ var TEST_DEFINITIONS = [
|
|
|
93
96
|
category: "transport",
|
|
94
97
|
required: false,
|
|
95
98
|
specRef: "basic/transports#streamable-http",
|
|
96
|
-
description: "Verifies that sending a JSON-RPC notification (no id field) returns HTTP 202 Accepted with no body. Per spec, servers MUST return 202 for notifications."
|
|
99
|
+
description: "Verifies that sending a JSON-RPC notification (no id field) returns HTTP 202 Accepted with no body. Per spec, servers MUST return 202 for notifications.",
|
|
100
|
+
recommendation: "Detect JSON-RPC messages without an id field and return HTTP 202 with an empty body. Do not attempt to send a JSON-RPC response for notifications."
|
|
97
101
|
},
|
|
98
102
|
{
|
|
99
103
|
id: "transport-session-id",
|
|
@@ -101,7 +105,8 @@ var TEST_DEFINITIONS = [
|
|
|
101
105
|
category: "transport",
|
|
102
106
|
required: false,
|
|
103
107
|
specRef: "basic/transports#streamable-http",
|
|
104
|
-
description: "Tests that the server returns HTTP 400 when MCP-Session-Id header is missing on requests after initialization (when the server issued a session ID)."
|
|
108
|
+
description: "Tests that the server returns HTTP 400 when MCP-Session-Id header is missing on requests after initialization (when the server issued a session ID).",
|
|
109
|
+
recommendation: "If your server issues an MCP-Session-Id header in the initialize response, reject subsequent requests that omit this header with HTTP 400."
|
|
105
110
|
},
|
|
106
111
|
{
|
|
107
112
|
id: "transport-get",
|
|
@@ -109,7 +114,8 @@ var TEST_DEFINITIONS = [
|
|
|
109
114
|
category: "transport",
|
|
110
115
|
required: false,
|
|
111
116
|
specRef: "basic/transports#streamable-http",
|
|
112
|
-
description: "Tests the GET endpoint for server-initiated messages. Server should return text/event-stream or 405 Method Not Allowed."
|
|
117
|
+
description: "Tests the GET endpoint for server-initiated messages. Server should return text/event-stream or 405 Method Not Allowed.",
|
|
118
|
+
recommendation: "If your server supports server-initiated messages, handle GET with text/event-stream. Otherwise, return 405 Method Not Allowed."
|
|
113
119
|
},
|
|
114
120
|
{
|
|
115
121
|
id: "transport-delete",
|
|
@@ -117,7 +123,8 @@ var TEST_DEFINITIONS = [
|
|
|
117
123
|
category: "transport",
|
|
118
124
|
required: false,
|
|
119
125
|
specRef: "basic/transports#streamable-http",
|
|
120
|
-
description: "Tests the DELETE endpoint for session termination. Server should accept the request or return 405 Method Not Allowed."
|
|
126
|
+
description: "Tests the DELETE endpoint for session termination. Server should accept the request or return 405 Method Not Allowed.",
|
|
127
|
+
recommendation: "Handle DELETE requests for session cleanup, or return 405 if session termination is not supported. Do not return 500."
|
|
121
128
|
},
|
|
122
129
|
{
|
|
123
130
|
id: "transport-batch-reject",
|
|
@@ -125,16 +132,45 @@ var TEST_DEFINITIONS = [
|
|
|
125
132
|
category: "transport",
|
|
126
133
|
required: true,
|
|
127
134
|
specRef: "basic/transports#streamable-http",
|
|
128
|
-
description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests."
|
|
135
|
+
description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests.",
|
|
136
|
+
recommendation: "Check if the parsed JSON body is an array. If so, return a JSON-RPC error or HTTP 400. Do not process batch requests \u2014 MCP explicitly forbids them."
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
id: "transport-content-type-init",
|
|
140
|
+
name: "Initialize response has valid content type",
|
|
141
|
+
category: "transport",
|
|
142
|
+
required: false,
|
|
143
|
+
specRef: "basic/transports#streamable-http",
|
|
144
|
+
description: "Validates that the initialize response uses application/json or text/event-stream content type. Some servers return other types for the handshake.",
|
|
145
|
+
recommendation: 'Ensure the initialize response uses Content-Type "application/json" or "text/event-stream". Do not return text/html or other types for JSON-RPC responses.'
|
|
129
146
|
},
|
|
130
|
-
|
|
147
|
+
{
|
|
148
|
+
id: "transport-get-stream",
|
|
149
|
+
name: "GET with session returns SSE or 405",
|
|
150
|
+
category: "transport",
|
|
151
|
+
required: false,
|
|
152
|
+
specRef: "basic/transports#streamable-http",
|
|
153
|
+
description: "Tests the GET endpoint with an active session ID for server-initiated messages. After initialization, the server should either return an SSE stream or 405.",
|
|
154
|
+
recommendation: "If your server supports server-initiated messages, return text/event-stream on GET with a valid session ID. Otherwise, return 405 Method Not Allowed."
|
|
155
|
+
},
|
|
156
|
+
{
|
|
157
|
+
id: "transport-concurrent",
|
|
158
|
+
name: "Handles concurrent requests",
|
|
159
|
+
category: "transport",
|
|
160
|
+
required: false,
|
|
161
|
+
specRef: "basic/transports#streamable-http",
|
|
162
|
+
description: "Sends multiple JSON-RPC requests in parallel and verifies the server responds to all with correct matching IDs. Tests that the server can handle concurrent connections.",
|
|
163
|
+
recommendation: "Ensure your server can handle multiple simultaneous requests. Each response must include the correct id matching the request. Use async handlers or connection pooling."
|
|
164
|
+
},
|
|
165
|
+
// ── Lifecycle (12 tests) ─────────────────────────────────────────
|
|
131
166
|
{
|
|
132
167
|
id: "lifecycle-init",
|
|
133
168
|
name: "Initialize handshake",
|
|
134
169
|
category: "lifecycle",
|
|
135
170
|
required: true,
|
|
136
171
|
specRef: "basic/lifecycle#initialization",
|
|
137
|
-
description: "Tests the initialize handshake by sending an initialize request with client capabilities. The server must return a result with protocolVersion."
|
|
172
|
+
description: "Tests the initialize handshake by sending an initialize request with client capabilities. The server must return a result with protocolVersion.",
|
|
173
|
+
recommendation: 'Implement the "initialize" method handler. Return a result object with at least protocolVersion, capabilities, and serverInfo fields.'
|
|
138
174
|
},
|
|
139
175
|
{
|
|
140
176
|
id: "lifecycle-proto-version",
|
|
@@ -142,7 +178,8 @@ var TEST_DEFINITIONS = [
|
|
|
142
178
|
category: "lifecycle",
|
|
143
179
|
required: true,
|
|
144
180
|
specRef: "basic/lifecycle#version-negotiation",
|
|
145
|
-
description: "Validates that the protocolVersion returned by the server matches the YYYY-MM-DD date format required by the spec."
|
|
181
|
+
description: "Validates that the protocolVersion returned by the server matches the YYYY-MM-DD date format required by the spec.",
|
|
182
|
+
recommendation: `Return protocolVersion as a YYYY-MM-DD string (e.g., "2025-11-25"). The server should negotiate based on the client's requested version.`
|
|
146
183
|
},
|
|
147
184
|
{
|
|
148
185
|
id: "lifecycle-server-info",
|
|
@@ -150,7 +187,8 @@ var TEST_DEFINITIONS = [
|
|
|
150
187
|
category: "lifecycle",
|
|
151
188
|
required: false,
|
|
152
189
|
specRef: "basic/lifecycle#initialization",
|
|
153
|
-
description: "Checks that the server includes a serverInfo object with at least a name field in its initialize response. While recommended, this is not strictly required."
|
|
190
|
+
description: "Checks that the server includes a serverInfo object with at least a name field in its initialize response. While recommended, this is not strictly required.",
|
|
191
|
+
recommendation: 'Add a serverInfo object to your initialize response: { name: "your-server", version: "1.0.0" }. This helps clients identify your server.'
|
|
154
192
|
},
|
|
155
193
|
{
|
|
156
194
|
id: "lifecycle-capabilities",
|
|
@@ -158,7 +196,8 @@ var TEST_DEFINITIONS = [
|
|
|
158
196
|
category: "lifecycle",
|
|
159
197
|
required: true,
|
|
160
198
|
specRef: "basic/lifecycle#capability-negotiation",
|
|
161
|
-
description: "Verifies the server returns a capabilities object in its initialize response. An empty object is valid (no optional features declared)."
|
|
199
|
+
description: "Verifies the server returns a capabilities object in its initialize response. An empty object is valid (no optional features declared).",
|
|
200
|
+
recommendation: "Include a capabilities object in your initialize response. Declare the features your server supports (tools, resources, prompts, logging, etc.). An empty object {} is valid."
|
|
162
201
|
},
|
|
163
202
|
{
|
|
164
203
|
id: "lifecycle-jsonrpc",
|
|
@@ -166,7 +205,8 @@ var TEST_DEFINITIONS = [
|
|
|
166
205
|
category: "lifecycle",
|
|
167
206
|
required: true,
|
|
168
207
|
specRef: "basic",
|
|
169
|
-
description: 'Validates that the initialize response is a proper JSON-RPC 2.0 message with jsonrpc="2.0", an id field, and either a result or error field.'
|
|
208
|
+
description: 'Validates that the initialize response is a proper JSON-RPC 2.0 message with jsonrpc="2.0", an id field, and either a result or error field.',
|
|
209
|
+
recommendation: 'Ensure every response includes jsonrpc: "2.0", the matching id from the request, and either a result or error field. Never omit the jsonrpc field.'
|
|
170
210
|
},
|
|
171
211
|
{
|
|
172
212
|
id: "lifecycle-ping",
|
|
@@ -174,7 +214,8 @@ var TEST_DEFINITIONS = [
|
|
|
174
214
|
category: "lifecycle",
|
|
175
215
|
required: true,
|
|
176
216
|
specRef: "basic/utilities#ping",
|
|
177
|
-
description: "Tests that the server responds to the ping method with an empty result object. This is a required utility method."
|
|
217
|
+
description: "Tests that the server responds to the ping method with an empty result object. This is a required utility method.",
|
|
218
|
+
recommendation: 'Implement a "ping" method handler that returns an empty result object {}. This is required by the MCP spec for keepalive and connectivity checking.'
|
|
178
219
|
},
|
|
179
220
|
{
|
|
180
221
|
id: "lifecycle-instructions",
|
|
@@ -182,7 +223,8 @@ var TEST_DEFINITIONS = [
|
|
|
182
223
|
category: "lifecycle",
|
|
183
224
|
required: false,
|
|
184
225
|
specRef: "basic/lifecycle#initialization",
|
|
185
|
-
description: "If the server includes an instructions field in the initialize response, validates it is a string. Instructions provide guidance for how the client should interact with the server."
|
|
226
|
+
description: "If the server includes an instructions field in the initialize response, validates it is a string. Instructions provide guidance for how the client should interact with the server.",
|
|
227
|
+
recommendation: "If you include an instructions field in the initialize response, ensure it is a string. Remove the field or fix the type if it is not a string."
|
|
186
228
|
},
|
|
187
229
|
{
|
|
188
230
|
id: "lifecycle-id-match",
|
|
@@ -190,7 +232,8 @@ var TEST_DEFINITIONS = [
|
|
|
190
232
|
category: "lifecycle",
|
|
191
233
|
required: true,
|
|
192
234
|
specRef: "basic",
|
|
193
|
-
description: "Verifies that the JSON-RPC response id matches the request id sent by the client. This is a fundamental JSON-RPC 2.0 requirement."
|
|
235
|
+
description: "Verifies that the JSON-RPC response id matches the request id sent by the client. This is a fundamental JSON-RPC 2.0 requirement.",
|
|
236
|
+
recommendation: "Copy the id field from the request into the response. This is a core JSON-RPC 2.0 requirement. Check that your framework does not modify or discard the request ID."
|
|
194
237
|
},
|
|
195
238
|
{
|
|
196
239
|
id: "lifecycle-logging",
|
|
@@ -198,7 +241,8 @@ var TEST_DEFINITIONS = [
|
|
|
198
241
|
category: "lifecycle",
|
|
199
242
|
required: false,
|
|
200
243
|
specRef: "server/utilities#logging",
|
|
201
|
-
description: "If the server declares logging capability, tests that logging/setLevel method is accepted with a valid log level."
|
|
244
|
+
description: "If the server declares logging capability, tests that logging/setLevel method is accepted with a valid log level.",
|
|
245
|
+
recommendation: 'If you declare logging in capabilities, implement the "logging/setLevel" handler. Accept standard log levels: debug, info, notice, warning, error, critical, alert, emergency.'
|
|
202
246
|
},
|
|
203
247
|
{
|
|
204
248
|
id: "lifecycle-completions",
|
|
@@ -206,7 +250,26 @@ var TEST_DEFINITIONS = [
|
|
|
206
250
|
category: "lifecycle",
|
|
207
251
|
required: false,
|
|
208
252
|
specRef: "server/utilities#completion",
|
|
209
|
-
description: "If the server declares completions capability, tests that the completion/complete method is accepted."
|
|
253
|
+
description: "If the server declares completions capability, tests that the completion/complete method is accepted.",
|
|
254
|
+
recommendation: 'If you declare completions in capabilities, implement the "completion/complete" handler. Return a completion object with a values array, even if empty.'
|
|
255
|
+
},
|
|
256
|
+
{
|
|
257
|
+
id: "lifecycle-cancellation",
|
|
258
|
+
name: "Handles cancellation notifications",
|
|
259
|
+
category: "lifecycle",
|
|
260
|
+
required: false,
|
|
261
|
+
specRef: "basic/utilities#cancellation",
|
|
262
|
+
description: "Tests that the server accepts notifications/cancelled without error. Servers should gracefully handle cancellation of unknown or completed requests.",
|
|
263
|
+
recommendation: "Accept notifications/cancelled and stop any in-progress work for the referenced requestId. If the request is unknown or already complete, silently ignore the cancellation."
|
|
264
|
+
},
|
|
265
|
+
{
|
|
266
|
+
id: "lifecycle-progress",
|
|
267
|
+
name: "Accepts progress notifications",
|
|
268
|
+
category: "lifecycle",
|
|
269
|
+
required: false,
|
|
270
|
+
specRef: "basic/utilities#progress",
|
|
271
|
+
description: "Tests that the server accepts notifications/progress without error. Servers should handle progress notifications for request tracking.",
|
|
272
|
+
recommendation: "Accept notifications/progress with progressToken, progress, and optional total fields. Ignore notifications for unknown progress tokens."
|
|
210
273
|
},
|
|
211
274
|
// ── Tools (4 tests) ──────────────────────────────────────────────
|
|
212
275
|
{
|
|
@@ -215,7 +278,8 @@ var TEST_DEFINITIONS = [
|
|
|
215
278
|
category: "tools",
|
|
216
279
|
required: false,
|
|
217
280
|
specRef: "server/tools#listing-tools",
|
|
218
|
-
description: "Calls tools/list and validates it returns an array of tool definitions. Required if the server declares tools capability."
|
|
281
|
+
description: "Calls tools/list and validates it returns an array of tool definitions. Required if the server declares tools capability.",
|
|
282
|
+
recommendation: "Implement the tools/list handler to return { tools: [...] } with an array of tool definition objects. Each tool needs at least a name and inputSchema."
|
|
219
283
|
},
|
|
220
284
|
{
|
|
221
285
|
id: "tools-call",
|
|
@@ -223,7 +287,8 @@ var TEST_DEFINITIONS = [
|
|
|
223
287
|
category: "tools",
|
|
224
288
|
required: false,
|
|
225
289
|
specRef: "server/tools#calling-tools",
|
|
226
|
-
description: "Calls the first tool with empty arguments and verifies the response format. Accepts both successful results and InvalidParams errors."
|
|
290
|
+
description: "Calls the first tool with empty arguments and verifies the response format. Accepts both successful results and InvalidParams errors.",
|
|
291
|
+
recommendation: "Ensure tools/call returns { content: [...] } with an array of content objects, each having a type field. Return isError: true for tool execution errors."
|
|
227
292
|
},
|
|
228
293
|
{
|
|
229
294
|
id: "tools-pagination",
|
|
@@ -231,7 +296,8 @@ var TEST_DEFINITIONS = [
|
|
|
231
296
|
category: "tools",
|
|
232
297
|
required: false,
|
|
233
298
|
specRef: "server/tools#listing-tools",
|
|
234
|
-
description: "Tests cursor-based pagination on tools/list. Validates nextCursor is a string if present and that fetching the next page returns a valid response."
|
|
299
|
+
description: "Tests cursor-based pagination on tools/list. Validates nextCursor is a string if present and that fetching the next page returns a valid response.",
|
|
300
|
+
recommendation: "If your server has many tools, include a nextCursor string in the response. Ensure passing this cursor back in a subsequent request returns the next page."
|
|
235
301
|
},
|
|
236
302
|
{
|
|
237
303
|
id: "tools-content-types",
|
|
@@ -239,7 +305,8 @@ var TEST_DEFINITIONS = [
|
|
|
239
305
|
category: "tools",
|
|
240
306
|
required: false,
|
|
241
307
|
specRef: "server/tools#calling-tools",
|
|
242
|
-
description: "Validates that content items returned by tools/call have a recognized type field (text, image, audio, resource, resource_link)."
|
|
308
|
+
description: "Validates that content items returned by tools/call have a recognized type field (text, image, audio, resource, resource_link).",
|
|
309
|
+
recommendation: 'Every content item returned by tools/call must have a type field set to one of: "text", "image", "audio", "resource", or "resource_link". Check for typos or missing type fields.'
|
|
243
310
|
},
|
|
244
311
|
// ── Resources (5 tests) ──────────────────────────────────────────
|
|
245
312
|
{
|
|
@@ -248,7 +315,8 @@ var TEST_DEFINITIONS = [
|
|
|
248
315
|
category: "resources",
|
|
249
316
|
required: false,
|
|
250
317
|
specRef: "server/resources#listing-resources",
|
|
251
|
-
description: "Calls resources/list and validates it returns an array. Required if the server declares resources capability."
|
|
318
|
+
description: "Calls resources/list and validates it returns an array. Required if the server declares resources capability.",
|
|
319
|
+
recommendation: "Implement resources/list to return { resources: [...] } with an array of resource objects. Each resource needs at least a uri and name."
|
|
252
320
|
},
|
|
253
321
|
{
|
|
254
322
|
id: "resources-read",
|
|
@@ -256,7 +324,8 @@ var TEST_DEFINITIONS = [
|
|
|
256
324
|
category: "resources",
|
|
257
325
|
required: false,
|
|
258
326
|
specRef: "server/resources#reading-resources",
|
|
259
|
-
description: "Reads the first resource and validates the response contains a contents array with proper uri and text/blob fields."
|
|
327
|
+
description: "Reads the first resource and validates the response contains a contents array with proper uri and text/blob fields.",
|
|
328
|
+
recommendation: "Implement resources/read to return { contents: [...] } where each item has a uri and either a text or blob field. Ensure the uri matches the requested resource."
|
|
260
329
|
},
|
|
261
330
|
{
|
|
262
331
|
id: "resources-templates",
|
|
@@ -264,7 +333,8 @@ var TEST_DEFINITIONS = [
|
|
|
264
333
|
category: "resources",
|
|
265
334
|
required: false,
|
|
266
335
|
specRef: "server/resources#resource-templates",
|
|
267
|
-
description: "Tests the resource templates endpoint. Accepts Method not found (-32601) since templates are optional."
|
|
336
|
+
description: "Tests the resource templates endpoint. Accepts Method not found (-32601) since templates are optional.",
|
|
337
|
+
recommendation: "If your server supports resource templates, implement resources/templates/list returning { resourceTemplates: [...] }. Otherwise, return error code -32601."
|
|
268
338
|
},
|
|
269
339
|
{
|
|
270
340
|
id: "resources-pagination",
|
|
@@ -272,7 +342,8 @@ var TEST_DEFINITIONS = [
|
|
|
272
342
|
category: "resources",
|
|
273
343
|
required: false,
|
|
274
344
|
specRef: "server/resources#listing-resources",
|
|
275
|
-
description: "Tests cursor-based pagination on resources/list. Validates nextCursor is a string if present and that fetching the next page works."
|
|
345
|
+
description: "Tests cursor-based pagination on resources/list. Validates nextCursor is a string if present and that fetching the next page works.",
|
|
346
|
+
recommendation: "If you return nextCursor in resources/list, ensure it is a string and that passing it back as cursor in the next request returns valid results."
|
|
276
347
|
},
|
|
277
348
|
{
|
|
278
349
|
id: "resources-subscribe",
|
|
@@ -280,7 +351,8 @@ var TEST_DEFINITIONS = [
|
|
|
280
351
|
category: "resources",
|
|
281
352
|
required: false,
|
|
282
353
|
specRef: "server/resources#subscriptions",
|
|
283
|
-
description: "If the server declares resources.subscribe capability, tests that resources/subscribe and resources/unsubscribe methods are accepted."
|
|
354
|
+
description: "If the server declares resources.subscribe capability, tests that resources/subscribe and resources/unsubscribe methods are accepted.",
|
|
355
|
+
recommendation: "If you declare resources.subscribe capability, implement both resources/subscribe and resources/unsubscribe handlers. Both should accept a uri parameter."
|
|
284
356
|
},
|
|
285
357
|
// ── Prompts (3 tests) ────────────────────────────────────────────
|
|
286
358
|
{
|
|
@@ -289,7 +361,8 @@ var TEST_DEFINITIONS = [
|
|
|
289
361
|
category: "prompts",
|
|
290
362
|
required: false,
|
|
291
363
|
specRef: "server/prompts#listing-prompts",
|
|
292
|
-
description: "Calls prompts/list and validates it returns an array. Required if the server declares prompts capability."
|
|
364
|
+
description: "Calls prompts/list and validates it returns an array. Required if the server declares prompts capability.",
|
|
365
|
+
recommendation: "Implement prompts/list to return { prompts: [...] } with an array of prompt objects. Each prompt needs at least a name field."
|
|
293
366
|
},
|
|
294
367
|
{
|
|
295
368
|
id: "prompts-get",
|
|
@@ -297,7 +370,8 @@ var TEST_DEFINITIONS = [
|
|
|
297
370
|
category: "prompts",
|
|
298
371
|
required: false,
|
|
299
372
|
specRef: "server/prompts#getting-a-prompt",
|
|
300
|
-
description: "Gets the first prompt and validates the response contains a messages array with proper role and content fields."
|
|
373
|
+
description: "Gets the first prompt and validates the response contains a messages array with proper role and content fields.",
|
|
374
|
+
recommendation: 'Implement prompts/get to return { messages: [...] } where each message has a role ("user" or "assistant") and a content field.'
|
|
301
375
|
},
|
|
302
376
|
{
|
|
303
377
|
id: "prompts-pagination",
|
|
@@ -305,7 +379,8 @@ var TEST_DEFINITIONS = [
|
|
|
305
379
|
category: "prompts",
|
|
306
380
|
required: false,
|
|
307
381
|
specRef: "server/prompts#listing-prompts",
|
|
308
|
-
description: "Tests cursor-based pagination on prompts/list. Validates nextCursor is a string if present and that fetching the next page works."
|
|
382
|
+
description: "Tests cursor-based pagination on prompts/list. Validates nextCursor is a string if present and that fetching the next page works.",
|
|
383
|
+
recommendation: "If you return nextCursor in prompts/list, ensure it is a string and that passing it back as cursor in the next request returns valid results."
|
|
309
384
|
},
|
|
310
385
|
// ── Error Handling (8 tests) ─────────────────────────────────────
|
|
311
386
|
{
|
|
@@ -314,7 +389,8 @@ var TEST_DEFINITIONS = [
|
|
|
314
389
|
category: "errors",
|
|
315
390
|
required: true,
|
|
316
391
|
specRef: "basic",
|
|
317
|
-
description: "Sends an unknown method and verifies the server returns a JSON-RPC error. The spec requires error code -32601 (Method not found)."
|
|
392
|
+
description: "Sends an unknown method and verifies the server returns a JSON-RPC error. The spec requires error code -32601 (Method not found).",
|
|
393
|
+
recommendation: "Return a JSON-RPC error with code -32601 (Method not found) for any unrecognized method name. Do not silently ignore unknown methods."
|
|
318
394
|
},
|
|
319
395
|
{
|
|
320
396
|
id: "error-method-code",
|
|
@@ -322,7 +398,8 @@ var TEST_DEFINITIONS = [
|
|
|
322
398
|
category: "errors",
|
|
323
399
|
required: false,
|
|
324
400
|
specRef: "basic",
|
|
325
|
-
description: "Checks the error code is specifically -32601 (Method not found) for unknown methods, as required by JSON-RPC 2.0."
|
|
401
|
+
description: "Checks the error code is specifically -32601 (Method not found) for unknown methods, as required by JSON-RPC 2.0.",
|
|
402
|
+
recommendation: "Use exactly error code -32601 for unknown methods. Do not use generic error codes like -32000. This is required by JSON-RPC 2.0."
|
|
326
403
|
},
|
|
327
404
|
{
|
|
328
405
|
id: "error-invalid-jsonrpc",
|
|
@@ -330,7 +407,8 @@ var TEST_DEFINITIONS = [
|
|
|
330
407
|
category: "errors",
|
|
331
408
|
required: true,
|
|
332
409
|
specRef: "basic",
|
|
333
|
-
description: "Sends a malformed JSON-RPC message (missing required fields) and verifies the server returns an error or 4xx status."
|
|
410
|
+
description: "Sends a malformed JSON-RPC message (missing required fields) and verifies the server returns an error or 4xx status.",
|
|
411
|
+
recommendation: "Validate incoming JSON-RPC messages for required fields (jsonrpc, method). Return error code -32600 (Invalid Request) or HTTP 400 for malformed messages."
|
|
334
412
|
},
|
|
335
413
|
{
|
|
336
414
|
id: "error-invalid-json",
|
|
@@ -338,7 +416,8 @@ var TEST_DEFINITIONS = [
|
|
|
338
416
|
category: "errors",
|
|
339
417
|
required: false,
|
|
340
418
|
specRef: "basic",
|
|
341
|
-
description: "Sends invalid JSON and verifies the server returns a parse error (-32700) or 4xx status code."
|
|
419
|
+
description: "Sends invalid JSON and verifies the server returns a parse error (-32700) or 4xx status code.",
|
|
420
|
+
recommendation: "Catch JSON parse errors and return error code -32700 (Parse error) with a descriptive message. Do not return 500 for malformed input."
|
|
342
421
|
},
|
|
343
422
|
{
|
|
344
423
|
id: "error-missing-params",
|
|
@@ -346,7 +425,8 @@ var TEST_DEFINITIONS = [
|
|
|
346
425
|
category: "errors",
|
|
347
426
|
required: false,
|
|
348
427
|
specRef: "server/tools#error-handling",
|
|
349
|
-
description: "Calls tools/call with an empty params object (missing required name field) and verifies an error is returned."
|
|
428
|
+
description: "Calls tools/call with an empty params object (missing required name field) and verifies an error is returned.",
|
|
429
|
+
recommendation: "Validate tools/call params and return error code -32602 (Invalid params) when the required name field is missing."
|
|
350
430
|
},
|
|
351
431
|
{
|
|
352
432
|
id: "error-parse-code",
|
|
@@ -354,7 +434,8 @@ var TEST_DEFINITIONS = [
|
|
|
354
434
|
category: "errors",
|
|
355
435
|
required: false,
|
|
356
436
|
specRef: "basic",
|
|
357
|
-
description: "Checks that the server returns the specific JSON-RPC error code -32700 (Parse error) when receiving invalid JSON, as required by the JSON-RPC 2.0 specification."
|
|
437
|
+
description: "Checks that the server returns the specific JSON-RPC error code -32700 (Parse error) when receiving invalid JSON, as required by the JSON-RPC 2.0 specification.",
|
|
438
|
+
recommendation: "Return exactly error code -32700 for JSON parse failures. Most JSON-RPC frameworks handle this automatically \u2014 check yours does not override the code."
|
|
358
439
|
},
|
|
359
440
|
{
|
|
360
441
|
id: "error-invalid-request-code",
|
|
@@ -362,7 +443,8 @@ var TEST_DEFINITIONS = [
|
|
|
362
443
|
category: "errors",
|
|
363
444
|
required: false,
|
|
364
445
|
specRef: "basic",
|
|
365
|
-
description: "Checks that the server returns the specific JSON-RPC error code -32600 (Invalid Request) for malformed JSON-RPC messages missing required fields."
|
|
446
|
+
description: "Checks that the server returns the specific JSON-RPC error code -32600 (Invalid Request) for malformed JSON-RPC messages missing required fields.",
|
|
447
|
+
recommendation: "Return exactly error code -32600 for structurally invalid JSON-RPC messages (e.g., missing method field). Check your JSON-RPC middleware configuration."
|
|
366
448
|
},
|
|
367
449
|
{
|
|
368
450
|
id: "tools-call-unknown",
|
|
@@ -370,7 +452,8 @@ var TEST_DEFINITIONS = [
|
|
|
370
452
|
category: "errors",
|
|
371
453
|
required: false,
|
|
372
454
|
specRef: "server/tools#error-handling",
|
|
373
|
-
description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response."
|
|
455
|
+
description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response.",
|
|
456
|
+
recommendation: "Return a JSON-RPC error or set isError: true when tools/call receives an unrecognized tool name. Do not return an empty success response."
|
|
374
457
|
},
|
|
375
458
|
// ── Schema Validation (6 tests) ──────────────────────────────────
|
|
376
459
|
{
|
|
@@ -379,7 +462,8 @@ var TEST_DEFINITIONS = [
|
|
|
379
462
|
category: "schema",
|
|
380
463
|
required: false,
|
|
381
464
|
specRef: "server/tools#data-types",
|
|
382
|
-
description: 'Validates every tool has a valid name (1-128 chars, alphanumeric/underscore/hyphen/dot) and a required inputSchema of type "object".'
|
|
465
|
+
description: 'Validates every tool has a valid name (1-128 chars, alphanumeric/underscore/hyphen/dot) and a required inputSchema of type "object".',
|
|
466
|
+
recommendation: 'Ensure every tool has a name (1-128 chars, [A-Za-z0-9_.-]) and an inputSchema with type: "object". Add descriptions to tools for better AI assistant integration.'
|
|
383
467
|
},
|
|
384
468
|
{
|
|
385
469
|
id: "tools-annotations",
|
|
@@ -387,7 +471,8 @@ var TEST_DEFINITIONS = [
|
|
|
387
471
|
category: "schema",
|
|
388
472
|
required: false,
|
|
389
473
|
specRef: "server/tools#annotations",
|
|
390
|
-
description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans; title should be a string."
|
|
474
|
+
description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans; title should be a string.",
|
|
475
|
+
recommendation: "If you include annotations on tools, ensure readOnlyHint, destructiveHint, idempotentHint, and openWorldHint are booleans. Title must be a string."
|
|
391
476
|
},
|
|
392
477
|
{
|
|
393
478
|
id: "tools-title-field",
|
|
@@ -395,7 +480,8 @@ var TEST_DEFINITIONS = [
|
|
|
395
480
|
category: "schema",
|
|
396
481
|
required: false,
|
|
397
482
|
specRef: "server/tools#data-types",
|
|
398
|
-
description: "Checks if tools include the optional title field for human-readable display names. Added in spec version 2025-11-25."
|
|
483
|
+
description: "Checks if tools include the optional title field for human-readable display names. Added in spec version 2025-11-25.",
|
|
484
|
+
recommendation: "Add a title field (human-readable string) to each tool definition. This helps MCP clients display your tools in a user-friendly way."
|
|
399
485
|
},
|
|
400
486
|
{
|
|
401
487
|
id: "tools-output-schema",
|
|
@@ -403,7 +489,8 @@ var TEST_DEFINITIONS = [
|
|
|
403
489
|
category: "schema",
|
|
404
490
|
required: false,
|
|
405
491
|
specRef: "server/tools#structured-content",
|
|
406
|
-
description: 'If tools declare an outputSchema, validates it is a valid JSON Schema object with type "object". Used for structured output validation.'
|
|
492
|
+
description: 'If tools declare an outputSchema, validates it is a valid JSON Schema object with type "object". Used for structured output validation.',
|
|
493
|
+
recommendation: 'If you declare outputSchema on a tool, ensure it is a valid JSON Schema object with type: "object". Remove outputSchema if you do not need structured output.'
|
|
407
494
|
},
|
|
408
495
|
{
|
|
409
496
|
id: "prompts-schema",
|
|
@@ -411,7 +498,8 @@ var TEST_DEFINITIONS = [
|
|
|
411
498
|
category: "schema",
|
|
412
499
|
required: false,
|
|
413
500
|
specRef: "server/prompts#data-types",
|
|
414
|
-
description: "Validates every prompt has a name and that any arguments array contains items with name fields."
|
|
501
|
+
description: "Validates every prompt has a name and that any arguments array contains items with name fields.",
|
|
502
|
+
recommendation: "Ensure every prompt has a name field. If the prompt has arguments, each argument object must include a name field."
|
|
415
503
|
},
|
|
416
504
|
{
|
|
417
505
|
id: "resources-schema",
|
|
@@ -419,7 +507,8 @@ var TEST_DEFINITIONS = [
|
|
|
419
507
|
category: "schema",
|
|
420
508
|
required: false,
|
|
421
509
|
specRef: "server/resources#data-types",
|
|
422
|
-
description: "Validates every resource has a valid URI (parseable as a URL) and a name field."
|
|
510
|
+
description: "Validates every resource has a valid URI (parseable as a URL) and a name field.",
|
|
511
|
+
recommendation: "Ensure every resource has a valid, parseable URI and a name field. Add description and mimeType for better client integration."
|
|
423
512
|
}
|
|
424
513
|
];
|
|
425
514
|
|
|
@@ -523,19 +612,39 @@ async function mcpNotification(backendUrl, method, params, extraHeaders, timeout
|
|
|
523
612
|
return { statusCode: res.statusCode, headers: responseHeaders };
|
|
524
613
|
}
|
|
525
614
|
async function runComplianceSuite(url, options = {}) {
|
|
526
|
-
let parsed;
|
|
527
615
|
try {
|
|
528
|
-
parsed = new URL(url);
|
|
616
|
+
const parsed = new URL(url);
|
|
529
617
|
if (!["http:", "https:"].includes(parsed.protocol)) {
|
|
530
618
|
throw new Error("Only HTTP and HTTPS URLs are supported");
|
|
531
619
|
}
|
|
532
620
|
} catch (e) {
|
|
533
|
-
if (e.message.includes("Only HTTP")) throw e;
|
|
621
|
+
if (e instanceof Error && e.message.includes("Only HTTP")) throw e;
|
|
534
622
|
throw new Error(`Invalid URL: ${url}`);
|
|
535
623
|
}
|
|
536
624
|
const backendUrl = url;
|
|
625
|
+
let serverReachable = true;
|
|
626
|
+
try {
|
|
627
|
+
const preflight = await request(backendUrl, {
|
|
628
|
+
method: "POST",
|
|
629
|
+
headers: {
|
|
630
|
+
"Content-Type": "application/json",
|
|
631
|
+
Accept: "application/json, text/event-stream",
|
|
632
|
+
...options.headers
|
|
633
|
+
},
|
|
634
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 0, method: "ping" }),
|
|
635
|
+
signal: AbortSignal.timeout(Math.min(options.timeout || 15e3, 1e4))
|
|
636
|
+
});
|
|
637
|
+
await preflight.body.text();
|
|
638
|
+
} catch {
|
|
639
|
+
serverReachable = false;
|
|
640
|
+
}
|
|
537
641
|
const tests = [];
|
|
538
642
|
const warnings = [];
|
|
643
|
+
if (!serverReachable) {
|
|
644
|
+
warnings.push(
|
|
645
|
+
`Server at ${url} is unreachable \u2014 all tests will fail. Check the URL and ensure the server is running.`
|
|
646
|
+
);
|
|
647
|
+
}
|
|
539
648
|
const nextId = createIdCounter(1e3);
|
|
540
649
|
const timeout = options.timeout || 15e3;
|
|
541
650
|
const retries = options.retries || 0;
|
|
@@ -580,7 +689,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
580
689
|
if (lastResult.passed) break;
|
|
581
690
|
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
|
|
582
691
|
} catch (err) {
|
|
583
|
-
|
|
692
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
693
|
+
lastResult = { passed: false, details: `Error: ${message}` };
|
|
584
694
|
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
|
|
585
695
|
}
|
|
586
696
|
}
|
|
@@ -609,10 +719,23 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
609
719
|
body: JSON.stringify({ jsonrpc: "2.0", id: 99901, method: "ping" }),
|
|
610
720
|
signal: AbortSignal.timeout(timeout)
|
|
611
721
|
});
|
|
612
|
-
await res.body.text();
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
722
|
+
const text = await res.body.text();
|
|
723
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
724
|
+
return { passed: true, details: `HTTP ${res.statusCode}` };
|
|
725
|
+
}
|
|
726
|
+
if (res.statusCode === 401 || res.statusCode === 403) {
|
|
727
|
+
return { passed: false, details: `HTTP ${res.statusCode} (auth required \u2014 pass --auth)` };
|
|
728
|
+
}
|
|
729
|
+
if (res.statusCode === 400) {
|
|
730
|
+
try {
|
|
731
|
+
const body = JSON.parse(text);
|
|
732
|
+
if (body?.error || body?.jsonrpc) {
|
|
733
|
+
return { passed: true, details: "HTTP 400 with JSON-RPC response (server requires initialization first)" };
|
|
734
|
+
}
|
|
735
|
+
} catch {
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
616
739
|
}
|
|
617
740
|
);
|
|
618
741
|
await test(
|
|
@@ -642,18 +765,29 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
642
765
|
false,
|
|
643
766
|
"basic/transports#streamable-http",
|
|
644
767
|
async () => {
|
|
768
|
+
const getHeaders = { Accept: "text/event-stream", ...buildHeaders() };
|
|
645
769
|
const res = await request(backendUrl, {
|
|
646
770
|
method: "GET",
|
|
647
|
-
headers:
|
|
771
|
+
headers: getHeaders,
|
|
648
772
|
signal: AbortSignal.timeout(timeout)
|
|
649
773
|
});
|
|
650
|
-
await res.body.text();
|
|
774
|
+
const body = await res.body.text();
|
|
651
775
|
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
652
776
|
if (res.statusCode === 405) {
|
|
653
777
|
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
654
778
|
}
|
|
655
779
|
if (ct.includes("text/event-stream")) {
|
|
656
|
-
|
|
780
|
+
if (body.trim().length > 0) {
|
|
781
|
+
const hasDataFields = body.includes("data:");
|
|
782
|
+
const hasEventFields = body.includes("event:");
|
|
783
|
+
if (!hasDataFields && !hasEventFields) {
|
|
784
|
+
return {
|
|
785
|
+
passed: false,
|
|
786
|
+
details: "Content-Type is text/event-stream but body has no SSE data: or event: fields"
|
|
787
|
+
};
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
return { passed: true, details: "Returns text/event-stream with valid SSE format" };
|
|
657
791
|
}
|
|
658
792
|
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
659
793
|
return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
|
|
@@ -661,31 +795,6 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
661
795
|
return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
|
|
662
796
|
}
|
|
663
797
|
);
|
|
664
|
-
await test(
|
|
665
|
-
"transport-delete",
|
|
666
|
-
"DELETE accepted or returns 405",
|
|
667
|
-
"transport",
|
|
668
|
-
false,
|
|
669
|
-
"basic/transports#streamable-http",
|
|
670
|
-
async () => {
|
|
671
|
-
const res = await request(backendUrl, {
|
|
672
|
-
method: "DELETE",
|
|
673
|
-
headers: { ...userHeaders },
|
|
674
|
-
signal: AbortSignal.timeout(timeout)
|
|
675
|
-
});
|
|
676
|
-
await res.body.text();
|
|
677
|
-
if (res.statusCode === 405) {
|
|
678
|
-
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
679
|
-
}
|
|
680
|
-
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
681
|
-
return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
|
|
682
|
-
}
|
|
683
|
-
if (res.statusCode === 400 || res.statusCode === 404) {
|
|
684
|
-
return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
|
|
685
|
-
}
|
|
686
|
-
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
687
|
-
}
|
|
688
|
-
);
|
|
689
798
|
await test(
|
|
690
799
|
"transport-batch-reject",
|
|
691
800
|
"Rejects JSON-RPC batch requests",
|
|
@@ -723,7 +832,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
723
832
|
try {
|
|
724
833
|
initRes = await rpc("initialize", {
|
|
725
834
|
protocolVersion: SPEC_VERSION,
|
|
726
|
-
capabilities: {
|
|
835
|
+
capabilities: {},
|
|
727
836
|
clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
|
|
728
837
|
});
|
|
729
838
|
const result = initRes?.body?.result;
|
|
@@ -736,7 +845,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
736
845
|
if (sid) sessionId = sid;
|
|
737
846
|
if (result.protocolVersion) negotiatedProtocolVersion = result.protocolVersion;
|
|
738
847
|
}
|
|
739
|
-
} catch
|
|
848
|
+
} catch {
|
|
740
849
|
}
|
|
741
850
|
try {
|
|
742
851
|
await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders(), timeout);
|
|
@@ -762,13 +871,13 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
762
871
|
true,
|
|
763
872
|
"basic/lifecycle#version-negotiation",
|
|
764
873
|
async () => {
|
|
765
|
-
const
|
|
766
|
-
if (!
|
|
767
|
-
const valid = /^\d{4}-\d{2}-\d{2}$/.test(
|
|
768
|
-
if (valid &&
|
|
769
|
-
warnings.push(`Server negotiated protocol version ${
|
|
874
|
+
const version3 = initRes?.body?.result?.protocolVersion;
|
|
875
|
+
if (!version3) return { passed: false, details: "No protocolVersion" };
|
|
876
|
+
const valid = /^\d{4}-\d{2}-\d{2}$/.test(version3);
|
|
877
|
+
if (valid && version3 !== SPEC_VERSION) {
|
|
878
|
+
warnings.push(`Server negotiated protocol version ${version3} (latest is ${SPEC_VERSION})`);
|
|
770
879
|
}
|
|
771
|
-
return { passed: valid, details: `Version: ${
|
|
880
|
+
return { passed: valid, details: `Version: ${version3}` };
|
|
772
881
|
}
|
|
773
882
|
);
|
|
774
883
|
await test(
|
|
@@ -855,7 +964,17 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
855
964
|
if (res.body?.error) {
|
|
856
965
|
return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
|
|
857
966
|
}
|
|
858
|
-
|
|
967
|
+
const invalidRes = await rpc("logging/setLevel", { level: "__invalid_level__" });
|
|
968
|
+
const validatesInput = !!invalidRes.body?.error;
|
|
969
|
+
const validLevels = ["debug", "warning", "error"];
|
|
970
|
+
const accepted = [];
|
|
971
|
+
for (const level of validLevels) {
|
|
972
|
+
const r = await rpc("logging/setLevel", { level });
|
|
973
|
+
if (!r.body?.error) accepted.push(level);
|
|
974
|
+
}
|
|
975
|
+
const details = validatesInput ? `logging/setLevel accepted (validates levels, ${accepted.length + 1} levels accepted)` : "logging/setLevel accepted (warning: server does not reject invalid log levels)";
|
|
976
|
+
if (!validatesInput) warnings.push("Server accepts invalid log levels without error");
|
|
977
|
+
return { passed: true, details };
|
|
859
978
|
}
|
|
860
979
|
);
|
|
861
980
|
const hasCompletions = !!serverInfo.capabilities.completions;
|
|
@@ -884,6 +1003,59 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
884
1003
|
return { passed: true, details: "completion/complete accepted" };
|
|
885
1004
|
}
|
|
886
1005
|
);
|
|
1006
|
+
await test(
|
|
1007
|
+
"lifecycle-cancellation",
|
|
1008
|
+
"Handles cancellation notifications",
|
|
1009
|
+
"lifecycle",
|
|
1010
|
+
false,
|
|
1011
|
+
"basic/utilities#cancellation",
|
|
1012
|
+
async () => {
|
|
1013
|
+
const res = await mcpNotification(
|
|
1014
|
+
backendUrl,
|
|
1015
|
+
"notifications/cancelled",
|
|
1016
|
+
{ requestId: 99999, reason: "compliance test" },
|
|
1017
|
+
buildHeaders(),
|
|
1018
|
+
timeout
|
|
1019
|
+
);
|
|
1020
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1021
|
+
return { passed: true, details: `HTTP ${res.statusCode} (cancellation accepted)` };
|
|
1022
|
+
}
|
|
1023
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should accept cancellation notifications` };
|
|
1024
|
+
}
|
|
1025
|
+
);
|
|
1026
|
+
await test(
|
|
1027
|
+
"lifecycle-progress",
|
|
1028
|
+
"Accepts progress notifications",
|
|
1029
|
+
"lifecycle",
|
|
1030
|
+
false,
|
|
1031
|
+
"basic/utilities#progress",
|
|
1032
|
+
async () => {
|
|
1033
|
+
const res = await mcpNotification(
|
|
1034
|
+
backendUrl,
|
|
1035
|
+
"notifications/progress",
|
|
1036
|
+
{ progressToken: "compliance-test-token", progress: 50, total: 100 },
|
|
1037
|
+
buildHeaders(),
|
|
1038
|
+
timeout
|
|
1039
|
+
);
|
|
1040
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1041
|
+
return { passed: true, details: `HTTP ${res.statusCode} (progress notification accepted)` };
|
|
1042
|
+
}
|
|
1043
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should accept progress notifications` };
|
|
1044
|
+
}
|
|
1045
|
+
);
|
|
1046
|
+
await test(
|
|
1047
|
+
"transport-content-type-init",
|
|
1048
|
+
"Initialize response has valid content type",
|
|
1049
|
+
"transport",
|
|
1050
|
+
false,
|
|
1051
|
+
"basic/transports#streamable-http",
|
|
1052
|
+
async () => {
|
|
1053
|
+
if (!initRes) return { passed: false, details: "No init response to check" };
|
|
1054
|
+
const ct = (initRes.headers["content-type"] || "").toLowerCase();
|
|
1055
|
+
const valid = ct.includes("application/json") || ct.includes("text/event-stream");
|
|
1056
|
+
return { passed: valid, details: `Content-Type: ${ct || "missing"}` };
|
|
1057
|
+
}
|
|
1058
|
+
);
|
|
887
1059
|
await test(
|
|
888
1060
|
"transport-notification-202",
|
|
889
1061
|
"Notification returns 202 Accepted",
|
|
@@ -941,6 +1113,88 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
941
1113
|
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
942
1114
|
}
|
|
943
1115
|
);
|
|
1116
|
+
await test(
|
|
1117
|
+
"transport-get-stream",
|
|
1118
|
+
"GET with session returns SSE or 405",
|
|
1119
|
+
"transport",
|
|
1120
|
+
false,
|
|
1121
|
+
"basic/transports#streamable-http",
|
|
1122
|
+
async () => {
|
|
1123
|
+
if (!sessionId) {
|
|
1124
|
+
return { passed: true, details: "No session ID \u2014 server-initiated messages not applicable" };
|
|
1125
|
+
}
|
|
1126
|
+
const res = await request(backendUrl, {
|
|
1127
|
+
method: "GET",
|
|
1128
|
+
headers: { Accept: "text/event-stream", ...buildHeaders() },
|
|
1129
|
+
signal: AbortSignal.timeout(Math.min(timeout, 3e3))
|
|
1130
|
+
});
|
|
1131
|
+
const body = await res.body.text();
|
|
1132
|
+
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
1133
|
+
if (res.statusCode === 405) {
|
|
1134
|
+
return { passed: true, details: "HTTP 405 (server does not support server-initiated messages)" };
|
|
1135
|
+
}
|
|
1136
|
+
if (ct.includes("text/event-stream")) {
|
|
1137
|
+
if (body.trim().length > 0) {
|
|
1138
|
+
const hasSSEFields = body.includes("data:") || body.includes("event:");
|
|
1139
|
+
if (!hasSSEFields) {
|
|
1140
|
+
return { passed: false, details: "Content-Type is text/event-stream but body has no SSE fields" };
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
return { passed: true, details: "GET with session returns SSE stream for server-initiated messages" };
|
|
1144
|
+
}
|
|
1145
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1146
|
+
return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
|
|
1147
|
+
}
|
|
1148
|
+
return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
|
|
1149
|
+
}
|
|
1150
|
+
);
|
|
1151
|
+
await test(
|
|
1152
|
+
"transport-concurrent",
|
|
1153
|
+
"Handles concurrent requests",
|
|
1154
|
+
"transport",
|
|
1155
|
+
false,
|
|
1156
|
+
"basic/transports#streamable-http",
|
|
1157
|
+
async () => {
|
|
1158
|
+
const ids = [createIdCounter(99930)(), createIdCounter(99931)(), createIdCounter(99932)()];
|
|
1159
|
+
const promises = ids.map(
|
|
1160
|
+
(id) => request(backendUrl, {
|
|
1161
|
+
method: "POST",
|
|
1162
|
+
headers: {
|
|
1163
|
+
"Content-Type": "application/json",
|
|
1164
|
+
Accept: "application/json, text/event-stream",
|
|
1165
|
+
...buildHeaders()
|
|
1166
|
+
},
|
|
1167
|
+
body: JSON.stringify({ jsonrpc: "2.0", id, method: "ping" }),
|
|
1168
|
+
signal: AbortSignal.timeout(timeout)
|
|
1169
|
+
}).then(async (res) => {
|
|
1170
|
+
const text = await res.body.text();
|
|
1171
|
+
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
1172
|
+
let body;
|
|
1173
|
+
if (ct.includes("text/event-stream")) {
|
|
1174
|
+
body = parseSSEResponse(text);
|
|
1175
|
+
}
|
|
1176
|
+
if (!body) {
|
|
1177
|
+
try {
|
|
1178
|
+
body = JSON.parse(text);
|
|
1179
|
+
} catch {
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
return { statusCode: res.statusCode, body, requestId: id };
|
|
1183
|
+
})
|
|
1184
|
+
);
|
|
1185
|
+
const results = await Promise.all(promises);
|
|
1186
|
+
const issues = [];
|
|
1187
|
+
for (const r of results) {
|
|
1188
|
+
if (r.statusCode < 200 || r.statusCode >= 300) {
|
|
1189
|
+
issues.push(`Request id=${r.requestId}: HTTP ${r.statusCode}`);
|
|
1190
|
+
} else if (r.body?.id !== r.requestId) {
|
|
1191
|
+
issues.push(`Request id=${r.requestId}: response id=${r.body?.id} (mismatch)`);
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1195
|
+
return { passed: true, details: `${results.length} concurrent requests handled correctly` };
|
|
1196
|
+
}
|
|
1197
|
+
);
|
|
944
1198
|
const hasTools = !!serverInfo.capabilities.tools;
|
|
945
1199
|
let cachedToolsList = null;
|
|
946
1200
|
await test(
|
|
@@ -962,6 +1216,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
962
1216
|
};
|
|
963
1217
|
}
|
|
964
1218
|
);
|
|
1219
|
+
const toolsListOk = cachedToolsList !== null;
|
|
965
1220
|
await test(
|
|
966
1221
|
"tools-schema",
|
|
967
1222
|
"All tools have name and inputSchema",
|
|
@@ -969,7 +1224,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
969
1224
|
hasTools,
|
|
970
1225
|
"server/tools#data-types",
|
|
971
1226
|
async () => {
|
|
972
|
-
|
|
1227
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1228
|
+
const tools = cachedToolsList ?? [];
|
|
973
1229
|
const issues = [];
|
|
974
1230
|
for (const tool of tools) {
|
|
975
1231
|
if (!tool.name) {
|
|
@@ -1001,7 +1257,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1001
1257
|
false,
|
|
1002
1258
|
"server/tools#annotations",
|
|
1003
1259
|
async () => {
|
|
1004
|
-
|
|
1260
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1261
|
+
const tools = cachedToolsList ?? [];
|
|
1005
1262
|
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1006
1263
|
const issues = [];
|
|
1007
1264
|
let annotatedCount = 0;
|
|
@@ -1031,7 +1288,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1031
1288
|
}
|
|
1032
1289
|
);
|
|
1033
1290
|
await test("tools-title-field", "Tools include title field", "schema", false, "server/tools#data-types", async () => {
|
|
1034
|
-
|
|
1291
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1292
|
+
const tools = cachedToolsList ?? [];
|
|
1035
1293
|
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1036
1294
|
const withTitle = tools.filter((t) => typeof t.title === "string");
|
|
1037
1295
|
const issues = [];
|
|
@@ -1053,7 +1311,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1053
1311
|
false,
|
|
1054
1312
|
"server/tools#structured-content",
|
|
1055
1313
|
async () => {
|
|
1056
|
-
|
|
1314
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1315
|
+
const tools = cachedToolsList ?? [];
|
|
1057
1316
|
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1058
1317
|
const issues = [];
|
|
1059
1318
|
let withSchema = 0;
|
|
@@ -1094,14 +1353,14 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1094
1353
|
return { passed: true, details: `Protocol error: code ${code} \u2014 ${error.message}` };
|
|
1095
1354
|
}
|
|
1096
1355
|
if (result?.content && Array.isArray(result.content)) {
|
|
1356
|
+
if (result.isError) {
|
|
1357
|
+
return { passed: true, details: "Tool returned execution error with content (valid)" };
|
|
1358
|
+
}
|
|
1097
1359
|
const badItems = result.content.filter((c) => !c.type);
|
|
1098
1360
|
if (badItems.length > 0)
|
|
1099
1361
|
return { passed: false, details: `${badItems.length} content item(s) missing 'type' field` };
|
|
1100
1362
|
return { passed: true, details: `Returned ${result.content.length} content item(s)` };
|
|
1101
1363
|
}
|
|
1102
|
-
if (result?.isError && result?.content && Array.isArray(result.content)) {
|
|
1103
|
-
return { passed: true, details: "Tool returned execution error with content (valid)" };
|
|
1104
|
-
}
|
|
1105
1364
|
return { passed: false, details: "Response missing content array" };
|
|
1106
1365
|
}
|
|
1107
1366
|
);
|
|
@@ -1203,6 +1462,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1203
1462
|
return { passed: true, details: `${resourceCount} resource(s)` };
|
|
1204
1463
|
}
|
|
1205
1464
|
);
|
|
1465
|
+
const resourcesListOk = cachedResourcesList !== null;
|
|
1206
1466
|
await test(
|
|
1207
1467
|
"resources-schema",
|
|
1208
1468
|
"Resources have uri and name",
|
|
@@ -1210,7 +1470,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1210
1470
|
true,
|
|
1211
1471
|
"server/resources#data-types",
|
|
1212
1472
|
async () => {
|
|
1213
|
-
|
|
1473
|
+
if (!resourcesListOk) return { passed: false, details: "Skipped: resources/list failed" };
|
|
1474
|
+
const resources = cachedResourcesList ?? [];
|
|
1214
1475
|
const issues = [];
|
|
1215
1476
|
for (const r of resources) {
|
|
1216
1477
|
if (!r.uri) issues.push("Resource missing uri");
|
|
@@ -1239,7 +1500,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1239
1500
|
false,
|
|
1240
1501
|
"server/resources#reading-resources",
|
|
1241
1502
|
async () => {
|
|
1242
|
-
const resources = cachedResourcesList ??
|
|
1503
|
+
const resources = cachedResourcesList ?? [];
|
|
1243
1504
|
const firstUri = resources[0]?.uri;
|
|
1244
1505
|
if (!firstUri) return { passed: false, details: "No resource URI to test" };
|
|
1245
1506
|
const readRes = await rpc("resources/read", { uri: firstUri });
|
|
@@ -1272,8 +1533,15 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1272
1533
|
if (!Array.isArray(templates)) return { passed: false, details: "No resourceTemplates array" };
|
|
1273
1534
|
const issues = [];
|
|
1274
1535
|
for (const t of templates) {
|
|
1275
|
-
if (!t.uriTemplate)
|
|
1536
|
+
if (!t.uriTemplate) {
|
|
1537
|
+
issues.push("Template missing uriTemplate");
|
|
1538
|
+
} else if (typeof t.uriTemplate !== "string") {
|
|
1539
|
+
issues.push(`uriTemplate should be a string, got ${typeof t.uriTemplate}`);
|
|
1540
|
+
} else if (!t.uriTemplate.includes("{") || !t.uriTemplate.includes("}")) {
|
|
1541
|
+
warnings.push(`Template "${t.name || t.uriTemplate}" has no URI template parameters (e.g., {id})`);
|
|
1542
|
+
}
|
|
1276
1543
|
if (!t.name) issues.push(`${t.uriTemplate || "?"}: missing name`);
|
|
1544
|
+
if (!t.description) warnings.push(`Template "${t.name || t.uriTemplate || "?"}" missing description`);
|
|
1277
1545
|
}
|
|
1278
1546
|
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1279
1547
|
return { passed: true, details: `${templates.length} resource template(s)` };
|
|
@@ -1315,7 +1583,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1315
1583
|
true,
|
|
1316
1584
|
"server/resources#subscriptions",
|
|
1317
1585
|
async () => {
|
|
1318
|
-
const resources = cachedResourcesList ??
|
|
1586
|
+
const resources = cachedResourcesList ?? [];
|
|
1319
1587
|
const firstUri = resources[0]?.uri;
|
|
1320
1588
|
if (!firstUri) return { passed: false, details: "No resource URI for subscribe test" };
|
|
1321
1589
|
const subRes = await rpc("resources/subscribe", { uri: firstUri });
|
|
@@ -1359,8 +1627,10 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1359
1627
|
};
|
|
1360
1628
|
}
|
|
1361
1629
|
);
|
|
1630
|
+
const promptsListOk = cachedPromptsList !== null;
|
|
1362
1631
|
await test("prompts-schema", "Prompts have name field", "schema", true, "server/prompts#data-types", async () => {
|
|
1363
|
-
|
|
1632
|
+
if (!promptsListOk) return { passed: false, details: "Skipped: prompts/list failed" };
|
|
1633
|
+
const prompts = cachedPromptsList ?? [];
|
|
1364
1634
|
const issues = [];
|
|
1365
1635
|
for (const p of prompts) {
|
|
1366
1636
|
if (!p.name) issues.push("Prompt missing name");
|
|
@@ -1556,6 +1826,60 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1556
1826
|
}
|
|
1557
1827
|
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error code -32600` };
|
|
1558
1828
|
});
|
|
1829
|
+
await test(
|
|
1830
|
+
"transport-delete",
|
|
1831
|
+
"DELETE accepted or returns 405",
|
|
1832
|
+
"transport",
|
|
1833
|
+
false,
|
|
1834
|
+
"basic/transports#streamable-http",
|
|
1835
|
+
async () => {
|
|
1836
|
+
const deleteHeaders = { ...buildHeaders() };
|
|
1837
|
+
const res = await request(backendUrl, {
|
|
1838
|
+
method: "DELETE",
|
|
1839
|
+
headers: deleteHeaders,
|
|
1840
|
+
signal: AbortSignal.timeout(timeout)
|
|
1841
|
+
});
|
|
1842
|
+
await res.body.text();
|
|
1843
|
+
if (res.statusCode === 405) {
|
|
1844
|
+
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
1845
|
+
}
|
|
1846
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1847
|
+
if (sessionId) {
|
|
1848
|
+
try {
|
|
1849
|
+
const verifyRes = await mcpRequest(
|
|
1850
|
+
backendUrl,
|
|
1851
|
+
"ping",
|
|
1852
|
+
void 0,
|
|
1853
|
+
createIdCounter(99920),
|
|
1854
|
+
deleteHeaders,
|
|
1855
|
+
timeout
|
|
1856
|
+
);
|
|
1857
|
+
if (verifyRes.statusCode === 400 || verifyRes.statusCode === 404 || verifyRes.statusCode === 409) {
|
|
1858
|
+
return {
|
|
1859
|
+
passed: true,
|
|
1860
|
+
details: `HTTP ${res.statusCode} (session terminated, post-delete request correctly rejected with ${verifyRes.statusCode})`
|
|
1861
|
+
};
|
|
1862
|
+
}
|
|
1863
|
+
} catch {
|
|
1864
|
+
return {
|
|
1865
|
+
passed: true,
|
|
1866
|
+
details: `HTTP ${res.statusCode} (session terminated, post-delete request rejected)`
|
|
1867
|
+
};
|
|
1868
|
+
}
|
|
1869
|
+
}
|
|
1870
|
+
return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
|
|
1871
|
+
}
|
|
1872
|
+
if (res.statusCode === 400 || res.statusCode === 404) {
|
|
1873
|
+
return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
|
|
1874
|
+
}
|
|
1875
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
1876
|
+
}
|
|
1877
|
+
);
|
|
1878
|
+
const MAX_WARNINGS = 50;
|
|
1879
|
+
if (warnings.length > MAX_WARNINGS) {
|
|
1880
|
+
const truncated = warnings.length - MAX_WARNINGS;
|
|
1881
|
+
warnings.splice(MAX_WARNINGS, truncated, `... and ${truncated} more warning(s) suppressed`);
|
|
1882
|
+
}
|
|
1559
1883
|
const { score, grade, overall, summary, categories } = computeScore(tests);
|
|
1560
1884
|
const badge = generateBadge(url);
|
|
1561
1885
|
return {
|
|
@@ -1585,7 +1909,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1585
1909
|
function registerTools(server) {
|
|
1586
1910
|
server.tool(
|
|
1587
1911
|
"mcp_compliance_test",
|
|
1588
|
-
"Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all
|
|
1912
|
+
"Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all 48 tests covering transport, lifecycle, tools, resources, prompts, errors, and schema validation.",
|
|
1589
1913
|
{
|
|
1590
1914
|
url: z.string().url().describe("The MCP server URL to test (must be HTTP or HTTPS)"),
|
|
1591
1915
|
auth: z.string().optional().describe('Authorization header value (e.g., "Bearer tok123")'),
|
|
@@ -1641,8 +1965,9 @@ ${JSON.stringify(report, null, 2)}` }
|
|
|
1641
1965
|
]
|
|
1642
1966
|
};
|
|
1643
1967
|
} catch (err) {
|
|
1968
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1644
1969
|
return {
|
|
1645
|
-
content: [{ type: "text", text: `Error running compliance test: ${
|
|
1970
|
+
content: [{ type: "text", text: `Error running compliance test: ${message}` }],
|
|
1646
1971
|
isError: true
|
|
1647
1972
|
};
|
|
1648
1973
|
}
|
|
@@ -1690,8 +2015,9 @@ ${JSON.stringify(report, null, 2)}` }
|
|
|
1690
2015
|
]
|
|
1691
2016
|
};
|
|
1692
2017
|
} catch (err) {
|
|
2018
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1693
2019
|
return {
|
|
1694
|
-
content: [{ type: "text", text: `Error: ${
|
|
2020
|
+
content: [{ type: "text", text: `Error: ${message}` }],
|
|
1695
2021
|
isError: true
|
|
1696
2022
|
};
|
|
1697
2023
|
}
|
|
@@ -1735,9 +2061,11 @@ ${TEST_DEFINITIONS.map((t) => t.id).join(", ")}`
|
|
|
1735
2061
|
`Name: ${def.name}`,
|
|
1736
2062
|
`Category: ${def.category}`,
|
|
1737
2063
|
`Required: ${def.required ? "Yes" : "No"}`,
|
|
1738
|
-
`Spec reference:
|
|
2064
|
+
`Spec reference: ${SPEC_BASE}/${def.specRef}`,
|
|
2065
|
+
"",
|
|
2066
|
+
def.description,
|
|
1739
2067
|
"",
|
|
1740
|
-
def.
|
|
2068
|
+
`Fix: ${def.recommendation}`
|
|
1741
2069
|
].join("\n")
|
|
1742
2070
|
}
|
|
1743
2071
|
]
|
|
@@ -1746,6 +2074,27 @@ ${TEST_DEFINITIONS.map((t) => t.id).join(", ")}`
|
|
|
1746
2074
|
);
|
|
1747
2075
|
}
|
|
1748
2076
|
|
|
2077
|
+
// src/mcp/server.ts
|
|
2078
|
+
var require2 = createRequire2(import.meta.url);
|
|
2079
|
+
var { version } = require2("../../package.json");
|
|
2080
|
+
function createComplianceServer() {
|
|
2081
|
+
const server = new McpServer({ name: "mcp-compliance", version });
|
|
2082
|
+
registerTools(server);
|
|
2083
|
+
return server;
|
|
2084
|
+
}
|
|
2085
|
+
async function startServer() {
|
|
2086
|
+
const server = createComplianceServer();
|
|
2087
|
+
const transport = new StdioServerTransport();
|
|
2088
|
+
await server.connect(transport);
|
|
2089
|
+
}
|
|
2090
|
+
var isDirectRun = process.argv[1]?.endsWith("mcp/server.js") || process.argv[1]?.endsWith("mcp\\server.js");
|
|
2091
|
+
if (isDirectRun) {
|
|
2092
|
+
startServer().catch((err) => {
|
|
2093
|
+
console.error("MCP server error:", err);
|
|
2094
|
+
process.exit(1);
|
|
2095
|
+
});
|
|
2096
|
+
}
|
|
2097
|
+
|
|
1749
2098
|
// src/reporter.ts
|
|
1750
2099
|
import chalk from "chalk";
|
|
1751
2100
|
var CATEGORY_LABELS = {
|
|
@@ -1788,8 +2137,16 @@ function testLine(t) {
|
|
|
1788
2137
|
const icon = t.passed ? chalk.green(" PASS") : chalk.red(" FAIL");
|
|
1789
2138
|
const req = t.required ? chalk.dim(" (required)") : "";
|
|
1790
2139
|
const dur = chalk.dim(` ${t.durationMs}ms`);
|
|
1791
|
-
|
|
2140
|
+
let line = `${icon} ${t.name}${req}${dur}
|
|
1792
2141
|
${chalk.dim(` ${t.details}`)}`;
|
|
2142
|
+
if (!t.passed) {
|
|
2143
|
+
const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
|
|
2144
|
+
if (def?.recommendation) {
|
|
2145
|
+
line += `
|
|
2146
|
+
${chalk.cyan(` Fix: ${def.recommendation}`)}`;
|
|
2147
|
+
}
|
|
2148
|
+
}
|
|
2149
|
+
return line;
|
|
1793
2150
|
}
|
|
1794
2151
|
function formatTerminal(report) {
|
|
1795
2152
|
const lines = [];
|
|
@@ -1872,10 +2229,85 @@ function formatTerminal(report) {
|
|
|
1872
2229
|
function formatJson(report) {
|
|
1873
2230
|
return JSON.stringify(report, null, 2);
|
|
1874
2231
|
}
|
|
2232
|
+
function formatSarif(report) {
|
|
2233
|
+
const rules = report.tests.map((t) => {
|
|
2234
|
+
const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
|
|
2235
|
+
return {
|
|
2236
|
+
id: t.id,
|
|
2237
|
+
name: t.name,
|
|
2238
|
+
shortDescription: { text: t.name },
|
|
2239
|
+
fullDescription: { text: def?.description || t.details },
|
|
2240
|
+
helpUri: t.specRef || `${SPEC_BASE}/basic`,
|
|
2241
|
+
properties: {
|
|
2242
|
+
category: t.category,
|
|
2243
|
+
required: t.required
|
|
2244
|
+
}
|
|
2245
|
+
};
|
|
2246
|
+
});
|
|
2247
|
+
const results = report.tests.filter((t) => !t.passed).map((t) => {
|
|
2248
|
+
const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
|
|
2249
|
+
return {
|
|
2250
|
+
ruleId: t.id,
|
|
2251
|
+
level: t.required ? "error" : "warning",
|
|
2252
|
+
message: {
|
|
2253
|
+
text: def?.recommendation ? `${t.details}. Fix: ${def.recommendation}` : t.details
|
|
2254
|
+
},
|
|
2255
|
+
locations: [
|
|
2256
|
+
{
|
|
2257
|
+
physicalLocation: {
|
|
2258
|
+
artifactLocation: {
|
|
2259
|
+
uri: report.url,
|
|
2260
|
+
uriBaseId: "MCP_SERVER"
|
|
2261
|
+
}
|
|
2262
|
+
}
|
|
2263
|
+
}
|
|
2264
|
+
],
|
|
2265
|
+
properties: {
|
|
2266
|
+
category: t.category,
|
|
2267
|
+
durationMs: t.durationMs
|
|
2268
|
+
}
|
|
2269
|
+
};
|
|
2270
|
+
});
|
|
2271
|
+
const sarif = {
|
|
2272
|
+
$schema: "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json",
|
|
2273
|
+
version: "2.1.0",
|
|
2274
|
+
runs: [
|
|
2275
|
+
{
|
|
2276
|
+
tool: {
|
|
2277
|
+
driver: {
|
|
2278
|
+
name: "mcp-compliance",
|
|
2279
|
+
version: report.toolVersion,
|
|
2280
|
+
informationUri: "https://github.com/YawLabs/mcp-compliance",
|
|
2281
|
+
rules
|
|
2282
|
+
}
|
|
2283
|
+
},
|
|
2284
|
+
results,
|
|
2285
|
+
invocations: [
|
|
2286
|
+
{
|
|
2287
|
+
executionSuccessful: report.overall !== "fail",
|
|
2288
|
+
properties: {
|
|
2289
|
+
grade: report.grade,
|
|
2290
|
+
score: report.score,
|
|
2291
|
+
overall: report.overall,
|
|
2292
|
+
specVersion: report.specVersion,
|
|
2293
|
+
serverUrl: report.url,
|
|
2294
|
+
serverName: report.serverInfo.name,
|
|
2295
|
+
serverVersion: report.serverInfo.version,
|
|
2296
|
+
protocolVersion: report.serverInfo.protocolVersion,
|
|
2297
|
+
testsPassed: report.summary.passed,
|
|
2298
|
+
testsTotal: report.summary.total
|
|
2299
|
+
}
|
|
2300
|
+
}
|
|
2301
|
+
]
|
|
2302
|
+
}
|
|
2303
|
+
]
|
|
2304
|
+
};
|
|
2305
|
+
return JSON.stringify(sarif, null, 2);
|
|
2306
|
+
}
|
|
1875
2307
|
|
|
1876
2308
|
// src/index.ts
|
|
1877
|
-
var
|
|
1878
|
-
var { version } =
|
|
2309
|
+
var require3 = createRequire3(import.meta.url);
|
|
2310
|
+
var { version: version2 } = require3("../package.json");
|
|
1879
2311
|
function parseHeaderArg(value, prev) {
|
|
1880
2312
|
const idx = value.indexOf(":");
|
|
1881
2313
|
if (idx === -1) {
|
|
@@ -1886,12 +2318,27 @@ function parseHeaderArg(value, prev) {
|
|
|
1886
2318
|
prev[key] = val;
|
|
1887
2319
|
return prev;
|
|
1888
2320
|
}
|
|
2321
|
+
function parsePositiveInt(value, name, min = 0) {
|
|
2322
|
+
const n = Number.parseInt(value, 10);
|
|
2323
|
+
if (Number.isNaN(n) || n < min) {
|
|
2324
|
+
throw new Error(`${name} must be an integer >= ${min}, got "${value}"`);
|
|
2325
|
+
}
|
|
2326
|
+
return n;
|
|
2327
|
+
}
|
|
1889
2328
|
function parseList(value) {
|
|
1890
2329
|
return value.split(",").map((s) => s.trim()).filter(Boolean);
|
|
1891
2330
|
}
|
|
1892
2331
|
var program = new Command();
|
|
1893
|
-
program.name("mcp-compliance").description("Test MCP servers for spec compliance").version(
|
|
1894
|
-
program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal or
|
|
2332
|
+
program.name("mcp-compliance").description("Test MCP servers for spec compliance").version(version2);
|
|
2333
|
+
program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal, json, or sarif", "terminal").option("--strict", "Exit with code 1 on any required test failure (for CI)").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").option("--retries <n>", "Number of retries for failed tests", "0").option(
|
|
2334
|
+
"--only <items>",
|
|
2335
|
+
'Only run matching categories or test IDs, comma-separated (e.g., "transport,lifecycle" or "transport-post,lifecycle-init")',
|
|
2336
|
+
parseList
|
|
2337
|
+
).option(
|
|
2338
|
+
"--skip <items>",
|
|
2339
|
+
'Skip matching categories or test IDs, comma-separated (e.g., "schema" or "tools-pagination")',
|
|
2340
|
+
parseList
|
|
2341
|
+
).option("--verbose", "Print each test result as it runs").action(
|
|
1895
2342
|
async (url, opts) => {
|
|
1896
2343
|
try {
|
|
1897
2344
|
const headers = { ...opts.header };
|
|
@@ -1903,8 +2350,8 @@ Testing ${url}...
|
|
|
1903
2350
|
}
|
|
1904
2351
|
const report = await runComplianceSuite(url, {
|
|
1905
2352
|
headers,
|
|
1906
|
-
timeout:
|
|
1907
|
-
retries:
|
|
2353
|
+
timeout: parsePositiveInt(opts.timeout, "--timeout", 1),
|
|
2354
|
+
retries: parsePositiveInt(opts.retries, "--retries"),
|
|
1908
2355
|
only: opts.only,
|
|
1909
2356
|
skip: opts.skip,
|
|
1910
2357
|
onProgress: opts.verbose ? (testId, passed, details) => {
|
|
@@ -1917,6 +2364,8 @@ Testing ${url}...
|
|
|
1917
2364
|
}
|
|
1918
2365
|
if (opts.format === "json") {
|
|
1919
2366
|
console.log(formatJson(report));
|
|
2367
|
+
} else if (opts.format === "sarif") {
|
|
2368
|
+
console.log(formatSarif(report));
|
|
1920
2369
|
} else {
|
|
1921
2370
|
console.log(formatTerminal(report));
|
|
1922
2371
|
}
|
|
@@ -1924,11 +2373,12 @@ Testing ${url}...
|
|
|
1924
2373
|
process.exit(1);
|
|
1925
2374
|
}
|
|
1926
2375
|
} catch (err) {
|
|
1927
|
-
|
|
1928
|
-
|
|
2376
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2377
|
+
if (opts.format === "json" || opts.format === "sarif") {
|
|
2378
|
+
console.error(JSON.stringify({ error: message }));
|
|
1929
2379
|
} else {
|
|
1930
2380
|
console.error(chalk2.red(`
|
|
1931
|
-
Error: ${
|
|
2381
|
+
Error: ${message}
|
|
1932
2382
|
`));
|
|
1933
2383
|
}
|
|
1934
2384
|
process.exit(1);
|
|
@@ -1944,23 +2394,21 @@ Testing ${url}...
|
|
|
1944
2394
|
`));
|
|
1945
2395
|
const report = await runComplianceSuite(url, {
|
|
1946
2396
|
headers,
|
|
1947
|
-
timeout:
|
|
2397
|
+
timeout: parsePositiveInt(opts.timeout, "--timeout", 1)
|
|
1948
2398
|
});
|
|
1949
2399
|
console.log(`Grade: ${report.grade} (${report.score}%)
|
|
1950
2400
|
`);
|
|
1951
2401
|
console.log(report.badge.markdown);
|
|
1952
2402
|
console.log("");
|
|
1953
2403
|
} catch (err) {
|
|
2404
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1954
2405
|
console.error(chalk2.red(`
|
|
1955
|
-
Error: ${
|
|
2406
|
+
Error: ${message}
|
|
1956
2407
|
`));
|
|
1957
2408
|
process.exit(1);
|
|
1958
2409
|
}
|
|
1959
2410
|
});
|
|
1960
2411
|
program.command("mcp").description("Start the MCP compliance server (stdio transport)").action(async () => {
|
|
1961
|
-
|
|
1962
|
-
registerTools(server);
|
|
1963
|
-
const transport = new StdioServerTransport();
|
|
1964
|
-
await server.connect(transport);
|
|
2412
|
+
await startServer();
|
|
1965
2413
|
});
|
|
1966
2414
|
program.parse();
|