@yawlabs/mcp-compliance 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +77 -12
- package/dist/{chunk-U66YZGE5.js → chunk-KNOSZ3TD.js} +86 -43
- package/dist/index.js +208 -62
- package/dist/mcp/server.d.ts +11 -1
- package/dist/mcp/server.js +26 -13
- package/dist/runner.d.ts +1 -0
- package/dist/runner.js +1 -1
- package/package.json +13 -5
package/dist/index.js
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
+
import { createRequire as createRequire3 } from "module";
|
|
5
|
+
import chalk2 from "chalk";
|
|
6
|
+
import { Command } from "commander";
|
|
7
|
+
|
|
8
|
+
// src/mcp/server.ts
|
|
4
9
|
import { createRequire as createRequire2 } from "module";
|
|
5
10
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
6
11
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
7
|
-
import chalk2 from "chalk";
|
|
8
|
-
import { Command } from "commander";
|
|
9
12
|
|
|
10
13
|
// src/mcp/tools.ts
|
|
11
14
|
import { z } from "zod";
|
|
@@ -77,7 +80,8 @@ var TEST_DEFINITIONS = [
|
|
|
77
80
|
category: "transport",
|
|
78
81
|
required: true,
|
|
79
82
|
specRef: "basic/transports#streamable-http",
|
|
80
|
-
description: "Verifies the server accepts HTTP POST requests and returns a 2xx status code. This is the fundamental transport requirement for Streamable HTTP MCP servers."
|
|
83
|
+
description: "Verifies the server accepts HTTP POST requests and returns a 2xx status code. This is the fundamental transport requirement for Streamable HTTP MCP servers.",
|
|
84
|
+
recommendation: "Ensure your server listens for POST requests on the MCP endpoint. If you see 401/403, pass --auth with a valid token. Check that the URL is correct and the server is running."
|
|
81
85
|
},
|
|
82
86
|
{
|
|
83
87
|
id: "transport-content-type",
|
|
@@ -85,7 +89,8 @@ var TEST_DEFINITIONS = [
|
|
|
85
89
|
category: "transport",
|
|
86
90
|
required: true,
|
|
87
91
|
specRef: "basic/transports#streamable-http",
|
|
88
|
-
description: "Checks that the server responds with Content-Type application/json or text/event-stream. MCP servers must use one of these two content types."
|
|
92
|
+
description: "Checks that the server responds with Content-Type application/json or text/event-stream. MCP servers must use one of these two content types.",
|
|
93
|
+
recommendation: 'Set the Content-Type response header to "application/json" for synchronous responses or "text/event-stream" for streaming. Do not use text/html or other types.'
|
|
89
94
|
},
|
|
90
95
|
{
|
|
91
96
|
id: "transport-notification-202",
|
|
@@ -93,7 +98,8 @@ var TEST_DEFINITIONS = [
|
|
|
93
98
|
category: "transport",
|
|
94
99
|
required: false,
|
|
95
100
|
specRef: "basic/transports#streamable-http",
|
|
96
|
-
description: "Verifies that sending a JSON-RPC notification (no id field) returns HTTP 202 Accepted with no body. Per spec, servers MUST return 202 for notifications."
|
|
101
|
+
description: "Verifies that sending a JSON-RPC notification (no id field) returns HTTP 202 Accepted with no body. Per spec, servers MUST return 202 for notifications.",
|
|
102
|
+
recommendation: "Detect JSON-RPC messages without an id field and return HTTP 202 with an empty body. Do not attempt to send a JSON-RPC response for notifications."
|
|
97
103
|
},
|
|
98
104
|
{
|
|
99
105
|
id: "transport-session-id",
|
|
@@ -101,7 +107,8 @@ var TEST_DEFINITIONS = [
|
|
|
101
107
|
category: "transport",
|
|
102
108
|
required: false,
|
|
103
109
|
specRef: "basic/transports#streamable-http",
|
|
104
|
-
description: "Tests that the server returns HTTP 400 when MCP-Session-Id header is missing on requests after initialization (when the server issued a session ID)."
|
|
110
|
+
description: "Tests that the server returns HTTP 400 when MCP-Session-Id header is missing on requests after initialization (when the server issued a session ID).",
|
|
111
|
+
recommendation: "If your server issues an MCP-Session-Id header in the initialize response, reject subsequent requests that omit this header with HTTP 400."
|
|
105
112
|
},
|
|
106
113
|
{
|
|
107
114
|
id: "transport-get",
|
|
@@ -109,7 +116,8 @@ var TEST_DEFINITIONS = [
|
|
|
109
116
|
category: "transport",
|
|
110
117
|
required: false,
|
|
111
118
|
specRef: "basic/transports#streamable-http",
|
|
112
|
-
description: "Tests the GET endpoint for server-initiated messages. Server should return text/event-stream or 405 Method Not Allowed."
|
|
119
|
+
description: "Tests the GET endpoint for server-initiated messages. Server should return text/event-stream or 405 Method Not Allowed.",
|
|
120
|
+
recommendation: "If your server supports server-initiated messages, handle GET with text/event-stream. Otherwise, return 405 Method Not Allowed."
|
|
113
121
|
},
|
|
114
122
|
{
|
|
115
123
|
id: "transport-delete",
|
|
@@ -117,7 +125,8 @@ var TEST_DEFINITIONS = [
|
|
|
117
125
|
category: "transport",
|
|
118
126
|
required: false,
|
|
119
127
|
specRef: "basic/transports#streamable-http",
|
|
120
|
-
description: "Tests the DELETE endpoint for session termination. Server should accept the request or return 405 Method Not Allowed."
|
|
128
|
+
description: "Tests the DELETE endpoint for session termination. Server should accept the request or return 405 Method Not Allowed.",
|
|
129
|
+
recommendation: "Handle DELETE requests for session cleanup, or return 405 if session termination is not supported. Do not return 500."
|
|
121
130
|
},
|
|
122
131
|
{
|
|
123
132
|
id: "transport-batch-reject",
|
|
@@ -125,7 +134,8 @@ var TEST_DEFINITIONS = [
|
|
|
125
134
|
category: "transport",
|
|
126
135
|
required: true,
|
|
127
136
|
specRef: "basic/transports#streamable-http",
|
|
128
|
-
description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests."
|
|
137
|
+
description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests.",
|
|
138
|
+
recommendation: "Check if the parsed JSON body is an array. If so, return a JSON-RPC error or HTTP 400. Do not process batch requests \u2014 MCP explicitly forbids them."
|
|
129
139
|
},
|
|
130
140
|
// ── Lifecycle (10 tests) ─────────────────────────────────────────
|
|
131
141
|
{
|
|
@@ -134,7 +144,8 @@ var TEST_DEFINITIONS = [
|
|
|
134
144
|
category: "lifecycle",
|
|
135
145
|
required: true,
|
|
136
146
|
specRef: "basic/lifecycle#initialization",
|
|
137
|
-
description: "Tests the initialize handshake by sending an initialize request with client capabilities. The server must return a result with protocolVersion."
|
|
147
|
+
description: "Tests the initialize handshake by sending an initialize request with client capabilities. The server must return a result with protocolVersion.",
|
|
148
|
+
recommendation: 'Implement the "initialize" method handler. Return a result object with at least protocolVersion, capabilities, and serverInfo fields.'
|
|
138
149
|
},
|
|
139
150
|
{
|
|
140
151
|
id: "lifecycle-proto-version",
|
|
@@ -142,7 +153,8 @@ var TEST_DEFINITIONS = [
|
|
|
142
153
|
category: "lifecycle",
|
|
143
154
|
required: true,
|
|
144
155
|
specRef: "basic/lifecycle#version-negotiation",
|
|
145
|
-
description: "Validates that the protocolVersion returned by the server matches the YYYY-MM-DD date format required by the spec."
|
|
156
|
+
description: "Validates that the protocolVersion returned by the server matches the YYYY-MM-DD date format required by the spec.",
|
|
157
|
+
recommendation: `Return protocolVersion as a YYYY-MM-DD string (e.g., "2025-11-25"). The server should negotiate based on the client's requested version.`
|
|
146
158
|
},
|
|
147
159
|
{
|
|
148
160
|
id: "lifecycle-server-info",
|
|
@@ -150,7 +162,8 @@ var TEST_DEFINITIONS = [
|
|
|
150
162
|
category: "lifecycle",
|
|
151
163
|
required: false,
|
|
152
164
|
specRef: "basic/lifecycle#initialization",
|
|
153
|
-
description: "Checks that the server includes a serverInfo object with at least a name field in its initialize response. While recommended, this is not strictly required."
|
|
165
|
+
description: "Checks that the server includes a serverInfo object with at least a name field in its initialize response. While recommended, this is not strictly required.",
|
|
166
|
+
recommendation: 'Add a serverInfo object to your initialize response: { name: "your-server", version: "1.0.0" }. This helps clients identify your server.'
|
|
154
167
|
},
|
|
155
168
|
{
|
|
156
169
|
id: "lifecycle-capabilities",
|
|
@@ -158,7 +171,8 @@ var TEST_DEFINITIONS = [
|
|
|
158
171
|
category: "lifecycle",
|
|
159
172
|
required: true,
|
|
160
173
|
specRef: "basic/lifecycle#capability-negotiation",
|
|
161
|
-
description: "Verifies the server returns a capabilities object in its initialize response. An empty object is valid (no optional features declared)."
|
|
174
|
+
description: "Verifies the server returns a capabilities object in its initialize response. An empty object is valid (no optional features declared).",
|
|
175
|
+
recommendation: "Include a capabilities object in your initialize response. Declare the features your server supports (tools, resources, prompts, logging, etc.). An empty object {} is valid."
|
|
162
176
|
},
|
|
163
177
|
{
|
|
164
178
|
id: "lifecycle-jsonrpc",
|
|
@@ -166,7 +180,8 @@ var TEST_DEFINITIONS = [
|
|
|
166
180
|
category: "lifecycle",
|
|
167
181
|
required: true,
|
|
168
182
|
specRef: "basic",
|
|
169
|
-
description: 'Validates that the initialize response is a proper JSON-RPC 2.0 message with jsonrpc="2.0", an id field, and either a result or error field.'
|
|
183
|
+
description: 'Validates that the initialize response is a proper JSON-RPC 2.0 message with jsonrpc="2.0", an id field, and either a result or error field.',
|
|
184
|
+
recommendation: 'Ensure every response includes jsonrpc: "2.0", the matching id from the request, and either a result or error field. Never omit the jsonrpc field.'
|
|
170
185
|
},
|
|
171
186
|
{
|
|
172
187
|
id: "lifecycle-ping",
|
|
@@ -174,7 +189,8 @@ var TEST_DEFINITIONS = [
|
|
|
174
189
|
category: "lifecycle",
|
|
175
190
|
required: true,
|
|
176
191
|
specRef: "basic/utilities#ping",
|
|
177
|
-
description: "Tests that the server responds to the ping method with an empty result object. This is a required utility method."
|
|
192
|
+
description: "Tests that the server responds to the ping method with an empty result object. This is a required utility method.",
|
|
193
|
+
recommendation: 'Implement a "ping" method handler that returns an empty result object {}. This is required by the MCP spec for keepalive and connectivity checking.'
|
|
178
194
|
},
|
|
179
195
|
{
|
|
180
196
|
id: "lifecycle-instructions",
|
|
@@ -182,7 +198,8 @@ var TEST_DEFINITIONS = [
|
|
|
182
198
|
category: "lifecycle",
|
|
183
199
|
required: false,
|
|
184
200
|
specRef: "basic/lifecycle#initialization",
|
|
185
|
-
description: "If the server includes an instructions field in the initialize response, validates it is a string. Instructions provide guidance for how the client should interact with the server."
|
|
201
|
+
description: "If the server includes an instructions field in the initialize response, validates it is a string. Instructions provide guidance for how the client should interact with the server.",
|
|
202
|
+
recommendation: "If you include an instructions field in the initialize response, ensure it is a string. Remove the field or fix the type if it is not a string."
|
|
186
203
|
},
|
|
187
204
|
{
|
|
188
205
|
id: "lifecycle-id-match",
|
|
@@ -190,7 +207,8 @@ var TEST_DEFINITIONS = [
|
|
|
190
207
|
category: "lifecycle",
|
|
191
208
|
required: true,
|
|
192
209
|
specRef: "basic",
|
|
193
|
-
description: "Verifies that the JSON-RPC response id matches the request id sent by the client. This is a fundamental JSON-RPC 2.0 requirement."
|
|
210
|
+
description: "Verifies that the JSON-RPC response id matches the request id sent by the client. This is a fundamental JSON-RPC 2.0 requirement.",
|
|
211
|
+
recommendation: "Copy the id field from the request into the response. This is a core JSON-RPC 2.0 requirement. Check that your framework does not modify or discard the request ID."
|
|
194
212
|
},
|
|
195
213
|
{
|
|
196
214
|
id: "lifecycle-logging",
|
|
@@ -198,7 +216,8 @@ var TEST_DEFINITIONS = [
|
|
|
198
216
|
category: "lifecycle",
|
|
199
217
|
required: false,
|
|
200
218
|
specRef: "server/utilities#logging",
|
|
201
|
-
description: "If the server declares logging capability, tests that logging/setLevel method is accepted with a valid log level."
|
|
219
|
+
description: "If the server declares logging capability, tests that logging/setLevel method is accepted with a valid log level.",
|
|
220
|
+
recommendation: 'If you declare logging in capabilities, implement the "logging/setLevel" handler. Accept standard log levels: debug, info, notice, warning, error, critical, alert, emergency.'
|
|
202
221
|
},
|
|
203
222
|
{
|
|
204
223
|
id: "lifecycle-completions",
|
|
@@ -206,7 +225,8 @@ var TEST_DEFINITIONS = [
|
|
|
206
225
|
category: "lifecycle",
|
|
207
226
|
required: false,
|
|
208
227
|
specRef: "server/utilities#completion",
|
|
209
|
-
description: "If the server declares completions capability, tests that the completion/complete method is accepted."
|
|
228
|
+
description: "If the server declares completions capability, tests that the completion/complete method is accepted.",
|
|
229
|
+
recommendation: 'If you declare completions in capabilities, implement the "completion/complete" handler. Return a completion object with a values array, even if empty.'
|
|
210
230
|
},
|
|
211
231
|
// ── Tools (4 tests) ──────────────────────────────────────────────
|
|
212
232
|
{
|
|
@@ -215,7 +235,8 @@ var TEST_DEFINITIONS = [
|
|
|
215
235
|
category: "tools",
|
|
216
236
|
required: false,
|
|
217
237
|
specRef: "server/tools#listing-tools",
|
|
218
|
-
description: "Calls tools/list and validates it returns an array of tool definitions. Required if the server declares tools capability."
|
|
238
|
+
description: "Calls tools/list and validates it returns an array of tool definitions. Required if the server declares tools capability.",
|
|
239
|
+
recommendation: "Implement the tools/list handler to return { tools: [...] } with an array of tool definition objects. Each tool needs at least a name and inputSchema."
|
|
219
240
|
},
|
|
220
241
|
{
|
|
221
242
|
id: "tools-call",
|
|
@@ -223,7 +244,8 @@ var TEST_DEFINITIONS = [
|
|
|
223
244
|
category: "tools",
|
|
224
245
|
required: false,
|
|
225
246
|
specRef: "server/tools#calling-tools",
|
|
226
|
-
description: "Calls the first tool with empty arguments and verifies the response format. Accepts both successful results and InvalidParams errors."
|
|
247
|
+
description: "Calls the first tool with empty arguments and verifies the response format. Accepts both successful results and InvalidParams errors.",
|
|
248
|
+
recommendation: "Ensure tools/call returns { content: [...] } with an array of content objects, each having a type field. Return isError: true for tool execution errors."
|
|
227
249
|
},
|
|
228
250
|
{
|
|
229
251
|
id: "tools-pagination",
|
|
@@ -231,7 +253,8 @@ var TEST_DEFINITIONS = [
|
|
|
231
253
|
category: "tools",
|
|
232
254
|
required: false,
|
|
233
255
|
specRef: "server/tools#listing-tools",
|
|
234
|
-
description: "Tests cursor-based pagination on tools/list. Validates nextCursor is a string if present and that fetching the next page returns a valid response."
|
|
256
|
+
description: "Tests cursor-based pagination on tools/list. Validates nextCursor is a string if present and that fetching the next page returns a valid response.",
|
|
257
|
+
recommendation: "If your server has many tools, include a nextCursor string in the response. Ensure passing this cursor back in a subsequent request returns the next page."
|
|
235
258
|
},
|
|
236
259
|
{
|
|
237
260
|
id: "tools-content-types",
|
|
@@ -239,7 +262,8 @@ var TEST_DEFINITIONS = [
|
|
|
239
262
|
category: "tools",
|
|
240
263
|
required: false,
|
|
241
264
|
specRef: "server/tools#calling-tools",
|
|
242
|
-
description: "Validates that content items returned by tools/call have a recognized type field (text, image, audio, resource, resource_link)."
|
|
265
|
+
description: "Validates that content items returned by tools/call have a recognized type field (text, image, audio, resource, resource_link).",
|
|
266
|
+
recommendation: 'Every content item returned by tools/call must have a type field set to one of: "text", "image", "audio", "resource", or "resource_link". Check for typos or missing type fields.'
|
|
243
267
|
},
|
|
244
268
|
// ── Resources (5 tests) ──────────────────────────────────────────
|
|
245
269
|
{
|
|
@@ -248,7 +272,8 @@ var TEST_DEFINITIONS = [
|
|
|
248
272
|
category: "resources",
|
|
249
273
|
required: false,
|
|
250
274
|
specRef: "server/resources#listing-resources",
|
|
251
|
-
description: "Calls resources/list and validates it returns an array. Required if the server declares resources capability."
|
|
275
|
+
description: "Calls resources/list and validates it returns an array. Required if the server declares resources capability.",
|
|
276
|
+
recommendation: "Implement resources/list to return { resources: [...] } with an array of resource objects. Each resource needs at least a uri and name."
|
|
252
277
|
},
|
|
253
278
|
{
|
|
254
279
|
id: "resources-read",
|
|
@@ -256,7 +281,8 @@ var TEST_DEFINITIONS = [
|
|
|
256
281
|
category: "resources",
|
|
257
282
|
required: false,
|
|
258
283
|
specRef: "server/resources#reading-resources",
|
|
259
|
-
description: "Reads the first resource and validates the response contains a contents array with proper uri and text/blob fields."
|
|
284
|
+
description: "Reads the first resource and validates the response contains a contents array with proper uri and text/blob fields.",
|
|
285
|
+
recommendation: "Implement resources/read to return { contents: [...] } where each item has a uri and either a text or blob field. Ensure the uri matches the requested resource."
|
|
260
286
|
},
|
|
261
287
|
{
|
|
262
288
|
id: "resources-templates",
|
|
@@ -264,7 +290,8 @@ var TEST_DEFINITIONS = [
|
|
|
264
290
|
category: "resources",
|
|
265
291
|
required: false,
|
|
266
292
|
specRef: "server/resources#resource-templates",
|
|
267
|
-
description: "Tests the resource templates endpoint. Accepts Method not found (-32601) since templates are optional."
|
|
293
|
+
description: "Tests the resource templates endpoint. Accepts Method not found (-32601) since templates are optional.",
|
|
294
|
+
recommendation: "If your server supports resource templates, implement resources/templates/list returning { resourceTemplates: [...] }. Otherwise, return error code -32601."
|
|
268
295
|
},
|
|
269
296
|
{
|
|
270
297
|
id: "resources-pagination",
|
|
@@ -272,7 +299,8 @@ var TEST_DEFINITIONS = [
|
|
|
272
299
|
category: "resources",
|
|
273
300
|
required: false,
|
|
274
301
|
specRef: "server/resources#listing-resources",
|
|
275
|
-
description: "Tests cursor-based pagination on resources/list. Validates nextCursor is a string if present and that fetching the next page works."
|
|
302
|
+
description: "Tests cursor-based pagination on resources/list. Validates nextCursor is a string if present and that fetching the next page works.",
|
|
303
|
+
recommendation: "If you return nextCursor in resources/list, ensure it is a string and that passing it back as cursor in the next request returns valid results."
|
|
276
304
|
},
|
|
277
305
|
{
|
|
278
306
|
id: "resources-subscribe",
|
|
@@ -280,7 +308,8 @@ var TEST_DEFINITIONS = [
|
|
|
280
308
|
category: "resources",
|
|
281
309
|
required: false,
|
|
282
310
|
specRef: "server/resources#subscriptions",
|
|
283
|
-
description: "If the server declares resources.subscribe capability, tests that resources/subscribe and resources/unsubscribe methods are accepted."
|
|
311
|
+
description: "If the server declares resources.subscribe capability, tests that resources/subscribe and resources/unsubscribe methods are accepted.",
|
|
312
|
+
recommendation: "If you declare resources.subscribe capability, implement both resources/subscribe and resources/unsubscribe handlers. Both should accept a uri parameter."
|
|
284
313
|
},
|
|
285
314
|
// ── Prompts (3 tests) ────────────────────────────────────────────
|
|
286
315
|
{
|
|
@@ -289,7 +318,8 @@ var TEST_DEFINITIONS = [
|
|
|
289
318
|
category: "prompts",
|
|
290
319
|
required: false,
|
|
291
320
|
specRef: "server/prompts#listing-prompts",
|
|
292
|
-
description: "Calls prompts/list and validates it returns an array. Required if the server declares prompts capability."
|
|
321
|
+
description: "Calls prompts/list and validates it returns an array. Required if the server declares prompts capability.",
|
|
322
|
+
recommendation: "Implement prompts/list to return { prompts: [...] } with an array of prompt objects. Each prompt needs at least a name field."
|
|
293
323
|
},
|
|
294
324
|
{
|
|
295
325
|
id: "prompts-get",
|
|
@@ -297,7 +327,8 @@ var TEST_DEFINITIONS = [
|
|
|
297
327
|
category: "prompts",
|
|
298
328
|
required: false,
|
|
299
329
|
specRef: "server/prompts#getting-a-prompt",
|
|
300
|
-
description: "Gets the first prompt and validates the response contains a messages array with proper role and content fields."
|
|
330
|
+
description: "Gets the first prompt and validates the response contains a messages array with proper role and content fields.",
|
|
331
|
+
recommendation: 'Implement prompts/get to return { messages: [...] } where each message has a role ("user" or "assistant") and a content field.'
|
|
301
332
|
},
|
|
302
333
|
{
|
|
303
334
|
id: "prompts-pagination",
|
|
@@ -305,7 +336,8 @@ var TEST_DEFINITIONS = [
|
|
|
305
336
|
category: "prompts",
|
|
306
337
|
required: false,
|
|
307
338
|
specRef: "server/prompts#listing-prompts",
|
|
308
|
-
description: "Tests cursor-based pagination on prompts/list. Validates nextCursor is a string if present and that fetching the next page works."
|
|
339
|
+
description: "Tests cursor-based pagination on prompts/list. Validates nextCursor is a string if present and that fetching the next page works.",
|
|
340
|
+
recommendation: "If you return nextCursor in prompts/list, ensure it is a string and that passing it back as cursor in the next request returns valid results."
|
|
309
341
|
},
|
|
310
342
|
// ── Error Handling (8 tests) ─────────────────────────────────────
|
|
311
343
|
{
|
|
@@ -314,7 +346,8 @@ var TEST_DEFINITIONS = [
|
|
|
314
346
|
category: "errors",
|
|
315
347
|
required: true,
|
|
316
348
|
specRef: "basic",
|
|
317
|
-
description: "Sends an unknown method and verifies the server returns a JSON-RPC error. The spec requires error code -32601 (Method not found)."
|
|
349
|
+
description: "Sends an unknown method and verifies the server returns a JSON-RPC error. The spec requires error code -32601 (Method not found).",
|
|
350
|
+
recommendation: "Return a JSON-RPC error with code -32601 (Method not found) for any unrecognized method name. Do not silently ignore unknown methods."
|
|
318
351
|
},
|
|
319
352
|
{
|
|
320
353
|
id: "error-method-code",
|
|
@@ -322,7 +355,8 @@ var TEST_DEFINITIONS = [
|
|
|
322
355
|
category: "errors",
|
|
323
356
|
required: false,
|
|
324
357
|
specRef: "basic",
|
|
325
|
-
description: "Checks the error code is specifically -32601 (Method not found) for unknown methods, as required by JSON-RPC 2.0."
|
|
358
|
+
description: "Checks the error code is specifically -32601 (Method not found) for unknown methods, as required by JSON-RPC 2.0.",
|
|
359
|
+
recommendation: "Use exactly error code -32601 for unknown methods. Do not use generic error codes like -32000. This is required by JSON-RPC 2.0."
|
|
326
360
|
},
|
|
327
361
|
{
|
|
328
362
|
id: "error-invalid-jsonrpc",
|
|
@@ -330,7 +364,8 @@ var TEST_DEFINITIONS = [
|
|
|
330
364
|
category: "errors",
|
|
331
365
|
required: true,
|
|
332
366
|
specRef: "basic",
|
|
333
|
-
description: "Sends a malformed JSON-RPC message (missing required fields) and verifies the server returns an error or 4xx status."
|
|
367
|
+
description: "Sends a malformed JSON-RPC message (missing required fields) and verifies the server returns an error or 4xx status.",
|
|
368
|
+
recommendation: "Validate incoming JSON-RPC messages for required fields (jsonrpc, method). Return error code -32600 (Invalid Request) or HTTP 400 for malformed messages."
|
|
334
369
|
},
|
|
335
370
|
{
|
|
336
371
|
id: "error-invalid-json",
|
|
@@ -338,7 +373,8 @@ var TEST_DEFINITIONS = [
|
|
|
338
373
|
category: "errors",
|
|
339
374
|
required: false,
|
|
340
375
|
specRef: "basic",
|
|
341
|
-
description: "Sends invalid JSON and verifies the server returns a parse error (-32700) or 4xx status code."
|
|
376
|
+
description: "Sends invalid JSON and verifies the server returns a parse error (-32700) or 4xx status code.",
|
|
377
|
+
recommendation: "Catch JSON parse errors and return error code -32700 (Parse error) with a descriptive message. Do not return 500 for malformed input."
|
|
342
378
|
},
|
|
343
379
|
{
|
|
344
380
|
id: "error-missing-params",
|
|
@@ -346,7 +382,8 @@ var TEST_DEFINITIONS = [
|
|
|
346
382
|
category: "errors",
|
|
347
383
|
required: false,
|
|
348
384
|
specRef: "server/tools#error-handling",
|
|
349
|
-
description: "Calls tools/call with an empty params object (missing required name field) and verifies an error is returned."
|
|
385
|
+
description: "Calls tools/call with an empty params object (missing required name field) and verifies an error is returned.",
|
|
386
|
+
recommendation: "Validate tools/call params and return error code -32602 (Invalid params) when the required name field is missing."
|
|
350
387
|
},
|
|
351
388
|
{
|
|
352
389
|
id: "error-parse-code",
|
|
@@ -354,7 +391,8 @@ var TEST_DEFINITIONS = [
|
|
|
354
391
|
category: "errors",
|
|
355
392
|
required: false,
|
|
356
393
|
specRef: "basic",
|
|
357
|
-
description: "Checks that the server returns the specific JSON-RPC error code -32700 (Parse error) when receiving invalid JSON, as required by the JSON-RPC 2.0 specification."
|
|
394
|
+
description: "Checks that the server returns the specific JSON-RPC error code -32700 (Parse error) when receiving invalid JSON, as required by the JSON-RPC 2.0 specification.",
|
|
395
|
+
recommendation: "Return exactly error code -32700 for JSON parse failures. Most JSON-RPC frameworks handle this automatically \u2014 check yours does not override the code."
|
|
358
396
|
},
|
|
359
397
|
{
|
|
360
398
|
id: "error-invalid-request-code",
|
|
@@ -362,7 +400,8 @@ var TEST_DEFINITIONS = [
|
|
|
362
400
|
category: "errors",
|
|
363
401
|
required: false,
|
|
364
402
|
specRef: "basic",
|
|
365
|
-
description: "Checks that the server returns the specific JSON-RPC error code -32600 (Invalid Request) for malformed JSON-RPC messages missing required fields."
|
|
403
|
+
description: "Checks that the server returns the specific JSON-RPC error code -32600 (Invalid Request) for malformed JSON-RPC messages missing required fields.",
|
|
404
|
+
recommendation: "Return exactly error code -32600 for structurally invalid JSON-RPC messages (e.g., missing method field). Check your JSON-RPC middleware configuration."
|
|
366
405
|
},
|
|
367
406
|
{
|
|
368
407
|
id: "tools-call-unknown",
|
|
@@ -370,7 +409,8 @@ var TEST_DEFINITIONS = [
|
|
|
370
409
|
category: "errors",
|
|
371
410
|
required: false,
|
|
372
411
|
specRef: "server/tools#error-handling",
|
|
373
|
-
description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response."
|
|
412
|
+
description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response.",
|
|
413
|
+
recommendation: "Return a JSON-RPC error or set isError: true when tools/call receives an unrecognized tool name. Do not return an empty success response."
|
|
374
414
|
},
|
|
375
415
|
// ── Schema Validation (6 tests) ──────────────────────────────────
|
|
376
416
|
{
|
|
@@ -379,7 +419,8 @@ var TEST_DEFINITIONS = [
|
|
|
379
419
|
category: "schema",
|
|
380
420
|
required: false,
|
|
381
421
|
specRef: "server/tools#data-types",
|
|
382
|
-
description: 'Validates every tool has a valid name (1-128 chars, alphanumeric/underscore/hyphen/dot) and a required inputSchema of type "object".'
|
|
422
|
+
description: 'Validates every tool has a valid name (1-128 chars, alphanumeric/underscore/hyphen/dot) and a required inputSchema of type "object".',
|
|
423
|
+
recommendation: 'Ensure every tool has a name (1-128 chars, [A-Za-z0-9_.-]) and an inputSchema with type: "object". Add descriptions to tools for better AI assistant integration.'
|
|
383
424
|
},
|
|
384
425
|
{
|
|
385
426
|
id: "tools-annotations",
|
|
@@ -387,7 +428,8 @@ var TEST_DEFINITIONS = [
|
|
|
387
428
|
category: "schema",
|
|
388
429
|
required: false,
|
|
389
430
|
specRef: "server/tools#annotations",
|
|
390
|
-
description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans; title should be a string."
|
|
431
|
+
description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans; title should be a string.",
|
|
432
|
+
recommendation: "If you include annotations on tools, ensure readOnlyHint, destructiveHint, idempotentHint, and openWorldHint are booleans. Title must be a string."
|
|
391
433
|
},
|
|
392
434
|
{
|
|
393
435
|
id: "tools-title-field",
|
|
@@ -395,7 +437,8 @@ var TEST_DEFINITIONS = [
|
|
|
395
437
|
category: "schema",
|
|
396
438
|
required: false,
|
|
397
439
|
specRef: "server/tools#data-types",
|
|
398
|
-
description: "Checks if tools include the optional title field for human-readable display names. Added in spec version 2025-11-25."
|
|
440
|
+
description: "Checks if tools include the optional title field for human-readable display names. Added in spec version 2025-11-25.",
|
|
441
|
+
recommendation: "Add a title field (human-readable string) to each tool definition. This helps MCP clients display your tools in a user-friendly way."
|
|
399
442
|
},
|
|
400
443
|
{
|
|
401
444
|
id: "tools-output-schema",
|
|
@@ -403,7 +446,8 @@ var TEST_DEFINITIONS = [
|
|
|
403
446
|
category: "schema",
|
|
404
447
|
required: false,
|
|
405
448
|
specRef: "server/tools#structured-content",
|
|
406
|
-
description: 'If tools declare an outputSchema, validates it is a valid JSON Schema object with type "object". Used for structured output validation.'
|
|
449
|
+
description: 'If tools declare an outputSchema, validates it is a valid JSON Schema object with type "object". Used for structured output validation.',
|
|
450
|
+
recommendation: 'If you declare outputSchema on a tool, ensure it is a valid JSON Schema object with type: "object". Remove outputSchema if you do not need structured output.'
|
|
407
451
|
},
|
|
408
452
|
{
|
|
409
453
|
id: "prompts-schema",
|
|
@@ -411,7 +455,8 @@ var TEST_DEFINITIONS = [
|
|
|
411
455
|
category: "schema",
|
|
412
456
|
required: false,
|
|
413
457
|
specRef: "server/prompts#data-types",
|
|
414
|
-
description: "Validates every prompt has a name and that any arguments array contains items with name fields."
|
|
458
|
+
description: "Validates every prompt has a name and that any arguments array contains items with name fields.",
|
|
459
|
+
recommendation: "Ensure every prompt has a name field. If the prompt has arguments, each argument object must include a name field."
|
|
415
460
|
},
|
|
416
461
|
{
|
|
417
462
|
id: "resources-schema",
|
|
@@ -419,7 +464,8 @@ var TEST_DEFINITIONS = [
|
|
|
419
464
|
category: "schema",
|
|
420
465
|
required: false,
|
|
421
466
|
specRef: "server/resources#data-types",
|
|
422
|
-
description: "Validates every resource has a valid URI (parseable as a URL) and a name field."
|
|
467
|
+
description: "Validates every resource has a valid URI (parseable as a URL) and a name field.",
|
|
468
|
+
recommendation: "Ensure every resource has a valid, parseable URI and a name field. Add description and mimeType for better client integration."
|
|
423
469
|
}
|
|
424
470
|
];
|
|
425
471
|
|
|
@@ -762,13 +808,13 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
762
808
|
true,
|
|
763
809
|
"basic/lifecycle#version-negotiation",
|
|
764
810
|
async () => {
|
|
765
|
-
const
|
|
766
|
-
if (!
|
|
767
|
-
const valid = /^\d{4}-\d{2}-\d{2}$/.test(
|
|
768
|
-
if (valid &&
|
|
769
|
-
warnings.push(`Server negotiated protocol version ${
|
|
811
|
+
const version3 = initRes?.body?.result?.protocolVersion;
|
|
812
|
+
if (!version3) return { passed: false, details: "No protocolVersion" };
|
|
813
|
+
const valid = /^\d{4}-\d{2}-\d{2}$/.test(version3);
|
|
814
|
+
if (valid && version3 !== SPEC_VERSION) {
|
|
815
|
+
warnings.push(`Server negotiated protocol version ${version3} (latest is ${SPEC_VERSION})`);
|
|
770
816
|
}
|
|
771
|
-
return { passed: valid, details: `Version: ${
|
|
817
|
+
return { passed: valid, details: `Version: ${version3}` };
|
|
772
818
|
}
|
|
773
819
|
);
|
|
774
820
|
await test(
|
|
@@ -1737,7 +1783,9 @@ ${TEST_DEFINITIONS.map((t) => t.id).join(", ")}`
|
|
|
1737
1783
|
`Required: ${def.required ? "Yes" : "No"}`,
|
|
1738
1784
|
`Spec reference: https://modelcontextprotocol.io/specification/2025-11-25/${def.specRef}`,
|
|
1739
1785
|
"",
|
|
1740
|
-
def.description
|
|
1786
|
+
def.description,
|
|
1787
|
+
"",
|
|
1788
|
+
`Fix: ${def.recommendation}`
|
|
1741
1789
|
].join("\n")
|
|
1742
1790
|
}
|
|
1743
1791
|
]
|
|
@@ -1746,6 +1794,27 @@ ${TEST_DEFINITIONS.map((t) => t.id).join(", ")}`
|
|
|
1746
1794
|
);
|
|
1747
1795
|
}
|
|
1748
1796
|
|
|
1797
|
+
// src/mcp/server.ts
|
|
1798
|
+
var require2 = createRequire2(import.meta.url);
|
|
1799
|
+
var { version } = require2("../../package.json");
|
|
1800
|
+
function createComplianceServer() {
|
|
1801
|
+
const server = new McpServer({ name: "mcp-compliance", version });
|
|
1802
|
+
registerTools(server);
|
|
1803
|
+
return server;
|
|
1804
|
+
}
|
|
1805
|
+
async function startServer() {
|
|
1806
|
+
const server = createComplianceServer();
|
|
1807
|
+
const transport = new StdioServerTransport();
|
|
1808
|
+
await server.connect(transport);
|
|
1809
|
+
}
|
|
1810
|
+
var isDirectRun = process.argv[1]?.endsWith("mcp/server.js") || process.argv[1]?.endsWith("mcp\\server.js");
|
|
1811
|
+
if (isDirectRun) {
|
|
1812
|
+
startServer().catch((err) => {
|
|
1813
|
+
console.error("MCP server error:", err);
|
|
1814
|
+
process.exit(1);
|
|
1815
|
+
});
|
|
1816
|
+
}
|
|
1817
|
+
|
|
1749
1818
|
// src/reporter.ts
|
|
1750
1819
|
import chalk from "chalk";
|
|
1751
1820
|
var CATEGORY_LABELS = {
|
|
@@ -1788,8 +1857,16 @@ function testLine(t) {
|
|
|
1788
1857
|
const icon = t.passed ? chalk.green(" PASS") : chalk.red(" FAIL");
|
|
1789
1858
|
const req = t.required ? chalk.dim(" (required)") : "";
|
|
1790
1859
|
const dur = chalk.dim(` ${t.durationMs}ms`);
|
|
1791
|
-
|
|
1860
|
+
let line = `${icon} ${t.name}${req}${dur}
|
|
1792
1861
|
${chalk.dim(` ${t.details}`)}`;
|
|
1862
|
+
if (!t.passed) {
|
|
1863
|
+
const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
|
|
1864
|
+
if (def?.recommendation) {
|
|
1865
|
+
line += `
|
|
1866
|
+
${chalk.cyan(` Fix: ${def.recommendation}`)}`;
|
|
1867
|
+
}
|
|
1868
|
+
}
|
|
1869
|
+
return line;
|
|
1793
1870
|
}
|
|
1794
1871
|
function formatTerminal(report) {
|
|
1795
1872
|
const lines = [];
|
|
@@ -1872,10 +1949,80 @@ function formatTerminal(report) {
|
|
|
1872
1949
|
function formatJson(report) {
|
|
1873
1950
|
return JSON.stringify(report, null, 2);
|
|
1874
1951
|
}
|
|
1952
|
+
function formatSarif(report) {
|
|
1953
|
+
const SPEC_BASE2 = `https://modelcontextprotocol.io/specification/${report.specVersion}`;
|
|
1954
|
+
const rules = report.tests.map((t) => {
|
|
1955
|
+
const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
|
|
1956
|
+
return {
|
|
1957
|
+
id: t.id,
|
|
1958
|
+
name: t.name,
|
|
1959
|
+
shortDescription: { text: t.name },
|
|
1960
|
+
fullDescription: { text: def?.description || t.details },
|
|
1961
|
+
helpUri: t.specRef || `${SPEC_BASE2}/basic`,
|
|
1962
|
+
properties: {
|
|
1963
|
+
category: t.category,
|
|
1964
|
+
required: t.required
|
|
1965
|
+
}
|
|
1966
|
+
};
|
|
1967
|
+
});
|
|
1968
|
+
const results = report.tests.filter((t) => !t.passed).map((t) => {
|
|
1969
|
+
const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
|
|
1970
|
+
return {
|
|
1971
|
+
ruleId: t.id,
|
|
1972
|
+
level: t.required ? "error" : "warning",
|
|
1973
|
+
message: {
|
|
1974
|
+
text: def?.recommendation ? `${t.details}. Fix: ${def.recommendation}` : t.details
|
|
1975
|
+
},
|
|
1976
|
+
locations: [
|
|
1977
|
+
{
|
|
1978
|
+
physicalLocation: {
|
|
1979
|
+
artifactLocation: {
|
|
1980
|
+
uri: report.url,
|
|
1981
|
+
uriBaseId: "MCP_SERVER"
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
}
|
|
1985
|
+
],
|
|
1986
|
+
properties: {
|
|
1987
|
+
category: t.category,
|
|
1988
|
+
durationMs: t.durationMs
|
|
1989
|
+
}
|
|
1990
|
+
};
|
|
1991
|
+
});
|
|
1992
|
+
const sarif = {
|
|
1993
|
+
$schema: "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json",
|
|
1994
|
+
version: "2.1.0",
|
|
1995
|
+
runs: [
|
|
1996
|
+
{
|
|
1997
|
+
tool: {
|
|
1998
|
+
driver: {
|
|
1999
|
+
name: "mcp-compliance",
|
|
2000
|
+
version: report.toolVersion,
|
|
2001
|
+
informationUri: "https://github.com/YawLabs/mcp-compliance",
|
|
2002
|
+
rules
|
|
2003
|
+
}
|
|
2004
|
+
},
|
|
2005
|
+
results,
|
|
2006
|
+
invocations: [
|
|
2007
|
+
{
|
|
2008
|
+
executionSuccessful: report.overall !== "fail",
|
|
2009
|
+
properties: {
|
|
2010
|
+
grade: report.grade,
|
|
2011
|
+
score: report.score,
|
|
2012
|
+
overall: report.overall,
|
|
2013
|
+
specVersion: report.specVersion
|
|
2014
|
+
}
|
|
2015
|
+
}
|
|
2016
|
+
]
|
|
2017
|
+
}
|
|
2018
|
+
]
|
|
2019
|
+
};
|
|
2020
|
+
return JSON.stringify(sarif, null, 2);
|
|
2021
|
+
}
|
|
1875
2022
|
|
|
1876
2023
|
// src/index.ts
|
|
1877
|
-
var
|
|
1878
|
-
var { version } =
|
|
2024
|
+
var require3 = createRequire3(import.meta.url);
|
|
2025
|
+
var { version: version2 } = require3("../package.json");
|
|
1879
2026
|
function parseHeaderArg(value, prev) {
|
|
1880
2027
|
const idx = value.indexOf(":");
|
|
1881
2028
|
if (idx === -1) {
|
|
@@ -1890,8 +2037,8 @@ function parseList(value) {
|
|
|
1890
2037
|
return value.split(",").map((s) => s.trim()).filter(Boolean);
|
|
1891
2038
|
}
|
|
1892
2039
|
var program = new Command();
|
|
1893
|
-
program.name("mcp-compliance").description("Test MCP servers for spec compliance").version(
|
|
1894
|
-
program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal or
|
|
2040
|
+
program.name("mcp-compliance").description("Test MCP servers for spec compliance").version(version2);
|
|
2041
|
+
program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal, json, or sarif", "terminal").option("--strict", "Exit with code 1 on any required test failure (for CI)").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").option("--retries <n>", "Number of retries for failed tests", "0").option("--only <items>", "Only run tests matching these categories or test IDs (comma-separated)", parseList).option("--skip <items>", "Skip tests matching these categories or test IDs (comma-separated)", parseList).option("--verbose", "Print each test result as it runs").action(
|
|
1895
2042
|
async (url, opts) => {
|
|
1896
2043
|
try {
|
|
1897
2044
|
const headers = { ...opts.header };
|
|
@@ -1917,6 +2064,8 @@ Testing ${url}...
|
|
|
1917
2064
|
}
|
|
1918
2065
|
if (opts.format === "json") {
|
|
1919
2066
|
console.log(formatJson(report));
|
|
2067
|
+
} else if (opts.format === "sarif") {
|
|
2068
|
+
console.log(formatSarif(report));
|
|
1920
2069
|
} else {
|
|
1921
2070
|
console.log(formatTerminal(report));
|
|
1922
2071
|
}
|
|
@@ -1924,7 +2073,7 @@ Testing ${url}...
|
|
|
1924
2073
|
process.exit(1);
|
|
1925
2074
|
}
|
|
1926
2075
|
} catch (err) {
|
|
1927
|
-
if (opts.format === "json") {
|
|
2076
|
+
if (opts.format === "json" || opts.format === "sarif") {
|
|
1928
2077
|
console.error(JSON.stringify({ error: err.message }));
|
|
1929
2078
|
} else {
|
|
1930
2079
|
console.error(chalk2.red(`
|
|
@@ -1958,9 +2107,6 @@ Error: ${err.message}
|
|
|
1958
2107
|
}
|
|
1959
2108
|
});
|
|
1960
2109
|
program.command("mcp").description("Start the MCP compliance server (stdio transport)").action(async () => {
|
|
1961
|
-
|
|
1962
|
-
registerTools(server);
|
|
1963
|
-
const transport = new StdioServerTransport();
|
|
1964
|
-
await server.connect(transport);
|
|
2110
|
+
await startServer();
|
|
1965
2111
|
});
|
|
1966
2112
|
program.parse();
|
package/dist/mcp/server.d.ts
CHANGED
|
@@ -1,2 +1,12 @@
|
|
|
1
|
+
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
1
2
|
|
|
2
|
-
|
|
3
|
+
/**
|
|
4
|
+
* Create and configure the MCP compliance server instance.
|
|
5
|
+
*/
|
|
6
|
+
declare function createComplianceServer(): McpServer;
|
|
7
|
+
/**
|
|
8
|
+
* Start the MCP compliance server with stdio transport.
|
|
9
|
+
*/
|
|
10
|
+
declare function startServer(): Promise<void>;
|
|
11
|
+
|
|
12
|
+
export { createComplianceServer, startServer };
|