@yawlabs/mcp-compliance 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +152 -24
- package/dist/chunk-KNOSZ3TD.js +1621 -0
- package/dist/index.js +1495 -687
- package/dist/mcp/server.d.ts +11 -1
- package/dist/mcp/server.js +109 -47
- package/dist/runner.d.ts +5 -4
- package/dist/runner.js +1 -1
- package/package.json +22 -7
- package/dist/chunk-SP24UFRC.js +0 -987
package/dist/index.js
CHANGED
|
@@ -1,13 +1,40 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import {
|
|
4
|
+
import { createRequire as createRequire3 } from "module";
|
|
5
5
|
import chalk2 from "chalk";
|
|
6
|
+
import { Command } from "commander";
|
|
7
|
+
|
|
8
|
+
// src/mcp/server.ts
|
|
6
9
|
import { createRequire as createRequire2 } from "module";
|
|
10
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
11
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
12
|
+
|
|
13
|
+
// src/mcp/tools.ts
|
|
14
|
+
import { z } from "zod";
|
|
7
15
|
|
|
8
16
|
// src/runner.ts
|
|
9
|
-
import { request } from "undici";
|
|
10
17
|
import { createRequire } from "module";
|
|
18
|
+
import { request } from "undici";
|
|
19
|
+
|
|
20
|
+
// src/badge.ts
|
|
21
|
+
function generateBadge(url) {
|
|
22
|
+
let parsed;
|
|
23
|
+
try {
|
|
24
|
+
parsed = new URL(url);
|
|
25
|
+
} catch {
|
|
26
|
+
parsed = new URL("https://unknown");
|
|
27
|
+
}
|
|
28
|
+
const encoded = encodeURIComponent(parsed.href);
|
|
29
|
+
const imageUrl = `https://mcp.hosting/api/compliance/${encoded}/badge`;
|
|
30
|
+
const reportUrl = `https://mcp.hosting/compliance/${encoded}`;
|
|
31
|
+
return {
|
|
32
|
+
imageUrl,
|
|
33
|
+
reportUrl,
|
|
34
|
+
markdown: `[](${reportUrl})`,
|
|
35
|
+
html: `<a href="${reportUrl}"><img src="${imageUrl}" alt="MCP Compliant"></a>`
|
|
36
|
+
};
|
|
37
|
+
}
|
|
11
38
|
|
|
12
39
|
// src/grader.ts
|
|
13
40
|
function computeGrade(score) {
|
|
@@ -44,77 +71,402 @@ function computeScore(tests) {
|
|
|
44
71
|
};
|
|
45
72
|
}
|
|
46
73
|
|
|
47
|
-
// src/badge.ts
|
|
48
|
-
function generateBadge(url) {
|
|
49
|
-
let parsed;
|
|
50
|
-
try {
|
|
51
|
-
parsed = new URL(url);
|
|
52
|
-
} catch {
|
|
53
|
-
parsed = new URL("https://unknown");
|
|
54
|
-
}
|
|
55
|
-
const encoded = encodeURIComponent(parsed.href);
|
|
56
|
-
const imageUrl = `https://mcp.hosting/api/compliance/${encoded}/badge`;
|
|
57
|
-
const reportUrl = `https://mcp.hosting/compliance/${encoded}`;
|
|
58
|
-
return {
|
|
59
|
-
imageUrl,
|
|
60
|
-
reportUrl,
|
|
61
|
-
markdown: `[](${reportUrl})`,
|
|
62
|
-
html: `<a href="${reportUrl}"><img src="${imageUrl}" alt="MCP Compliant"></a>`
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
|
|
66
74
|
// src/types.ts
|
|
67
75
|
var TEST_DEFINITIONS = [
|
|
68
76
|
// ── Transport (7 tests) ──────────────────────────────────────────
|
|
69
|
-
{
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
77
|
+
{
|
|
78
|
+
id: "transport-post",
|
|
79
|
+
name: "HTTP POST accepted",
|
|
80
|
+
category: "transport",
|
|
81
|
+
required: true,
|
|
82
|
+
specRef: "basic/transports#streamable-http",
|
|
83
|
+
description: "Verifies the server accepts HTTP POST requests and returns a 2xx status code. This is the fundamental transport requirement for Streamable HTTP MCP servers.",
|
|
84
|
+
recommendation: "Ensure your server listens for POST requests on the MCP endpoint. If you see 401/403, pass --auth with a valid token. Check that the URL is correct and the server is running."
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
id: "transport-content-type",
|
|
88
|
+
name: "Responds with JSON or SSE",
|
|
89
|
+
category: "transport",
|
|
90
|
+
required: true,
|
|
91
|
+
specRef: "basic/transports#streamable-http",
|
|
92
|
+
description: "Checks that the server responds with Content-Type application/json or text/event-stream. MCP servers must use one of these two content types.",
|
|
93
|
+
recommendation: 'Set the Content-Type response header to "application/json" for synchronous responses or "text/event-stream" for streaming. Do not use text/html or other types.'
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
id: "transport-notification-202",
|
|
97
|
+
name: "Notification returns 202 Accepted",
|
|
98
|
+
category: "transport",
|
|
99
|
+
required: false,
|
|
100
|
+
specRef: "basic/transports#streamable-http",
|
|
101
|
+
description: "Verifies that sending a JSON-RPC notification (no id field) returns HTTP 202 Accepted with no body. Per spec, servers MUST return 202 for notifications.",
|
|
102
|
+
recommendation: "Detect JSON-RPC messages without an id field and return HTTP 202 with an empty body. Do not attempt to send a JSON-RPC response for notifications."
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
id: "transport-session-id",
|
|
106
|
+
name: "Enforces MCP-Session-Id after init",
|
|
107
|
+
category: "transport",
|
|
108
|
+
required: false,
|
|
109
|
+
specRef: "basic/transports#streamable-http",
|
|
110
|
+
description: "Tests that the server returns HTTP 400 when MCP-Session-Id header is missing on requests after initialization (when the server issued a session ID).",
|
|
111
|
+
recommendation: "If your server issues an MCP-Session-Id header in the initialize response, reject subsequent requests that omit this header with HTTP 400."
|
|
112
|
+
},
|
|
113
|
+
{
|
|
114
|
+
id: "transport-get",
|
|
115
|
+
name: "GET returns SSE stream or 405",
|
|
116
|
+
category: "transport",
|
|
117
|
+
required: false,
|
|
118
|
+
specRef: "basic/transports#streamable-http",
|
|
119
|
+
description: "Tests the GET endpoint for server-initiated messages. Server should return text/event-stream or 405 Method Not Allowed.",
|
|
120
|
+
recommendation: "If your server supports server-initiated messages, handle GET with text/event-stream. Otherwise, return 405 Method Not Allowed."
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
id: "transport-delete",
|
|
124
|
+
name: "DELETE accepted or returns 405",
|
|
125
|
+
category: "transport",
|
|
126
|
+
required: false,
|
|
127
|
+
specRef: "basic/transports#streamable-http",
|
|
128
|
+
description: "Tests the DELETE endpoint for session termination. Server should accept the request or return 405 Method Not Allowed.",
|
|
129
|
+
recommendation: "Handle DELETE requests for session cleanup, or return 405 if session termination is not supported. Do not return 500."
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
id: "transport-batch-reject",
|
|
133
|
+
name: "Rejects JSON-RPC batch requests",
|
|
134
|
+
category: "transport",
|
|
135
|
+
required: true,
|
|
136
|
+
specRef: "basic/transports#streamable-http",
|
|
137
|
+
description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests.",
|
|
138
|
+
recommendation: "Check if the parsed JSON body is an array. If so, return a JSON-RPC error or HTTP 400. Do not process batch requests \u2014 MCP explicitly forbids them."
|
|
139
|
+
},
|
|
76
140
|
// ── Lifecycle (10 tests) ─────────────────────────────────────────
|
|
77
|
-
{
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
{
|
|
141
|
+
{
|
|
142
|
+
id: "lifecycle-init",
|
|
143
|
+
name: "Initialize handshake",
|
|
144
|
+
category: "lifecycle",
|
|
145
|
+
required: true,
|
|
146
|
+
specRef: "basic/lifecycle#initialization",
|
|
147
|
+
description: "Tests the initialize handshake by sending an initialize request with client capabilities. The server must return a result with protocolVersion.",
|
|
148
|
+
recommendation: 'Implement the "initialize" method handler. Return a result object with at least protocolVersion, capabilities, and serverInfo fields.'
|
|
149
|
+
},
|
|
150
|
+
{
|
|
151
|
+
id: "lifecycle-proto-version",
|
|
152
|
+
name: "Returns valid protocol version",
|
|
153
|
+
category: "lifecycle",
|
|
154
|
+
required: true,
|
|
155
|
+
specRef: "basic/lifecycle#version-negotiation",
|
|
156
|
+
description: "Validates that the protocolVersion returned by the server matches the YYYY-MM-DD date format required by the spec.",
|
|
157
|
+
recommendation: `Return protocolVersion as a YYYY-MM-DD string (e.g., "2025-11-25"). The server should negotiate based on the client's requested version.`
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
id: "lifecycle-server-info",
|
|
161
|
+
name: "Includes serverInfo",
|
|
162
|
+
category: "lifecycle",
|
|
163
|
+
required: false,
|
|
164
|
+
specRef: "basic/lifecycle#initialization",
|
|
165
|
+
description: "Checks that the server includes a serverInfo object with at least a name field in its initialize response. While recommended, this is not strictly required.",
|
|
166
|
+
recommendation: 'Add a serverInfo object to your initialize response: { name: "your-server", version: "1.0.0" }. This helps clients identify your server.'
|
|
167
|
+
},
|
|
168
|
+
{
|
|
169
|
+
id: "lifecycle-capabilities",
|
|
170
|
+
name: "Returns capabilities object",
|
|
171
|
+
category: "lifecycle",
|
|
172
|
+
required: true,
|
|
173
|
+
specRef: "basic/lifecycle#capability-negotiation",
|
|
174
|
+
description: "Verifies the server returns a capabilities object in its initialize response. An empty object is valid (no optional features declared).",
|
|
175
|
+
recommendation: "Include a capabilities object in your initialize response. Declare the features your server supports (tools, resources, prompts, logging, etc.). An empty object {} is valid."
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
id: "lifecycle-jsonrpc",
|
|
179
|
+
name: "Response is valid JSON-RPC 2.0",
|
|
180
|
+
category: "lifecycle",
|
|
181
|
+
required: true,
|
|
182
|
+
specRef: "basic",
|
|
183
|
+
description: 'Validates that the initialize response is a proper JSON-RPC 2.0 message with jsonrpc="2.0", an id field, and either a result or error field.',
|
|
184
|
+
recommendation: 'Ensure every response includes jsonrpc: "2.0", the matching id from the request, and either a result or error field. Never omit the jsonrpc field.'
|
|
185
|
+
},
|
|
186
|
+
{
|
|
187
|
+
id: "lifecycle-ping",
|
|
188
|
+
name: "Responds to ping",
|
|
189
|
+
category: "lifecycle",
|
|
190
|
+
required: true,
|
|
191
|
+
specRef: "basic/utilities#ping",
|
|
192
|
+
description: "Tests that the server responds to the ping method with an empty result object. This is a required utility method.",
|
|
193
|
+
recommendation: 'Implement a "ping" method handler that returns an empty result object {}. This is required by the MCP spec for keepalive and connectivity checking.'
|
|
194
|
+
},
|
|
195
|
+
{
|
|
196
|
+
id: "lifecycle-instructions",
|
|
197
|
+
name: "Instructions field is valid",
|
|
198
|
+
category: "lifecycle",
|
|
199
|
+
required: false,
|
|
200
|
+
specRef: "basic/lifecycle#initialization",
|
|
201
|
+
description: "If the server includes an instructions field in the initialize response, validates it is a string. Instructions provide guidance for how the client should interact with the server.",
|
|
202
|
+
recommendation: "If you include an instructions field in the initialize response, ensure it is a string. Remove the field or fix the type if it is not a string."
|
|
203
|
+
},
|
|
204
|
+
{
|
|
205
|
+
id: "lifecycle-id-match",
|
|
206
|
+
name: "Response ID matches request ID",
|
|
207
|
+
category: "lifecycle",
|
|
208
|
+
required: true,
|
|
209
|
+
specRef: "basic",
|
|
210
|
+
description: "Verifies that the JSON-RPC response id matches the request id sent by the client. This is a fundamental JSON-RPC 2.0 requirement.",
|
|
211
|
+
recommendation: "Copy the id field from the request into the response. This is a core JSON-RPC 2.0 requirement. Check that your framework does not modify or discard the request ID."
|
|
212
|
+
},
|
|
213
|
+
{
|
|
214
|
+
id: "lifecycle-logging",
|
|
215
|
+
name: "logging/setLevel accepted",
|
|
216
|
+
category: "lifecycle",
|
|
217
|
+
required: false,
|
|
218
|
+
specRef: "server/utilities#logging",
|
|
219
|
+
description: "If the server declares logging capability, tests that logging/setLevel method is accepted with a valid log level.",
|
|
220
|
+
recommendation: 'If you declare logging in capabilities, implement the "logging/setLevel" handler. Accept standard log levels: debug, info, notice, warning, error, critical, alert, emergency.'
|
|
221
|
+
},
|
|
222
|
+
{
|
|
223
|
+
id: "lifecycle-completions",
|
|
224
|
+
name: "completion/complete accepted",
|
|
225
|
+
category: "lifecycle",
|
|
226
|
+
required: false,
|
|
227
|
+
specRef: "server/utilities#completion",
|
|
228
|
+
description: "If the server declares completions capability, tests that the completion/complete method is accepted.",
|
|
229
|
+
recommendation: 'If you declare completions in capabilities, implement the "completion/complete" handler. Return a completion object with a values array, even if empty.'
|
|
230
|
+
},
|
|
87
231
|
// ── Tools (4 tests) ──────────────────────────────────────────────
|
|
88
|
-
{
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
232
|
+
{
|
|
233
|
+
id: "tools-list",
|
|
234
|
+
name: "tools/list returns valid response",
|
|
235
|
+
category: "tools",
|
|
236
|
+
required: false,
|
|
237
|
+
specRef: "server/tools#listing-tools",
|
|
238
|
+
description: "Calls tools/list and validates it returns an array of tool definitions. Required if the server declares tools capability.",
|
|
239
|
+
recommendation: "Implement the tools/list handler to return { tools: [...] } with an array of tool definition objects. Each tool needs at least a name and inputSchema."
|
|
240
|
+
},
|
|
241
|
+
{
|
|
242
|
+
id: "tools-call",
|
|
243
|
+
name: "tools/call responds correctly",
|
|
244
|
+
category: "tools",
|
|
245
|
+
required: false,
|
|
246
|
+
specRef: "server/tools#calling-tools",
|
|
247
|
+
description: "Calls the first tool with empty arguments and verifies the response format. Accepts both successful results and InvalidParams errors.",
|
|
248
|
+
recommendation: "Ensure tools/call returns { content: [...] } with an array of content objects, each having a type field. Return isError: true for tool execution errors."
|
|
249
|
+
},
|
|
250
|
+
{
|
|
251
|
+
id: "tools-pagination",
|
|
252
|
+
name: "tools/list supports pagination",
|
|
253
|
+
category: "tools",
|
|
254
|
+
required: false,
|
|
255
|
+
specRef: "server/tools#listing-tools",
|
|
256
|
+
description: "Tests cursor-based pagination on tools/list. Validates nextCursor is a string if present and that fetching the next page returns a valid response.",
|
|
257
|
+
recommendation: "If your server has many tools, include a nextCursor string in the response. Ensure passing this cursor back in a subsequent request returns the next page."
|
|
258
|
+
},
|
|
259
|
+
{
|
|
260
|
+
id: "tools-content-types",
|
|
261
|
+
name: "Tool content items have valid types",
|
|
262
|
+
category: "tools",
|
|
263
|
+
required: false,
|
|
264
|
+
specRef: "server/tools#calling-tools",
|
|
265
|
+
description: "Validates that content items returned by tools/call have a recognized type field (text, image, audio, resource, resource_link).",
|
|
266
|
+
recommendation: 'Every content item returned by tools/call must have a type field set to one of: "text", "image", "audio", "resource", or "resource_link". Check for typos or missing type fields.'
|
|
267
|
+
},
|
|
92
268
|
// ── Resources (5 tests) ──────────────────────────────────────────
|
|
93
|
-
{
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
269
|
+
{
|
|
270
|
+
id: "resources-list",
|
|
271
|
+
name: "resources/list returns valid response",
|
|
272
|
+
category: "resources",
|
|
273
|
+
required: false,
|
|
274
|
+
specRef: "server/resources#listing-resources",
|
|
275
|
+
description: "Calls resources/list and validates it returns an array. Required if the server declares resources capability.",
|
|
276
|
+
recommendation: "Implement resources/list to return { resources: [...] } with an array of resource objects. Each resource needs at least a uri and name."
|
|
277
|
+
},
|
|
278
|
+
{
|
|
279
|
+
id: "resources-read",
|
|
280
|
+
name: "resources/read returns content",
|
|
281
|
+
category: "resources",
|
|
282
|
+
required: false,
|
|
283
|
+
specRef: "server/resources#reading-resources",
|
|
284
|
+
description: "Reads the first resource and validates the response contains a contents array with proper uri and text/blob fields.",
|
|
285
|
+
recommendation: "Implement resources/read to return { contents: [...] } where each item has a uri and either a text or blob field. Ensure the uri matches the requested resource."
|
|
286
|
+
},
|
|
287
|
+
{
|
|
288
|
+
id: "resources-templates",
|
|
289
|
+
name: "resources/templates/list returns valid response",
|
|
290
|
+
category: "resources",
|
|
291
|
+
required: false,
|
|
292
|
+
specRef: "server/resources#resource-templates",
|
|
293
|
+
description: "Tests the resource templates endpoint. Accepts Method not found (-32601) since templates are optional.",
|
|
294
|
+
recommendation: "If your server supports resource templates, implement resources/templates/list returning { resourceTemplates: [...] }. Otherwise, return error code -32601."
|
|
295
|
+
},
|
|
296
|
+
{
|
|
297
|
+
id: "resources-pagination",
|
|
298
|
+
name: "resources/list supports pagination",
|
|
299
|
+
category: "resources",
|
|
300
|
+
required: false,
|
|
301
|
+
specRef: "server/resources#listing-resources",
|
|
302
|
+
description: "Tests cursor-based pagination on resources/list. Validates nextCursor is a string if present and that fetching the next page works.",
|
|
303
|
+
recommendation: "If you return nextCursor in resources/list, ensure it is a string and that passing it back as cursor in the next request returns valid results."
|
|
304
|
+
},
|
|
305
|
+
{
|
|
306
|
+
id: "resources-subscribe",
|
|
307
|
+
name: "Resource subscribe/unsubscribe",
|
|
308
|
+
category: "resources",
|
|
309
|
+
required: false,
|
|
310
|
+
specRef: "server/resources#subscriptions",
|
|
311
|
+
description: "If the server declares resources.subscribe capability, tests that resources/subscribe and resources/unsubscribe methods are accepted.",
|
|
312
|
+
recommendation: "If you declare resources.subscribe capability, implement both resources/subscribe and resources/unsubscribe handlers. Both should accept a uri parameter."
|
|
313
|
+
},
|
|
98
314
|
// ── Prompts (3 tests) ────────────────────────────────────────────
|
|
99
|
-
{
|
|
100
|
-
|
|
101
|
-
|
|
315
|
+
{
|
|
316
|
+
id: "prompts-list",
|
|
317
|
+
name: "prompts/list returns valid response",
|
|
318
|
+
category: "prompts",
|
|
319
|
+
required: false,
|
|
320
|
+
specRef: "server/prompts#listing-prompts",
|
|
321
|
+
description: "Calls prompts/list and validates it returns an array. Required if the server declares prompts capability.",
|
|
322
|
+
recommendation: "Implement prompts/list to return { prompts: [...] } with an array of prompt objects. Each prompt needs at least a name field."
|
|
323
|
+
},
|
|
324
|
+
{
|
|
325
|
+
id: "prompts-get",
|
|
326
|
+
name: "prompts/get returns valid messages",
|
|
327
|
+
category: "prompts",
|
|
328
|
+
required: false,
|
|
329
|
+
specRef: "server/prompts#getting-a-prompt",
|
|
330
|
+
description: "Gets the first prompt and validates the response contains a messages array with proper role and content fields.",
|
|
331
|
+
recommendation: 'Implement prompts/get to return { messages: [...] } where each message has a role ("user" or "assistant") and a content field.'
|
|
332
|
+
},
|
|
333
|
+
{
|
|
334
|
+
id: "prompts-pagination",
|
|
335
|
+
name: "prompts/list supports pagination",
|
|
336
|
+
category: "prompts",
|
|
337
|
+
required: false,
|
|
338
|
+
specRef: "server/prompts#listing-prompts",
|
|
339
|
+
description: "Tests cursor-based pagination on prompts/list. Validates nextCursor is a string if present and that fetching the next page works.",
|
|
340
|
+
recommendation: "If you return nextCursor in prompts/list, ensure it is a string and that passing it back as cursor in the next request returns valid results."
|
|
341
|
+
},
|
|
102
342
|
// ── Error Handling (8 tests) ─────────────────────────────────────
|
|
103
|
-
{
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
343
|
+
{
|
|
344
|
+
id: "error-unknown-method",
|
|
345
|
+
name: "Returns JSON-RPC error for unknown method",
|
|
346
|
+
category: "errors",
|
|
347
|
+
required: true,
|
|
348
|
+
specRef: "basic",
|
|
349
|
+
description: "Sends an unknown method and verifies the server returns a JSON-RPC error. The spec requires error code -32601 (Method not found).",
|
|
350
|
+
recommendation: "Return a JSON-RPC error with code -32601 (Method not found) for any unrecognized method name. Do not silently ignore unknown methods."
|
|
351
|
+
},
|
|
352
|
+
{
|
|
353
|
+
id: "error-method-code",
|
|
354
|
+
name: "Uses correct JSON-RPC error code for unknown method",
|
|
355
|
+
category: "errors",
|
|
356
|
+
required: false,
|
|
357
|
+
specRef: "basic",
|
|
358
|
+
description: "Checks the error code is specifically -32601 (Method not found) for unknown methods, as required by JSON-RPC 2.0.",
|
|
359
|
+
recommendation: "Use exactly error code -32601 for unknown methods. Do not use generic error codes like -32000. This is required by JSON-RPC 2.0."
|
|
360
|
+
},
|
|
361
|
+
{
|
|
362
|
+
id: "error-invalid-jsonrpc",
|
|
363
|
+
name: "Handles malformed JSON-RPC",
|
|
364
|
+
category: "errors",
|
|
365
|
+
required: true,
|
|
366
|
+
specRef: "basic",
|
|
367
|
+
description: "Sends a malformed JSON-RPC message (missing required fields) and verifies the server returns an error or 4xx status.",
|
|
368
|
+
recommendation: "Validate incoming JSON-RPC messages for required fields (jsonrpc, method). Return error code -32600 (Invalid Request) or HTTP 400 for malformed messages."
|
|
369
|
+
},
|
|
370
|
+
{
|
|
371
|
+
id: "error-invalid-json",
|
|
372
|
+
name: "Handles invalid JSON body",
|
|
373
|
+
category: "errors",
|
|
374
|
+
required: false,
|
|
375
|
+
specRef: "basic",
|
|
376
|
+
description: "Sends invalid JSON and verifies the server returns a parse error (-32700) or 4xx status code.",
|
|
377
|
+
recommendation: "Catch JSON parse errors and return error code -32700 (Parse error) with a descriptive message. Do not return 500 for malformed input."
|
|
378
|
+
},
|
|
379
|
+
{
|
|
380
|
+
id: "error-missing-params",
|
|
381
|
+
name: "Returns error for tools/call without name",
|
|
382
|
+
category: "errors",
|
|
383
|
+
required: false,
|
|
384
|
+
specRef: "server/tools#error-handling",
|
|
385
|
+
description: "Calls tools/call with an empty params object (missing required name field) and verifies an error is returned.",
|
|
386
|
+
recommendation: "Validate tools/call params and return error code -32602 (Invalid params) when the required name field is missing."
|
|
387
|
+
},
|
|
388
|
+
{
|
|
389
|
+
id: "error-parse-code",
|
|
390
|
+
name: "Returns -32700 for invalid JSON",
|
|
391
|
+
category: "errors",
|
|
392
|
+
required: false,
|
|
393
|
+
specRef: "basic",
|
|
394
|
+
description: "Checks that the server returns the specific JSON-RPC error code -32700 (Parse error) when receiving invalid JSON, as required by the JSON-RPC 2.0 specification.",
|
|
395
|
+
recommendation: "Return exactly error code -32700 for JSON parse failures. Most JSON-RPC frameworks handle this automatically \u2014 check yours does not override the code."
|
|
396
|
+
},
|
|
397
|
+
{
|
|
398
|
+
id: "error-invalid-request-code",
|
|
399
|
+
name: "Returns -32600 for invalid request",
|
|
400
|
+
category: "errors",
|
|
401
|
+
required: false,
|
|
402
|
+
specRef: "basic",
|
|
403
|
+
description: "Checks that the server returns the specific JSON-RPC error code -32600 (Invalid Request) for malformed JSON-RPC messages missing required fields.",
|
|
404
|
+
recommendation: "Return exactly error code -32600 for structurally invalid JSON-RPC messages (e.g., missing method field). Check your JSON-RPC middleware configuration."
|
|
405
|
+
},
|
|
406
|
+
{
|
|
407
|
+
id: "tools-call-unknown",
|
|
408
|
+
name: "Returns error for unknown tool name",
|
|
409
|
+
category: "errors",
|
|
410
|
+
required: false,
|
|
411
|
+
specRef: "server/tools#error-handling",
|
|
412
|
+
description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response.",
|
|
413
|
+
recommendation: "Return a JSON-RPC error or set isError: true when tools/call receives an unrecognized tool name. Do not return an empty success response."
|
|
414
|
+
},
|
|
111
415
|
// ── Schema Validation (6 tests) ──────────────────────────────────
|
|
112
|
-
{
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
416
|
+
{
|
|
417
|
+
id: "tools-schema",
|
|
418
|
+
name: "All tools have name and inputSchema",
|
|
419
|
+
category: "schema",
|
|
420
|
+
required: false,
|
|
421
|
+
specRef: "server/tools#data-types",
|
|
422
|
+
description: 'Validates every tool has a valid name (1-128 chars, alphanumeric/underscore/hyphen/dot) and a required inputSchema of type "object".',
|
|
423
|
+
recommendation: 'Ensure every tool has a name (1-128 chars, [A-Za-z0-9_.-]) and an inputSchema with type: "object". Add descriptions to tools for better AI assistant integration.'
|
|
424
|
+
},
|
|
425
|
+
{
|
|
426
|
+
id: "tools-annotations",
|
|
427
|
+
name: "Tool annotations are valid",
|
|
428
|
+
category: "schema",
|
|
429
|
+
required: false,
|
|
430
|
+
specRef: "server/tools#annotations",
|
|
431
|
+
description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans; title should be a string.",
|
|
432
|
+
recommendation: "If you include annotations on tools, ensure readOnlyHint, destructiveHint, idempotentHint, and openWorldHint are booleans. Title must be a string."
|
|
433
|
+
},
|
|
434
|
+
{
|
|
435
|
+
id: "tools-title-field",
|
|
436
|
+
name: "Tools include title field",
|
|
437
|
+
category: "schema",
|
|
438
|
+
required: false,
|
|
439
|
+
specRef: "server/tools#data-types",
|
|
440
|
+
description: "Checks if tools include the optional title field for human-readable display names. Added in spec version 2025-11-25.",
|
|
441
|
+
recommendation: "Add a title field (human-readable string) to each tool definition. This helps MCP clients display your tools in a user-friendly way."
|
|
442
|
+
},
|
|
443
|
+
{
|
|
444
|
+
id: "tools-output-schema",
|
|
445
|
+
name: "Tools with outputSchema are valid",
|
|
446
|
+
category: "schema",
|
|
447
|
+
required: false,
|
|
448
|
+
specRef: "server/tools#structured-content",
|
|
449
|
+
description: 'If tools declare an outputSchema, validates it is a valid JSON Schema object with type "object". Used for structured output validation.',
|
|
450
|
+
recommendation: 'If you declare outputSchema on a tool, ensure it is a valid JSON Schema object with type: "object". Remove outputSchema if you do not need structured output.'
|
|
451
|
+
},
|
|
452
|
+
{
|
|
453
|
+
id: "prompts-schema",
|
|
454
|
+
name: "Prompts have name field",
|
|
455
|
+
category: "schema",
|
|
456
|
+
required: false,
|
|
457
|
+
specRef: "server/prompts#data-types",
|
|
458
|
+
description: "Validates every prompt has a name and that any arguments array contains items with name fields.",
|
|
459
|
+
recommendation: "Ensure every prompt has a name field. If the prompt has arguments, each argument object must include a name field."
|
|
460
|
+
},
|
|
461
|
+
{
|
|
462
|
+
id: "resources-schema",
|
|
463
|
+
name: "Resources have uri and name",
|
|
464
|
+
category: "schema",
|
|
465
|
+
required: false,
|
|
466
|
+
specRef: "server/resources#data-types",
|
|
467
|
+
description: "Validates every resource has a valid URI (parseable as a URL) and a name field.",
|
|
468
|
+
recommendation: "Ensure every resource has a valid, parseable URI and a name field. Add description and mimeType for better client integration."
|
|
469
|
+
}
|
|
118
470
|
];
|
|
119
471
|
|
|
120
472
|
// src/runner.ts
|
|
@@ -165,7 +517,7 @@ async function mcpRequest(backendUrl, method, params, nextId, extraHeaders, time
|
|
|
165
517
|
});
|
|
166
518
|
const headers = {
|
|
167
519
|
"Content-Type": "application/json",
|
|
168
|
-
|
|
520
|
+
Accept: "application/json, text/event-stream",
|
|
169
521
|
...extraHeaders
|
|
170
522
|
};
|
|
171
523
|
const res = await request(backendUrl, {
|
|
@@ -200,7 +552,7 @@ async function mcpRequest(backendUrl, method, params, nextId, extraHeaders, time
|
|
|
200
552
|
async function mcpNotification(backendUrl, method, params, extraHeaders, timeout) {
|
|
201
553
|
const headers = {
|
|
202
554
|
"Content-Type": "application/json",
|
|
203
|
-
|
|
555
|
+
Accept: "application/json, text/event-stream",
|
|
204
556
|
...extraHeaders
|
|
205
557
|
};
|
|
206
558
|
const res = await request(backendUrl, {
|
|
@@ -252,7 +604,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
252
604
|
}
|
|
253
605
|
return true;
|
|
254
606
|
}
|
|
255
|
-
|
|
607
|
+
const serverInfo = {
|
|
256
608
|
protocolVersion: null,
|
|
257
609
|
name: null,
|
|
258
610
|
version: null,
|
|
@@ -290,94 +642,129 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
290
642
|
});
|
|
291
643
|
options.onProgress?.(id, lastResult.passed, lastResult.details);
|
|
292
644
|
}
|
|
293
|
-
await test(
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
signal: AbortSignal.timeout(timeout)
|
|
311
|
-
});
|
|
312
|
-
await res.body.text();
|
|
313
|
-
const rawCt = res.headers["content-type"];
|
|
314
|
-
const ct = (Array.isArray(rawCt) ? rawCt[0] : rawCt || "").toLowerCase();
|
|
315
|
-
const valid = ct.includes("application/json") || ct.includes("text/event-stream");
|
|
316
|
-
return { passed: valid, details: `Content-Type: ${ct}` };
|
|
317
|
-
});
|
|
318
|
-
await test("transport-get", "GET returns SSE stream or 405", "transport", false, "basic/transports#streamable-http", async () => {
|
|
319
|
-
const res = await request(backendUrl, {
|
|
320
|
-
method: "GET",
|
|
321
|
-
headers: { "Accept": "text/event-stream", ...userHeaders },
|
|
322
|
-
signal: AbortSignal.timeout(timeout)
|
|
323
|
-
});
|
|
324
|
-
await res.body.text();
|
|
325
|
-
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
326
|
-
if (res.statusCode === 405) {
|
|
327
|
-
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
328
|
-
}
|
|
329
|
-
if (ct.includes("text/event-stream")) {
|
|
330
|
-
return { passed: true, details: "Returns text/event-stream for SSE" };
|
|
331
|
-
}
|
|
332
|
-
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
333
|
-
return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
|
|
334
|
-
}
|
|
335
|
-
return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
|
|
336
|
-
});
|
|
337
|
-
await test("transport-delete", "DELETE accepted or returns 405", "transport", false, "basic/transports#streamable-http", async () => {
|
|
338
|
-
const res = await request(backendUrl, {
|
|
339
|
-
method: "DELETE",
|
|
340
|
-
headers: { ...userHeaders },
|
|
341
|
-
signal: AbortSignal.timeout(timeout)
|
|
342
|
-
});
|
|
343
|
-
await res.body.text();
|
|
344
|
-
if (res.statusCode === 405) {
|
|
345
|
-
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
645
|
+
await test(
|
|
646
|
+
"transport-post",
|
|
647
|
+
"HTTP POST accepted",
|
|
648
|
+
"transport",
|
|
649
|
+
true,
|
|
650
|
+
"basic/transports#streamable-http",
|
|
651
|
+
async () => {
|
|
652
|
+
const res = await request(backendUrl, {
|
|
653
|
+
method: "POST",
|
|
654
|
+
headers: { "Content-Type": "application/json", Accept: "application/json, text/event-stream", ...userHeaders },
|
|
655
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 99901, method: "ping" }),
|
|
656
|
+
signal: AbortSignal.timeout(timeout)
|
|
657
|
+
});
|
|
658
|
+
await res.body.text();
|
|
659
|
+
const passed = res.statusCode >= 200 && res.statusCode < 300;
|
|
660
|
+
const note = res.statusCode === 401 || res.statusCode === 403 ? " (auth required)" : "";
|
|
661
|
+
return { passed, details: `HTTP ${res.statusCode}${note}` };
|
|
346
662
|
}
|
|
347
|
-
|
|
348
|
-
|
|
663
|
+
);
|
|
664
|
+
await test(
|
|
665
|
+
"transport-content-type",
|
|
666
|
+
"Responds with JSON or SSE",
|
|
667
|
+
"transport",
|
|
668
|
+
true,
|
|
669
|
+
"basic/transports#streamable-http",
|
|
670
|
+
async () => {
|
|
671
|
+
const res = await request(backendUrl, {
|
|
672
|
+
method: "POST",
|
|
673
|
+
headers: { "Content-Type": "application/json", Accept: "application/json, text/event-stream", ...userHeaders },
|
|
674
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 99902, method: "ping" }),
|
|
675
|
+
signal: AbortSignal.timeout(timeout)
|
|
676
|
+
});
|
|
677
|
+
await res.body.text();
|
|
678
|
+
const rawCt = res.headers["content-type"];
|
|
679
|
+
const ct = (Array.isArray(rawCt) ? rawCt[0] : rawCt || "").toLowerCase();
|
|
680
|
+
const valid = ct.includes("application/json") || ct.includes("text/event-stream");
|
|
681
|
+
return { passed: valid, details: `Content-Type: ${ct}` };
|
|
349
682
|
}
|
|
350
|
-
|
|
351
|
-
|
|
683
|
+
);
|
|
684
|
+
await test(
|
|
685
|
+
"transport-get",
|
|
686
|
+
"GET returns SSE stream or 405",
|
|
687
|
+
"transport",
|
|
688
|
+
false,
|
|
689
|
+
"basic/transports#streamable-http",
|
|
690
|
+
async () => {
|
|
691
|
+
const res = await request(backendUrl, {
|
|
692
|
+
method: "GET",
|
|
693
|
+
headers: { Accept: "text/event-stream", ...userHeaders },
|
|
694
|
+
signal: AbortSignal.timeout(timeout)
|
|
695
|
+
});
|
|
696
|
+
await res.body.text();
|
|
697
|
+
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
698
|
+
if (res.statusCode === 405) {
|
|
699
|
+
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
700
|
+
}
|
|
701
|
+
if (ct.includes("text/event-stream")) {
|
|
702
|
+
return { passed: true, details: "Returns text/event-stream for SSE" };
|
|
703
|
+
}
|
|
704
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
705
|
+
return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
|
|
706
|
+
}
|
|
707
|
+
return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
|
|
352
708
|
}
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
709
|
+
);
|
|
710
|
+
await test(
|
|
711
|
+
"transport-delete",
|
|
712
|
+
"DELETE accepted or returns 405",
|
|
713
|
+
"transport",
|
|
714
|
+
false,
|
|
715
|
+
"basic/transports#streamable-http",
|
|
716
|
+
async () => {
|
|
717
|
+
const res = await request(backendUrl, {
|
|
718
|
+
method: "DELETE",
|
|
719
|
+
headers: { ...userHeaders },
|
|
720
|
+
signal: AbortSignal.timeout(timeout)
|
|
721
|
+
});
|
|
722
|
+
await res.body.text();
|
|
723
|
+
if (res.statusCode === 405) {
|
|
724
|
+
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
725
|
+
}
|
|
726
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
727
|
+
return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
|
|
728
|
+
}
|
|
729
|
+
if (res.statusCode === 400 || res.statusCode === 404) {
|
|
730
|
+
return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
|
|
731
|
+
}
|
|
732
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
368
733
|
}
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
734
|
+
);
|
|
735
|
+
await test(
|
|
736
|
+
"transport-batch-reject",
|
|
737
|
+
"Rejects JSON-RPC batch requests",
|
|
738
|
+
"transport",
|
|
739
|
+
true,
|
|
740
|
+
"basic/transports#streamable-http",
|
|
741
|
+
async () => {
|
|
742
|
+
const res = await request(backendUrl, {
|
|
743
|
+
method: "POST",
|
|
744
|
+
headers: { "Content-Type": "application/json", Accept: "application/json, text/event-stream", ...userHeaders },
|
|
745
|
+
body: JSON.stringify([
|
|
746
|
+
{ jsonrpc: "2.0", id: 99903, method: "ping" },
|
|
747
|
+
{ jsonrpc: "2.0", id: 99904, method: "ping" }
|
|
748
|
+
]),
|
|
749
|
+
signal: AbortSignal.timeout(timeout)
|
|
750
|
+
});
|
|
751
|
+
const text = await res.body.text();
|
|
752
|
+
if (res.statusCode >= 400 && res.statusCode < 500) {
|
|
753
|
+
return { passed: true, details: `HTTP ${res.statusCode} (batch rejected)` };
|
|
373
754
|
}
|
|
374
|
-
|
|
375
|
-
|
|
755
|
+
try {
|
|
756
|
+
const body = JSON.parse(text);
|
|
757
|
+
if (body?.error) {
|
|
758
|
+
return { passed: true, details: `JSON-RPC error: ${body.error.code} \u2014 ${body.error.message}` };
|
|
759
|
+
}
|
|
760
|
+
if (Array.isArray(body)) {
|
|
761
|
+
return { passed: false, details: "Server processed batch request (MCP forbids batch)" };
|
|
762
|
+
}
|
|
763
|
+
} catch {
|
|
376
764
|
}
|
|
377
|
-
|
|
765
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error or 4xx for batch request` };
|
|
378
766
|
}
|
|
379
|
-
|
|
380
|
-
});
|
|
767
|
+
);
|
|
381
768
|
let initRes = null;
|
|
382
769
|
try {
|
|
383
770
|
initRes = await rpc("initialize", {
|
|
@@ -401,35 +788,69 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
401
788
|
await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders(), timeout);
|
|
402
789
|
} catch {
|
|
403
790
|
}
|
|
404
|
-
await test(
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
warnings.push(`Server negotiated protocol version ${version2} (latest is ${SPEC_VERSION})`);
|
|
791
|
+
await test(
|
|
792
|
+
"lifecycle-init",
|
|
793
|
+
"Initialize handshake",
|
|
794
|
+
"lifecycle",
|
|
795
|
+
true,
|
|
796
|
+
"basic/lifecycle#initialization",
|
|
797
|
+
async () => {
|
|
798
|
+
if (!initRes) return { passed: false, details: "Initialize request failed" };
|
|
799
|
+
const result = initRes.body?.result;
|
|
800
|
+
if (!result) return { passed: false, details: "No result in response" };
|
|
801
|
+
return { passed: !!result.protocolVersion, details: `Protocol: ${result.protocolVersion || "missing"}` };
|
|
416
802
|
}
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
803
|
+
);
|
|
804
|
+
await test(
|
|
805
|
+
"lifecycle-proto-version",
|
|
806
|
+
"Returns valid protocol version",
|
|
807
|
+
"lifecycle",
|
|
808
|
+
true,
|
|
809
|
+
"basic/lifecycle#version-negotiation",
|
|
810
|
+
async () => {
|
|
811
|
+
const version3 = initRes?.body?.result?.protocolVersion;
|
|
812
|
+
if (!version3) return { passed: false, details: "No protocolVersion" };
|
|
813
|
+
const valid = /^\d{4}-\d{2}-\d{2}$/.test(version3);
|
|
814
|
+
if (valid && version3 !== SPEC_VERSION) {
|
|
815
|
+
warnings.push(`Server negotiated protocol version ${version3} (latest is ${SPEC_VERSION})`);
|
|
816
|
+
}
|
|
817
|
+
return { passed: valid, details: `Version: ${version3}` };
|
|
818
|
+
}
|
|
819
|
+
);
|
|
820
|
+
await test(
|
|
821
|
+
"lifecycle-server-info",
|
|
822
|
+
"Includes serverInfo",
|
|
823
|
+
"lifecycle",
|
|
824
|
+
false,
|
|
825
|
+
"basic/lifecycle#initialization",
|
|
826
|
+
async () => {
|
|
827
|
+
const info = initRes?.body?.result?.serverInfo;
|
|
828
|
+
return { passed: !!info?.name, details: info ? `${info.name} v${info.version || "?"}` : "Missing serverInfo" };
|
|
829
|
+
}
|
|
830
|
+
);
|
|
831
|
+
await test(
|
|
832
|
+
"lifecycle-capabilities",
|
|
833
|
+
"Returns capabilities object",
|
|
834
|
+
"lifecycle",
|
|
835
|
+
true,
|
|
836
|
+
"basic/lifecycle#capability-negotiation",
|
|
837
|
+
async () => {
|
|
838
|
+
const caps = initRes?.body?.result?.capabilities;
|
|
839
|
+
if (!caps || typeof caps !== "object") return { passed: false, details: "No capabilities object in response" };
|
|
840
|
+
const declared = Object.keys(caps).filter((k) => caps[k] !== void 0);
|
|
841
|
+
return {
|
|
842
|
+
passed: true,
|
|
843
|
+
details: declared.length > 0 ? `Capabilities: ${declared.join(", ")}` : "Empty capabilities (valid)"
|
|
844
|
+
};
|
|
845
|
+
}
|
|
846
|
+
);
|
|
429
847
|
await test("lifecycle-jsonrpc", "Response is valid JSON-RPC 2.0", "lifecycle", true, "basic", async () => {
|
|
430
848
|
const body = initRes?.body;
|
|
431
849
|
const valid = body?.jsonrpc === "2.0" && body?.id !== void 0 && (body?.result !== void 0 || body?.error !== void 0);
|
|
432
|
-
return {
|
|
850
|
+
return {
|
|
851
|
+
passed: valid,
|
|
852
|
+
details: valid ? "Valid JSON-RPC 2.0 response" : `Missing fields: jsonrpc=${body?.jsonrpc}, id=${body?.id}`
|
|
853
|
+
};
|
|
433
854
|
});
|
|
434
855
|
await test("lifecycle-ping", "Responds to ping", "lifecycle", true, "basic/utilities#ping", async () => {
|
|
435
856
|
const res = await rpc("ping");
|
|
@@ -438,149 +859,223 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
438
859
|
if (body?.result !== void 0) return { passed: true, details: "Ping responded successfully" };
|
|
439
860
|
return { passed: false, details: "No result in ping response" };
|
|
440
861
|
});
|
|
441
|
-
await test(
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
const
|
|
449
|
-
return { passed:
|
|
862
|
+
await test(
|
|
863
|
+
"lifecycle-instructions",
|
|
864
|
+
"Instructions field is valid",
|
|
865
|
+
"lifecycle",
|
|
866
|
+
false,
|
|
867
|
+
"basic/lifecycle#initialization",
|
|
868
|
+
async () => {
|
|
869
|
+
const result = initRes?.body?.result;
|
|
870
|
+
if (!result) return { passed: false, details: "No init result" };
|
|
871
|
+
if (result.instructions === void 0) {
|
|
872
|
+
return { passed: true, details: "No instructions field (optional)" };
|
|
873
|
+
}
|
|
874
|
+
if (typeof result.instructions === "string") {
|
|
875
|
+
const preview = result.instructions.length > 80 ? result.instructions.slice(0, 80) + "..." : result.instructions;
|
|
876
|
+
return { passed: true, details: `Instructions: "${preview}"` };
|
|
877
|
+
}
|
|
878
|
+
return { passed: false, details: `instructions should be a string, got ${typeof result.instructions}` };
|
|
450
879
|
}
|
|
451
|
-
|
|
452
|
-
});
|
|
880
|
+
);
|
|
453
881
|
await test("lifecycle-id-match", "Response ID matches request ID", "lifecycle", true, "basic", async () => {
|
|
454
882
|
const res = await rpc("ping");
|
|
455
883
|
const body = res.body;
|
|
456
884
|
if (body?.id === void 0) return { passed: false, details: "No id in response" };
|
|
457
885
|
const match = body.id === res.requestId;
|
|
458
|
-
return {
|
|
886
|
+
return {
|
|
887
|
+
passed: match,
|
|
888
|
+
details: match ? `Request id=${res.requestId}, response id=${body.id} (match)` : `Request id=${res.requestId}, response id=${body.id} (MISMATCH)`
|
|
889
|
+
};
|
|
459
890
|
});
|
|
460
891
|
const hasLogging = !!serverInfo.capabilities.logging;
|
|
461
|
-
await test(
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
892
|
+
await test(
|
|
893
|
+
"lifecycle-logging",
|
|
894
|
+
"logging/setLevel accepted",
|
|
895
|
+
"lifecycle",
|
|
896
|
+
hasLogging,
|
|
897
|
+
"server/utilities#logging",
|
|
898
|
+
async () => {
|
|
899
|
+
if (!hasLogging) return { passed: true, details: "Server does not declare logging capability (skipped)" };
|
|
900
|
+
const res = await rpc("logging/setLevel", { level: "info" });
|
|
901
|
+
if (res.body?.error) {
|
|
902
|
+
return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
|
|
903
|
+
}
|
|
904
|
+
return { passed: true, details: "logging/setLevel accepted" };
|
|
466
905
|
}
|
|
467
|
-
|
|
468
|
-
});
|
|
906
|
+
);
|
|
469
907
|
const hasCompletions = !!serverInfo.capabilities.completions;
|
|
470
|
-
await test(
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
if (
|
|
478
|
-
|
|
908
|
+
await test(
|
|
909
|
+
"lifecycle-completions",
|
|
910
|
+
"completion/complete accepted",
|
|
911
|
+
"lifecycle",
|
|
912
|
+
hasCompletions,
|
|
913
|
+
"server/utilities#completion",
|
|
914
|
+
async () => {
|
|
915
|
+
if (!hasCompletions) return { passed: true, details: "Server does not declare completions capability (skipped)" };
|
|
916
|
+
const res = await rpc("completion/complete", {
|
|
917
|
+
ref: { type: "ref/prompt", name: "__test__" },
|
|
918
|
+
argument: { name: "test", value: "" }
|
|
919
|
+
});
|
|
920
|
+
if (res.body?.error) {
|
|
921
|
+
if (res.body.error.code === -32602) {
|
|
922
|
+
return { passed: true, details: "InvalidParams for test ref (acceptable)" };
|
|
923
|
+
}
|
|
924
|
+
return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
|
|
479
925
|
}
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
return { passed: true, details:
|
|
485
|
-
}
|
|
486
|
-
return { passed: true, details: "completion/complete accepted" };
|
|
487
|
-
});
|
|
488
|
-
await test("transport-notification-202", "Notification returns 202 Accepted", "transport", false, "basic/transports#streamable-http", async () => {
|
|
489
|
-
const res = await request(backendUrl, {
|
|
490
|
-
method: "POST",
|
|
491
|
-
headers: {
|
|
492
|
-
"Content-Type": "application/json",
|
|
493
|
-
"Accept": "application/json, text/event-stream",
|
|
494
|
-
...buildHeaders()
|
|
495
|
-
},
|
|
496
|
-
body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/cancelled", params: { requestId: "nonexistent", reason: "compliance test" } }),
|
|
497
|
-
signal: AbortSignal.timeout(timeout)
|
|
498
|
-
});
|
|
499
|
-
await res.body.text();
|
|
500
|
-
if (res.statusCode === 202) {
|
|
501
|
-
return { passed: true, details: "HTTP 202 Accepted (correct)" };
|
|
502
|
-
}
|
|
503
|
-
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
504
|
-
return { passed: true, details: `HTTP ${res.statusCode} (accepted, but 202 is preferred)` };
|
|
505
|
-
}
|
|
506
|
-
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected 202 Accepted for notifications` };
|
|
507
|
-
});
|
|
508
|
-
await test("transport-session-id", "Enforces MCP-Session-Id after init", "transport", false, "basic/transports#streamable-http", async () => {
|
|
509
|
-
if (!sessionId) {
|
|
510
|
-
warnings.push("Server did not issue MCP-Session-Id header");
|
|
511
|
-
return { passed: true, details: "Server did not issue session ID (test not applicable)" };
|
|
512
|
-
}
|
|
513
|
-
const headersWithout = { ...userHeaders };
|
|
514
|
-
if (negotiatedProtocolVersion) headersWithout["mcp-protocol-version"] = negotiatedProtocolVersion;
|
|
515
|
-
const res = await mcpRequest(backendUrl, "ping", void 0, createIdCounter(99910), headersWithout, timeout);
|
|
516
|
-
if (res.statusCode === 400) {
|
|
517
|
-
return { passed: true, details: "HTTP 400 for missing session ID (correct)" };
|
|
926
|
+
const values = res.body?.result?.completion?.values;
|
|
927
|
+
if (Array.isArray(values)) {
|
|
928
|
+
return { passed: true, details: `Returned ${values.length} completion(s)` };
|
|
929
|
+
}
|
|
930
|
+
return { passed: true, details: "completion/complete accepted" };
|
|
518
931
|
}
|
|
519
|
-
|
|
520
|
-
|
|
932
|
+
);
|
|
933
|
+
await test(
|
|
934
|
+
"transport-notification-202",
|
|
935
|
+
"Notification returns 202 Accepted",
|
|
936
|
+
"transport",
|
|
937
|
+
false,
|
|
938
|
+
"basic/transports#streamable-http",
|
|
939
|
+
async () => {
|
|
940
|
+
const res = await request(backendUrl, {
|
|
941
|
+
method: "POST",
|
|
942
|
+
headers: {
|
|
943
|
+
"Content-Type": "application/json",
|
|
944
|
+
Accept: "application/json, text/event-stream",
|
|
945
|
+
...buildHeaders()
|
|
946
|
+
},
|
|
947
|
+
body: JSON.stringify({
|
|
948
|
+
jsonrpc: "2.0",
|
|
949
|
+
method: "notifications/cancelled",
|
|
950
|
+
params: { requestId: "nonexistent", reason: "compliance test" }
|
|
951
|
+
}),
|
|
952
|
+
signal: AbortSignal.timeout(timeout)
|
|
953
|
+
});
|
|
954
|
+
await res.body.text();
|
|
955
|
+
if (res.statusCode === 202) {
|
|
956
|
+
return { passed: true, details: "HTTP 202 Accepted (correct)" };
|
|
957
|
+
}
|
|
958
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
959
|
+
return { passed: true, details: `HTTP ${res.statusCode} (accepted, but 202 is preferred)` };
|
|
960
|
+
}
|
|
961
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected 202 Accepted for notifications` };
|
|
521
962
|
}
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
return { passed: true, details: `${toolCount} tool(s): ${toolNames.slice(0, 5).join(", ")}${toolCount > 5 ? "..." : ""}` };
|
|
534
|
-
});
|
|
535
|
-
await test("tools-schema", "All tools have name and inputSchema", "schema", hasTools, "server/tools#data-types", async () => {
|
|
536
|
-
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
537
|
-
const issues = [];
|
|
538
|
-
for (const tool of tools) {
|
|
539
|
-
if (!tool.name) {
|
|
540
|
-
issues.push("Tool missing name");
|
|
541
|
-
continue;
|
|
963
|
+
);
|
|
964
|
+
await test(
|
|
965
|
+
"transport-session-id",
|
|
966
|
+
"Enforces MCP-Session-Id after init",
|
|
967
|
+
"transport",
|
|
968
|
+
false,
|
|
969
|
+
"basic/transports#streamable-http",
|
|
970
|
+
async () => {
|
|
971
|
+
if (!sessionId) {
|
|
972
|
+
warnings.push("Server did not issue MCP-Session-Id header");
|
|
973
|
+
return { passed: true, details: "Server did not issue session ID (test not applicable)" };
|
|
542
974
|
}
|
|
543
|
-
|
|
544
|
-
|
|
975
|
+
const headersWithout = { ...userHeaders };
|
|
976
|
+
if (negotiatedProtocolVersion) headersWithout["mcp-protocol-version"] = negotiatedProtocolVersion;
|
|
977
|
+
const res = await mcpRequest(backendUrl, "ping", void 0, createIdCounter(99910), headersWithout, timeout);
|
|
978
|
+
if (res.statusCode === 400) {
|
|
979
|
+
return { passed: true, details: "HTTP 400 for missing session ID (correct)" };
|
|
545
980
|
}
|
|
546
|
-
if (
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
} else if (tool.inputSchema.type !== "object") {
|
|
552
|
-
issues.push(`${tool.name}: inputSchema.type must be "object" (got "${tool.inputSchema.type || "undefined"}")`);
|
|
981
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
982
|
+
return {
|
|
983
|
+
passed: false,
|
|
984
|
+
details: `HTTP ${res.statusCode} \u2014 server should return 400 when session ID is missing`
|
|
985
|
+
};
|
|
553
986
|
}
|
|
987
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
554
988
|
}
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
await test(
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
if (
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
989
|
+
);
|
|
990
|
+
const hasTools = !!serverInfo.capabilities.tools;
|
|
991
|
+
let cachedToolsList = null;
|
|
992
|
+
await test(
|
|
993
|
+
"tools-list",
|
|
994
|
+
"tools/list returns valid response",
|
|
995
|
+
"tools",
|
|
996
|
+
hasTools,
|
|
997
|
+
"server/tools#listing-tools",
|
|
998
|
+
async () => {
|
|
999
|
+
const res = await rpc("tools/list");
|
|
1000
|
+
const tools = res.body?.result?.tools;
|
|
1001
|
+
if (!Array.isArray(tools)) return { passed: false, details: "No tools array in result" };
|
|
1002
|
+
cachedToolsList = tools;
|
|
1003
|
+
toolCount = tools.length;
|
|
1004
|
+
toolNames = tools.map((t) => t.name).filter(Boolean);
|
|
1005
|
+
return {
|
|
1006
|
+
passed: true,
|
|
1007
|
+
details: `${toolCount} tool(s): ${toolNames.slice(0, 5).join(", ")}${toolCount > 5 ? "..." : ""}`
|
|
1008
|
+
};
|
|
1009
|
+
}
|
|
1010
|
+
);
|
|
1011
|
+
await test(
|
|
1012
|
+
"tools-schema",
|
|
1013
|
+
"All tools have name and inputSchema",
|
|
1014
|
+
"schema",
|
|
1015
|
+
hasTools,
|
|
1016
|
+
"server/tools#data-types",
|
|
1017
|
+
async () => {
|
|
1018
|
+
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
1019
|
+
const issues = [];
|
|
1020
|
+
for (const tool of tools) {
|
|
1021
|
+
if (!tool.name) {
|
|
1022
|
+
issues.push("Tool missing name");
|
|
1023
|
+
continue;
|
|
1024
|
+
}
|
|
1025
|
+
if (tool.name.length > 128 || !/^[A-Za-z0-9_.\-]+$/.test(tool.name)) {
|
|
1026
|
+
issues.push(`${tool.name}: name format invalid`);
|
|
1027
|
+
}
|
|
1028
|
+
if (!tool.description) warnings.push(`Tool "${tool.name}" missing description`);
|
|
1029
|
+
if (!tool.inputSchema) {
|
|
1030
|
+
issues.push(`${tool.name}: missing inputSchema (required)`);
|
|
1031
|
+
} else if (typeof tool.inputSchema !== "object" || tool.inputSchema === null) {
|
|
1032
|
+
issues.push(`${tool.name}: inputSchema must be a valid JSON Schema object`);
|
|
1033
|
+
} else if (tool.inputSchema.type !== "object") {
|
|
1034
|
+
issues.push(
|
|
1035
|
+
`${tool.name}: inputSchema.type must be "object" (got "${tool.inputSchema.type || "undefined"}")`
|
|
1036
|
+
);
|
|
575
1037
|
}
|
|
576
1038
|
}
|
|
577
|
-
|
|
578
|
-
|
|
1039
|
+
const detail = issues.length === 0 ? "All tools have valid schemas" : issues.join("; ");
|
|
1040
|
+
return { passed: issues.length === 0, details: detail };
|
|
1041
|
+
}
|
|
1042
|
+
);
|
|
1043
|
+
await test(
|
|
1044
|
+
"tools-annotations",
|
|
1045
|
+
"Tool annotations are valid",
|
|
1046
|
+
"schema",
|
|
1047
|
+
false,
|
|
1048
|
+
"server/tools#annotations",
|
|
1049
|
+
async () => {
|
|
1050
|
+
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
1051
|
+
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1052
|
+
const issues = [];
|
|
1053
|
+
let annotatedCount = 0;
|
|
1054
|
+
for (const tool of tools) {
|
|
1055
|
+
const ann = tool.annotations;
|
|
1056
|
+
if (!ann) continue;
|
|
1057
|
+
annotatedCount++;
|
|
1058
|
+
if (typeof ann !== "object" || ann === null) {
|
|
1059
|
+
issues.push(`${tool.name}: annotations must be an object`);
|
|
1060
|
+
continue;
|
|
1061
|
+
}
|
|
1062
|
+
const boolFields = ["readOnlyHint", "destructiveHint", "idempotentHint", "openWorldHint"];
|
|
1063
|
+
for (const field of boolFields) {
|
|
1064
|
+
if (ann[field] !== void 0 && typeof ann[field] !== "boolean") {
|
|
1065
|
+
issues.push(`${tool.name}: annotations.${field} should be boolean, got ${typeof ann[field]}`);
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
if (ann.title !== void 0 && typeof ann.title !== "string") {
|
|
1069
|
+
issues.push(`${tool.name}: annotations.title should be string`);
|
|
1070
|
+
}
|
|
579
1071
|
}
|
|
1072
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1073
|
+
return {
|
|
1074
|
+
passed: true,
|
|
1075
|
+
details: annotatedCount > 0 ? `${annotatedCount} tool(s) with valid annotations` : "No tools have annotations (optional)"
|
|
1076
|
+
};
|
|
580
1077
|
}
|
|
581
|
-
|
|
582
|
-
return { passed: true, details: annotatedCount > 0 ? `${annotatedCount} tool(s) with valid annotations` : "No tools have annotations (optional)" };
|
|
583
|
-
});
|
|
1078
|
+
);
|
|
584
1079
|
await test("tools-title-field", "Tools include title field", "schema", false, "server/tools#data-types", async () => {
|
|
585
1080
|
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
586
1081
|
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
@@ -597,211 +1092,319 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
597
1092
|
}
|
|
598
1093
|
return { passed: true, details: `${withTitle.length}/${tools.length} tool(s) have title field` };
|
|
599
1094
|
});
|
|
600
|
-
await test(
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
if (
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
1095
|
+
await test(
|
|
1096
|
+
"tools-output-schema",
|
|
1097
|
+
"Tools with outputSchema are valid",
|
|
1098
|
+
"schema",
|
|
1099
|
+
false,
|
|
1100
|
+
"server/tools#structured-content",
|
|
1101
|
+
async () => {
|
|
1102
|
+
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
1103
|
+
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1104
|
+
const issues = [];
|
|
1105
|
+
let withSchema = 0;
|
|
1106
|
+
for (const tool of tools) {
|
|
1107
|
+
if (tool.outputSchema === void 0) continue;
|
|
1108
|
+
withSchema++;
|
|
1109
|
+
if (typeof tool.outputSchema !== "object" || tool.outputSchema === null) {
|
|
1110
|
+
issues.push(`${tool.name}: outputSchema must be a JSON Schema object`);
|
|
1111
|
+
} else if (tool.outputSchema.type !== "object") {
|
|
1112
|
+
issues.push(
|
|
1113
|
+
`${tool.name}: outputSchema.type must be "object" (got "${tool.outputSchema.type || "undefined"}")`
|
|
1114
|
+
);
|
|
1115
|
+
}
|
|
612
1116
|
}
|
|
1117
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1118
|
+
return {
|
|
1119
|
+
passed: true,
|
|
1120
|
+
details: withSchema > 0 ? `${withSchema} tool(s) with valid outputSchema` : "No tools have outputSchema (optional)"
|
|
1121
|
+
};
|
|
613
1122
|
}
|
|
614
|
-
|
|
615
|
-
return { passed: true, details: withSchema > 0 ? `${withSchema} tool(s) with valid outputSchema` : "No tools have outputSchema (optional)" };
|
|
616
|
-
});
|
|
1123
|
+
);
|
|
617
1124
|
if (toolNames.length > 0) {
|
|
618
|
-
await test(
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
1125
|
+
await test(
|
|
1126
|
+
"tools-call",
|
|
1127
|
+
"tools/call responds correctly",
|
|
1128
|
+
"tools",
|
|
1129
|
+
false,
|
|
1130
|
+
"server/tools#calling-tools",
|
|
1131
|
+
async () => {
|
|
1132
|
+
const res = await rpc("tools/call", { name: toolNames[0], arguments: {} });
|
|
1133
|
+
const result = res.body?.result;
|
|
1134
|
+
const error = res.body?.error;
|
|
1135
|
+
if (error) {
|
|
1136
|
+
const code = error.code;
|
|
1137
|
+
if (code === -32602 || code === -32600) {
|
|
1138
|
+
return { passed: true, details: `Invalid params error (acceptable): code ${code}` };
|
|
1139
|
+
}
|
|
1140
|
+
return { passed: true, details: `Protocol error: code ${code} \u2014 ${error.message}` };
|
|
626
1141
|
}
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
return { passed: false, details: "Response missing content array" };
|
|
638
|
-
});
|
|
639
|
-
await test("tools-content-types", "Tool content items have valid types", "tools", false, "server/tools#calling-tools", async () => {
|
|
640
|
-
const res = await rpc("tools/call", { name: toolNames[0], arguments: {} });
|
|
641
|
-
const result = res.body?.result;
|
|
642
|
-
const error = res.body?.error;
|
|
643
|
-
if (error) {
|
|
644
|
-
return { passed: true, details: `Tool returned error (content types not applicable): code ${error.code}` };
|
|
645
|
-
}
|
|
646
|
-
const content = result?.content;
|
|
647
|
-
if (!Array.isArray(content) || content.length === 0) {
|
|
648
|
-
return { passed: true, details: "No content items to validate" };
|
|
1142
|
+
if (result?.content && Array.isArray(result.content)) {
|
|
1143
|
+
const badItems = result.content.filter((c) => !c.type);
|
|
1144
|
+
if (badItems.length > 0)
|
|
1145
|
+
return { passed: false, details: `${badItems.length} content item(s) missing 'type' field` };
|
|
1146
|
+
return { passed: true, details: `Returned ${result.content.length} content item(s)` };
|
|
1147
|
+
}
|
|
1148
|
+
if (result?.isError && result?.content && Array.isArray(result.content)) {
|
|
1149
|
+
return { passed: true, details: "Tool returned execution error with content (valid)" };
|
|
1150
|
+
}
|
|
1151
|
+
return { passed: false, details: "Response missing content array" };
|
|
649
1152
|
}
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
1153
|
+
);
|
|
1154
|
+
await test(
|
|
1155
|
+
"tools-content-types",
|
|
1156
|
+
"Tool content items have valid types",
|
|
1157
|
+
"tools",
|
|
1158
|
+
false,
|
|
1159
|
+
"server/tools#calling-tools",
|
|
1160
|
+
async () => {
|
|
1161
|
+
const res = await rpc("tools/call", { name: toolNames[0], arguments: {} });
|
|
1162
|
+
const result = res.body?.result;
|
|
1163
|
+
const error = res.body?.error;
|
|
1164
|
+
if (error) {
|
|
1165
|
+
return { passed: true, details: `Tool returned error (content types not applicable): code ${error.code}` };
|
|
1166
|
+
}
|
|
1167
|
+
const content = result?.content;
|
|
1168
|
+
if (!Array.isArray(content) || content.length === 0) {
|
|
1169
|
+
return { passed: true, details: "No content items to validate" };
|
|
1170
|
+
}
|
|
1171
|
+
const issues = [];
|
|
1172
|
+
const types = /* @__PURE__ */ new Set();
|
|
1173
|
+
for (const item of content) {
|
|
1174
|
+
if (!item.type) {
|
|
1175
|
+
issues.push("Content item missing type field");
|
|
1176
|
+
} else if (!VALID_CONTENT_TYPES.includes(item.type)) {
|
|
1177
|
+
issues.push(`Unknown content type: "${item.type}"`);
|
|
1178
|
+
} else {
|
|
1179
|
+
types.add(item.type);
|
|
1180
|
+
}
|
|
659
1181
|
}
|
|
1182
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1183
|
+
return { passed: true, details: `Content types: ${[...types].join(", ")}` };
|
|
660
1184
|
}
|
|
661
|
-
|
|
662
|
-
return { passed: true, details: `Content types: ${[...types].join(", ")}` };
|
|
663
|
-
});
|
|
1185
|
+
);
|
|
664
1186
|
}
|
|
665
1187
|
if (hasTools) {
|
|
666
|
-
await test(
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
if (
|
|
678
|
-
|
|
1188
|
+
await test(
|
|
1189
|
+
"tools-pagination",
|
|
1190
|
+
"tools/list supports pagination",
|
|
1191
|
+
"tools",
|
|
1192
|
+
false,
|
|
1193
|
+
"server/tools#listing-tools",
|
|
1194
|
+
async () => {
|
|
1195
|
+
const res = await rpc("tools/list");
|
|
1196
|
+
const result = res.body?.result;
|
|
1197
|
+
if (!result) return { passed: false, details: "No result from tools/list" };
|
|
1198
|
+
if (!Array.isArray(result.tools)) return { passed: false, details: "No tools array" };
|
|
1199
|
+
if (result.nextCursor !== void 0) {
|
|
1200
|
+
if (typeof result.nextCursor !== "string") {
|
|
1201
|
+
return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
|
|
1202
|
+
}
|
|
1203
|
+
const nextRes = await rpc("tools/list", { cursor: result.nextCursor });
|
|
1204
|
+
const nextResult = nextRes.body?.result;
|
|
1205
|
+
if (!nextResult || !Array.isArray(nextResult.tools)) {
|
|
1206
|
+
return { passed: false, details: "Next page failed to return tools array" };
|
|
1207
|
+
}
|
|
1208
|
+
return {
|
|
1209
|
+
passed: true,
|
|
1210
|
+
details: `Pagination works: page 1 had ${result.tools.length} tools, page 2 had ${nextResult.tools.length} tools`
|
|
1211
|
+
};
|
|
679
1212
|
}
|
|
680
|
-
return { passed: true, details:
|
|
1213
|
+
return { passed: true, details: `${result.tools.length} tool(s), no nextCursor (single page)` };
|
|
681
1214
|
}
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
1215
|
+
);
|
|
1216
|
+
await test(
|
|
1217
|
+
"tools-call-unknown",
|
|
1218
|
+
"Returns error for unknown tool name",
|
|
1219
|
+
"errors",
|
|
1220
|
+
false,
|
|
1221
|
+
"server/tools#error-handling",
|
|
1222
|
+
async () => {
|
|
1223
|
+
const res = await rpc("tools/call", { name: "__nonexistent_tool_compliance_test__", arguments: {} });
|
|
1224
|
+
const error = res.body?.error;
|
|
1225
|
+
const isError = res.body?.result?.isError;
|
|
1226
|
+
if (error) return { passed: true, details: `Error code: ${error.code} \u2014 ${error.message}` };
|
|
1227
|
+
if (isError) return { passed: true, details: "Tool execution error with isError=true (valid)" };
|
|
1228
|
+
return { passed: false, details: "No error returned for nonexistent tool" };
|
|
1229
|
+
}
|
|
1230
|
+
);
|
|
692
1231
|
}
|
|
693
1232
|
const hasResources = !!serverInfo.capabilities.resources;
|
|
694
1233
|
const hasSubscribe = !!serverInfo.capabilities.resources?.subscribe;
|
|
695
1234
|
if (hasResources) {
|
|
696
1235
|
let cachedResourcesList = null;
|
|
697
|
-
await test(
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
1236
|
+
await test(
|
|
1237
|
+
"resources-list",
|
|
1238
|
+
"resources/list returns valid response",
|
|
1239
|
+
"resources",
|
|
1240
|
+
true,
|
|
1241
|
+
"server/resources#listing-resources",
|
|
1242
|
+
async () => {
|
|
1243
|
+
const res = await rpc("resources/list");
|
|
1244
|
+
const resources = res.body?.result?.resources;
|
|
1245
|
+
if (!Array.isArray(resources)) return { passed: false, details: "No resources array" };
|
|
1246
|
+
cachedResourcesList = resources;
|
|
1247
|
+
resourceCount = resources.length;
|
|
1248
|
+
resourceNames = resources.map((r) => r.name).filter(Boolean);
|
|
1249
|
+
return { passed: true, details: `${resourceCount} resource(s)` };
|
|
1250
|
+
}
|
|
1251
|
+
);
|
|
1252
|
+
await test(
|
|
1253
|
+
"resources-schema",
|
|
1254
|
+
"Resources have uri and name",
|
|
1255
|
+
"schema",
|
|
1256
|
+
true,
|
|
1257
|
+
"server/resources#data-types",
|
|
1258
|
+
async () => {
|
|
1259
|
+
const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
|
|
1260
|
+
const issues = [];
|
|
1261
|
+
for (const r of resources) {
|
|
1262
|
+
if (!r.uri) issues.push("Resource missing uri");
|
|
1263
|
+
else {
|
|
1264
|
+
try {
|
|
1265
|
+
new URL(r.uri);
|
|
1266
|
+
} catch {
|
|
1267
|
+
issues.push(`${r.uri}: invalid URI format`);
|
|
1268
|
+
}
|
|
716
1269
|
}
|
|
1270
|
+
if (!r.name) issues.push(`${r.uri || "?"}: missing name`);
|
|
1271
|
+
if (!r.description) warnings.push(`Resource "${r.name || r.uri}" missing description`);
|
|
1272
|
+
if (!r.mimeType) warnings.push(`Resource "${r.name || r.uri}" missing mimeType`);
|
|
717
1273
|
}
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
1274
|
+
return {
|
|
1275
|
+
passed: issues.length === 0,
|
|
1276
|
+
details: issues.length === 0 ? "All resources valid" : issues.join("; ")
|
|
1277
|
+
};
|
|
721
1278
|
}
|
|
722
|
-
|
|
723
|
-
});
|
|
1279
|
+
);
|
|
724
1280
|
if (resourceCount > 0) {
|
|
725
|
-
await test(
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
1281
|
+
await test(
|
|
1282
|
+
"resources-read",
|
|
1283
|
+
"resources/read returns content",
|
|
1284
|
+
"resources",
|
|
1285
|
+
false,
|
|
1286
|
+
"server/resources#reading-resources",
|
|
1287
|
+
async () => {
|
|
1288
|
+
const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
|
|
1289
|
+
const firstUri = resources[0]?.uri;
|
|
1290
|
+
if (!firstUri) return { passed: false, details: "No resource URI to test" };
|
|
1291
|
+
const readRes = await rpc("resources/read", { uri: firstUri });
|
|
1292
|
+
const contents = readRes.body?.result?.contents;
|
|
1293
|
+
if (!Array.isArray(contents)) return { passed: false, details: "No contents array" };
|
|
1294
|
+
const issues = [];
|
|
1295
|
+
for (const c of contents) {
|
|
1296
|
+
if (!c.uri) issues.push("Content item missing uri");
|
|
1297
|
+
if (!c.text && !c.blob) issues.push(`Content item for ${c.uri || "?"} missing both text and blob`);
|
|
1298
|
+
}
|
|
1299
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1300
|
+
return { passed: true, details: `Read ${contents.length} content item(s) from ${firstUri}` };
|
|
1301
|
+
}
|
|
1302
|
+
);
|
|
1303
|
+
}
|
|
1304
|
+
await test(
|
|
1305
|
+
"resources-templates",
|
|
1306
|
+
"resources/templates/list returns valid response",
|
|
1307
|
+
"resources",
|
|
1308
|
+
false,
|
|
1309
|
+
"server/resources#resource-templates",
|
|
1310
|
+
async () => {
|
|
1311
|
+
const res = await rpc("resources/templates/list");
|
|
1312
|
+
const error = res.body?.error;
|
|
1313
|
+
if (error) {
|
|
1314
|
+
if (error.code === -32601) return { passed: true, details: "Method not supported (acceptable)" };
|
|
1315
|
+
return { passed: false, details: `Error: ${error.message}` };
|
|
1316
|
+
}
|
|
1317
|
+
const templates = res.body?.result?.resourceTemplates;
|
|
1318
|
+
if (!Array.isArray(templates)) return { passed: false, details: "No resourceTemplates array" };
|
|
732
1319
|
const issues = [];
|
|
733
|
-
for (const
|
|
734
|
-
if (!
|
|
735
|
-
if (!
|
|
1320
|
+
for (const t of templates) {
|
|
1321
|
+
if (!t.uriTemplate) issues.push("Template missing uriTemplate");
|
|
1322
|
+
if (!t.name) issues.push(`${t.uriTemplate || "?"}: missing name`);
|
|
736
1323
|
}
|
|
737
1324
|
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
738
|
-
return { passed: true, details:
|
|
739
|
-
});
|
|
740
|
-
}
|
|
741
|
-
await test("resources-templates", "resources/templates/list returns valid response", "resources", false, "server/resources#resource-templates", async () => {
|
|
742
|
-
const res = await rpc("resources/templates/list");
|
|
743
|
-
const error = res.body?.error;
|
|
744
|
-
if (error) {
|
|
745
|
-
if (error.code === -32601) return { passed: true, details: "Method not supported (acceptable)" };
|
|
746
|
-
return { passed: false, details: `Error: ${error.message}` };
|
|
747
|
-
}
|
|
748
|
-
const templates = res.body?.result?.resourceTemplates;
|
|
749
|
-
if (!Array.isArray(templates)) return { passed: false, details: "No resourceTemplates array" };
|
|
750
|
-
const issues = [];
|
|
751
|
-
for (const t of templates) {
|
|
752
|
-
if (!t.uriTemplate) issues.push("Template missing uriTemplate");
|
|
753
|
-
if (!t.name) issues.push(`${t.uriTemplate || "?"}: missing name`);
|
|
1325
|
+
return { passed: true, details: `${templates.length} resource template(s)` };
|
|
754
1326
|
}
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
}
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
1327
|
+
);
|
|
1328
|
+
await test(
|
|
1329
|
+
"resources-pagination",
|
|
1330
|
+
"resources/list supports pagination",
|
|
1331
|
+
"resources",
|
|
1332
|
+
false,
|
|
1333
|
+
"server/resources#listing-resources",
|
|
1334
|
+
async () => {
|
|
1335
|
+
const res = await rpc("resources/list");
|
|
1336
|
+
const result = res.body?.result;
|
|
1337
|
+
if (!result) return { passed: false, details: "No result from resources/list" };
|
|
1338
|
+
if (!Array.isArray(result.resources)) return { passed: false, details: "No resources array" };
|
|
1339
|
+
if (result.nextCursor !== void 0) {
|
|
1340
|
+
if (typeof result.nextCursor !== "string") {
|
|
1341
|
+
return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
|
|
1342
|
+
}
|
|
1343
|
+
const nextRes = await rpc("resources/list", { cursor: result.nextCursor });
|
|
1344
|
+
const nextResult = nextRes.body?.result;
|
|
1345
|
+
if (!nextResult || !Array.isArray(nextResult.resources)) {
|
|
1346
|
+
return { passed: false, details: "Next page failed to return resources array" };
|
|
1347
|
+
}
|
|
1348
|
+
return {
|
|
1349
|
+
passed: true,
|
|
1350
|
+
details: `Pagination works: page 1 had ${result.resources.length}, page 2 had ${nextResult.resources.length}`
|
|
1351
|
+
};
|
|
771
1352
|
}
|
|
772
|
-
return { passed: true, details:
|
|
1353
|
+
return { passed: true, details: `${result.resources.length} resource(s), no nextCursor (single page)` };
|
|
773
1354
|
}
|
|
774
|
-
|
|
775
|
-
});
|
|
1355
|
+
);
|
|
776
1356
|
if (hasSubscribe && resourceCount > 0) {
|
|
777
|
-
await test(
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
1357
|
+
await test(
|
|
1358
|
+
"resources-subscribe",
|
|
1359
|
+
"Resource subscribe/unsubscribe",
|
|
1360
|
+
"resources",
|
|
1361
|
+
true,
|
|
1362
|
+
"server/resources#subscriptions",
|
|
1363
|
+
async () => {
|
|
1364
|
+
const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
|
|
1365
|
+
const firstUri = resources[0]?.uri;
|
|
1366
|
+
if (!firstUri) return { passed: false, details: "No resource URI for subscribe test" };
|
|
1367
|
+
const subRes = await rpc("resources/subscribe", { uri: firstUri });
|
|
1368
|
+
if (subRes.body?.error) {
|
|
1369
|
+
return {
|
|
1370
|
+
passed: false,
|
|
1371
|
+
details: `Subscribe error: ${subRes.body.error.code} \u2014 ${subRes.body.error.message}`
|
|
1372
|
+
};
|
|
1373
|
+
}
|
|
1374
|
+
const unsubRes = await rpc("resources/unsubscribe", { uri: firstUri });
|
|
1375
|
+
if (unsubRes.body?.error) {
|
|
1376
|
+
return {
|
|
1377
|
+
passed: false,
|
|
1378
|
+
details: `Unsubscribe error: ${unsubRes.body.error.code} \u2014 ${unsubRes.body.error.message}`
|
|
1379
|
+
};
|
|
1380
|
+
}
|
|
1381
|
+
return { passed: true, details: `Subscribe/unsubscribe for ${firstUri} succeeded` };
|
|
788
1382
|
}
|
|
789
|
-
|
|
790
|
-
});
|
|
1383
|
+
);
|
|
791
1384
|
}
|
|
792
1385
|
}
|
|
793
1386
|
const hasPrompts = !!serverInfo.capabilities.prompts;
|
|
794
1387
|
if (hasPrompts) {
|
|
795
1388
|
let cachedPromptsList = null;
|
|
796
|
-
await test(
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
1389
|
+
await test(
|
|
1390
|
+
"prompts-list",
|
|
1391
|
+
"prompts/list returns valid response",
|
|
1392
|
+
"prompts",
|
|
1393
|
+
true,
|
|
1394
|
+
"server/prompts#listing-prompts",
|
|
1395
|
+
async () => {
|
|
1396
|
+
const res = await rpc("prompts/list");
|
|
1397
|
+
const prompts = res.body?.result?.prompts;
|
|
1398
|
+
if (!Array.isArray(prompts)) return { passed: false, details: "No prompts array" };
|
|
1399
|
+
cachedPromptsList = prompts;
|
|
1400
|
+
promptCount = prompts.length;
|
|
1401
|
+
promptNames = prompts.map((p) => p.name).filter(Boolean);
|
|
1402
|
+
return {
|
|
1403
|
+
passed: true,
|
|
1404
|
+
details: `${promptCount} prompt(s): ${promptNames.slice(0, 5).join(", ")}${promptCount > 5 ? "..." : ""}`
|
|
1405
|
+
};
|
|
1406
|
+
}
|
|
1407
|
+
);
|
|
805
1408
|
await test("prompts-schema", "Prompts have name field", "schema", true, "server/prompts#data-types", async () => {
|
|
806
1409
|
const prompts = cachedPromptsList ?? (await rpc("prompts/list")).body?.result?.prompts ?? [];
|
|
807
1410
|
const issues = [];
|
|
@@ -818,39 +1421,56 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
818
1421
|
return { passed: issues.length === 0, details: issues.length === 0 ? "All prompts valid" : issues.join("; ") };
|
|
819
1422
|
});
|
|
820
1423
|
if (promptNames.length > 0) {
|
|
821
|
-
await test(
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
if (
|
|
1424
|
+
await test(
|
|
1425
|
+
"prompts-get",
|
|
1426
|
+
"prompts/get returns valid messages",
|
|
1427
|
+
"prompts",
|
|
1428
|
+
false,
|
|
1429
|
+
"server/prompts#getting-a-prompt",
|
|
1430
|
+
async () => {
|
|
1431
|
+
const res = await rpc("prompts/get", { name: promptNames[0] });
|
|
1432
|
+
const error = res.body?.error;
|
|
1433
|
+
if (error) return { passed: true, details: `Error (may need arguments): code ${error.code}` };
|
|
1434
|
+
const messages = res.body?.result?.messages;
|
|
1435
|
+
if (!Array.isArray(messages)) return { passed: false, details: "No messages array in result" };
|
|
1436
|
+
const issues = [];
|
|
1437
|
+
for (const msg of messages) {
|
|
1438
|
+
if (!msg.role || !["user", "assistant"].includes(msg.role)) issues.push(`Invalid role: ${msg.role}`);
|
|
1439
|
+
if (!msg.content) issues.push("Message missing content");
|
|
1440
|
+
}
|
|
1441
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1442
|
+
return { passed: true, details: `${messages.length} message(s) from ${promptNames[0]}` };
|
|
831
1443
|
}
|
|
832
|
-
|
|
833
|
-
return { passed: true, details: `${messages.length} message(s) from ${promptNames[0]}` };
|
|
834
|
-
});
|
|
1444
|
+
);
|
|
835
1445
|
}
|
|
836
|
-
await test(
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
if (
|
|
848
|
-
|
|
1446
|
+
await test(
|
|
1447
|
+
"prompts-pagination",
|
|
1448
|
+
"prompts/list supports pagination",
|
|
1449
|
+
"prompts",
|
|
1450
|
+
false,
|
|
1451
|
+
"server/prompts#listing-prompts",
|
|
1452
|
+
async () => {
|
|
1453
|
+
const res = await rpc("prompts/list");
|
|
1454
|
+
const result = res.body?.result;
|
|
1455
|
+
if (!result) return { passed: false, details: "No result from prompts/list" };
|
|
1456
|
+
if (!Array.isArray(result.prompts)) return { passed: false, details: "No prompts array" };
|
|
1457
|
+
if (result.nextCursor !== void 0) {
|
|
1458
|
+
if (typeof result.nextCursor !== "string") {
|
|
1459
|
+
return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
|
|
1460
|
+
}
|
|
1461
|
+
const nextRes = await rpc("prompts/list", { cursor: result.nextCursor });
|
|
1462
|
+
const nextResult = nextRes.body?.result;
|
|
1463
|
+
if (!nextResult || !Array.isArray(nextResult.prompts)) {
|
|
1464
|
+
return { passed: false, details: "Next page failed to return prompts array" };
|
|
1465
|
+
}
|
|
1466
|
+
return {
|
|
1467
|
+
passed: true,
|
|
1468
|
+
details: `Pagination works: page 1 had ${result.prompts.length}, page 2 had ${nextResult.prompts.length}`
|
|
1469
|
+
};
|
|
849
1470
|
}
|
|
850
|
-
return { passed: true, details:
|
|
1471
|
+
return { passed: true, details: `${result.prompts.length} prompt(s), no nextCursor (single page)` };
|
|
851
1472
|
}
|
|
852
|
-
|
|
853
|
-
});
|
|
1473
|
+
);
|
|
854
1474
|
}
|
|
855
1475
|
await test("error-unknown-method", "Returns JSON-RPC error for unknown method", "errors", true, "basic", async () => {
|
|
856
1476
|
const res = await rpc("nonexistent/method");
|
|
@@ -862,16 +1482,23 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
862
1482
|
details: `Error code: ${error.code}${correctCode ? " (correct: Method not found)" : " (expected -32601)"} \u2014 ${error.message}`
|
|
863
1483
|
};
|
|
864
1484
|
});
|
|
865
|
-
await test(
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
1485
|
+
await test(
|
|
1486
|
+
"error-method-code",
|
|
1487
|
+
"Uses correct JSON-RPC error code for unknown method",
|
|
1488
|
+
"errors",
|
|
1489
|
+
false,
|
|
1490
|
+
"basic",
|
|
1491
|
+
async () => {
|
|
1492
|
+
const res = await rpc("nonexistent/method");
|
|
1493
|
+
const error = res.body?.error;
|
|
1494
|
+
if (!error) return { passed: false, details: "No error returned" };
|
|
1495
|
+
return { passed: error.code === -32601, details: `Expected -32601, got ${error.code}` };
|
|
1496
|
+
}
|
|
1497
|
+
);
|
|
871
1498
|
await test("error-invalid-jsonrpc", "Handles malformed JSON-RPC", "errors", true, "basic", async () => {
|
|
872
1499
|
const res = await request(backendUrl, {
|
|
873
1500
|
method: "POST",
|
|
874
|
-
headers: { "Content-Type": "application/json",
|
|
1501
|
+
headers: { "Content-Type": "application/json", Accept: "application/json, text/event-stream", ...buildHeaders() },
|
|
875
1502
|
body: JSON.stringify({ not: "a valid jsonrpc message" }),
|
|
876
1503
|
signal: AbortSignal.timeout(timeout)
|
|
877
1504
|
});
|
|
@@ -880,17 +1507,21 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
880
1507
|
const body = JSON.parse(text);
|
|
881
1508
|
if (body?.error) {
|
|
882
1509
|
const correctCode = body.error.code === -32600;
|
|
883
|
-
return {
|
|
1510
|
+
return {
|
|
1511
|
+
passed: true,
|
|
1512
|
+
details: `Error code: ${body.error.code}${correctCode ? " (correct: Invalid Request)" : ""} \u2014 ${body.error.message}`
|
|
1513
|
+
};
|
|
884
1514
|
}
|
|
885
1515
|
} catch {
|
|
886
1516
|
}
|
|
887
|
-
if (res.statusCode >= 400 && res.statusCode < 500)
|
|
1517
|
+
if (res.statusCode >= 400 && res.statusCode < 500)
|
|
1518
|
+
return { passed: true, details: `HTTP ${res.statusCode} (acceptable)` };
|
|
888
1519
|
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected JSON-RPC error or 4xx status` };
|
|
889
1520
|
});
|
|
890
1521
|
await test("error-invalid-json", "Handles invalid JSON body", "errors", false, "basic", async () => {
|
|
891
1522
|
const res = await request(backendUrl, {
|
|
892
1523
|
method: "POST",
|
|
893
|
-
headers: { "Content-Type": "application/json",
|
|
1524
|
+
headers: { "Content-Type": "application/json", Accept: "application/json, text/event-stream", ...buildHeaders() },
|
|
894
1525
|
body: "{this is not valid json!!!",
|
|
895
1526
|
signal: AbortSignal.timeout(timeout)
|
|
896
1527
|
});
|
|
@@ -900,24 +1531,35 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
900
1531
|
if (body?.error) return { passed: true, details: `Error code: ${body.error.code} \u2014 ${body.error.message}` };
|
|
901
1532
|
} catch {
|
|
902
1533
|
}
|
|
903
|
-
if (res.statusCode >= 400 && res.statusCode < 500)
|
|
1534
|
+
if (res.statusCode >= 400 && res.statusCode < 500)
|
|
1535
|
+
return { passed: true, details: `HTTP ${res.statusCode} (acceptable)` };
|
|
904
1536
|
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected parse error or 4xx status` };
|
|
905
1537
|
});
|
|
906
|
-
await test(
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
1538
|
+
await test(
|
|
1539
|
+
"error-missing-params",
|
|
1540
|
+
"Returns error for tools/call without name",
|
|
1541
|
+
"errors",
|
|
1542
|
+
false,
|
|
1543
|
+
"server/tools#error-handling",
|
|
1544
|
+
async () => {
|
|
1545
|
+
const res = await rpc("tools/call", {});
|
|
1546
|
+
const error = res.body?.error;
|
|
1547
|
+
const isError = res.body?.result?.isError;
|
|
1548
|
+
if (error) {
|
|
1549
|
+
const correctCode = error.code === -32602;
|
|
1550
|
+
return {
|
|
1551
|
+
passed: true,
|
|
1552
|
+
details: `Error code: ${error.code}${correctCode ? " (correct: Invalid params)" : ""} \u2014 ${error.message}`
|
|
1553
|
+
};
|
|
1554
|
+
}
|
|
1555
|
+
if (isError) return { passed: true, details: "Tool execution error (valid)" };
|
|
1556
|
+
return { passed: false, details: "No error for tools/call without name" };
|
|
913
1557
|
}
|
|
914
|
-
|
|
915
|
-
return { passed: false, details: "No error for tools/call without name" };
|
|
916
|
-
});
|
|
1558
|
+
);
|
|
917
1559
|
await test("error-parse-code", "Returns -32700 for invalid JSON", "errors", false, "basic", async () => {
|
|
918
1560
|
const res = await request(backendUrl, {
|
|
919
1561
|
method: "POST",
|
|
920
|
-
headers: { "Content-Type": "application/json",
|
|
1562
|
+
headers: { "Content-Type": "application/json", Accept: "application/json, text/event-stream", ...buildHeaders() },
|
|
921
1563
|
body: "<<<not json>>>",
|
|
922
1564
|
signal: AbortSignal.timeout(timeout)
|
|
923
1565
|
});
|
|
@@ -940,7 +1582,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
940
1582
|
await test("error-invalid-request-code", "Returns -32600 for invalid request", "errors", false, "basic", async () => {
|
|
941
1583
|
const res = await request(backendUrl, {
|
|
942
1584
|
method: "POST",
|
|
943
|
-
headers: { "Content-Type": "application/json",
|
|
1585
|
+
headers: { "Content-Type": "application/json", Accept: "application/json, text/event-stream", ...buildHeaders() },
|
|
944
1586
|
body: JSON.stringify({ jsonrpc: "2.0", id: 99999 }),
|
|
945
1587
|
signal: AbortSignal.timeout(timeout)
|
|
946
1588
|
});
|
|
@@ -985,6 +1627,194 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
985
1627
|
};
|
|
986
1628
|
}
|
|
987
1629
|
|
|
1630
|
+
// src/mcp/tools.ts
|
|
1631
|
+
function registerTools(server) {
|
|
1632
|
+
server.tool(
|
|
1633
|
+
"mcp_compliance_test",
|
|
1634
|
+
"Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all 43 tests covering transport, lifecycle, tools, resources, prompts, errors, and schema validation.",
|
|
1635
|
+
{
|
|
1636
|
+
url: z.string().url().describe("The MCP server URL to test (must be HTTP or HTTPS)"),
|
|
1637
|
+
auth: z.string().optional().describe('Authorization header value (e.g., "Bearer tok123")'),
|
|
1638
|
+
headers: z.record(z.string()).optional().describe('Additional headers to include on all requests (e.g., {"X-Api-Key": "abc"})'),
|
|
1639
|
+
timeout: z.number().optional().describe("Request timeout in milliseconds (default: 15000)"),
|
|
1640
|
+
retries: z.number().optional().describe("Number of retries for failed tests (default: 0)"),
|
|
1641
|
+
only: z.array(z.string()).optional().describe("Only run tests matching these categories or test IDs"),
|
|
1642
|
+
skip: z.array(z.string()).optional().describe("Skip tests matching these categories or test IDs")
|
|
1643
|
+
},
|
|
1644
|
+
{
|
|
1645
|
+
title: "Run MCP Compliance Tests",
|
|
1646
|
+
readOnlyHint: true,
|
|
1647
|
+
destructiveHint: false,
|
|
1648
|
+
idempotentHint: true,
|
|
1649
|
+
openWorldHint: true
|
|
1650
|
+
},
|
|
1651
|
+
async ({ url, auth, headers: extraHeaders, timeout, retries, only, skip }) => {
|
|
1652
|
+
try {
|
|
1653
|
+
const headers = { ...extraHeaders };
|
|
1654
|
+
if (auth) headers.Authorization = auth;
|
|
1655
|
+
const report = await runComplianceSuite(url, {
|
|
1656
|
+
headers: Object.keys(headers).length > 0 ? headers : void 0,
|
|
1657
|
+
timeout,
|
|
1658
|
+
retries,
|
|
1659
|
+
only,
|
|
1660
|
+
skip
|
|
1661
|
+
});
|
|
1662
|
+
const summary = [
|
|
1663
|
+
`Grade: ${report.grade} (${report.score}%)`,
|
|
1664
|
+
`Overall: ${report.overall}`,
|
|
1665
|
+
`Tests: ${report.summary.passed}/${report.summary.total} passed (${report.summary.requiredPassed}/${report.summary.required} required)`,
|
|
1666
|
+
"",
|
|
1667
|
+
...report.tests.map(
|
|
1668
|
+
(t) => `${t.passed ? "PASS" : "FAIL"} ${t.name}${t.required ? " (required)" : ""} \u2014 ${t.details}`
|
|
1669
|
+
)
|
|
1670
|
+
];
|
|
1671
|
+
if (report.serverInfo.name) {
|
|
1672
|
+
summary.unshift(`Server: ${report.serverInfo.name} v${report.serverInfo.version || "?"}`);
|
|
1673
|
+
}
|
|
1674
|
+
if (report.warnings.length > 0) {
|
|
1675
|
+
summary.push("", `Warnings (${report.warnings.length}):`);
|
|
1676
|
+
for (const w of report.warnings) {
|
|
1677
|
+
summary.push(` - ${w}`);
|
|
1678
|
+
}
|
|
1679
|
+
}
|
|
1680
|
+
return {
|
|
1681
|
+
content: [
|
|
1682
|
+
{ type: "text", text: summary.join("\n") },
|
|
1683
|
+
{ type: "text", text: `
|
|
1684
|
+
|
|
1685
|
+
Full report:
|
|
1686
|
+
${JSON.stringify(report, null, 2)}` }
|
|
1687
|
+
]
|
|
1688
|
+
};
|
|
1689
|
+
} catch (err) {
|
|
1690
|
+
return {
|
|
1691
|
+
content: [{ type: "text", text: `Error running compliance test: ${err.message}` }],
|
|
1692
|
+
isError: true
|
|
1693
|
+
};
|
|
1694
|
+
}
|
|
1695
|
+
}
|
|
1696
|
+
);
|
|
1697
|
+
server.tool(
|
|
1698
|
+
"mcp_compliance_badge",
|
|
1699
|
+
"Get the badge markdown embed code for an MCP server. Runs the compliance test suite first to determine the grade.",
|
|
1700
|
+
{
|
|
1701
|
+
url: z.string().url().describe("The MCP server URL to test"),
|
|
1702
|
+
auth: z.string().optional().describe('Authorization header value (e.g., "Bearer tok123")'),
|
|
1703
|
+
headers: z.record(z.string()).optional().describe("Additional headers to include on all requests"),
|
|
1704
|
+
timeout: z.number().optional().describe("Request timeout in milliseconds (default: 15000)")
|
|
1705
|
+
},
|
|
1706
|
+
{
|
|
1707
|
+
title: "Get Compliance Badge",
|
|
1708
|
+
readOnlyHint: true,
|
|
1709
|
+
destructiveHint: false,
|
|
1710
|
+
idempotentHint: true,
|
|
1711
|
+
openWorldHint: true
|
|
1712
|
+
},
|
|
1713
|
+
async ({ url, auth, headers: extraHeaders, timeout }) => {
|
|
1714
|
+
try {
|
|
1715
|
+
const headers = { ...extraHeaders };
|
|
1716
|
+
if (auth) headers.Authorization = auth;
|
|
1717
|
+
const report = await runComplianceSuite(url, {
|
|
1718
|
+
headers: Object.keys(headers).length > 0 ? headers : void 0,
|
|
1719
|
+
timeout
|
|
1720
|
+
});
|
|
1721
|
+
const badge = report.badge;
|
|
1722
|
+
return {
|
|
1723
|
+
content: [
|
|
1724
|
+
{
|
|
1725
|
+
type: "text",
|
|
1726
|
+
text: [
|
|
1727
|
+
`Grade: ${report.grade} (${report.score}%)`,
|
|
1728
|
+
"",
|
|
1729
|
+
"Markdown:",
|
|
1730
|
+
badge.markdown,
|
|
1731
|
+
"",
|
|
1732
|
+
"HTML:",
|
|
1733
|
+
badge.html
|
|
1734
|
+
].join("\n")
|
|
1735
|
+
}
|
|
1736
|
+
]
|
|
1737
|
+
};
|
|
1738
|
+
} catch (err) {
|
|
1739
|
+
return {
|
|
1740
|
+
content: [{ type: "text", text: `Error: ${err.message}` }],
|
|
1741
|
+
isError: true
|
|
1742
|
+
};
|
|
1743
|
+
}
|
|
1744
|
+
}
|
|
1745
|
+
);
|
|
1746
|
+
server.tool(
|
|
1747
|
+
"mcp_compliance_explain",
|
|
1748
|
+
"Explain what a specific compliance test ID checks and why it matters.",
|
|
1749
|
+
{
|
|
1750
|
+
testId: z.string().describe('The test ID to explain (e.g., "transport-post", "lifecycle-init", "tools-schema")')
|
|
1751
|
+
},
|
|
1752
|
+
{
|
|
1753
|
+
title: "Explain Compliance Test",
|
|
1754
|
+
readOnlyHint: true,
|
|
1755
|
+
destructiveHint: false,
|
|
1756
|
+
idempotentHint: true,
|
|
1757
|
+
openWorldHint: false
|
|
1758
|
+
},
|
|
1759
|
+
async ({ testId }) => {
|
|
1760
|
+
const def = TEST_DEFINITIONS.find((t) => t.id === testId);
|
|
1761
|
+
if (!def) {
|
|
1762
|
+
return {
|
|
1763
|
+
content: [
|
|
1764
|
+
{
|
|
1765
|
+
type: "text",
|
|
1766
|
+
text: `Unknown test ID: "${testId}"
|
|
1767
|
+
|
|
1768
|
+
Valid test IDs:
|
|
1769
|
+
${TEST_DEFINITIONS.map((t) => t.id).join(", ")}`
|
|
1770
|
+
}
|
|
1771
|
+
],
|
|
1772
|
+
isError: true
|
|
1773
|
+
};
|
|
1774
|
+
}
|
|
1775
|
+
return {
|
|
1776
|
+
content: [
|
|
1777
|
+
{
|
|
1778
|
+
type: "text",
|
|
1779
|
+
text: [
|
|
1780
|
+
`Test: ${def.id}`,
|
|
1781
|
+
`Name: ${def.name}`,
|
|
1782
|
+
`Category: ${def.category}`,
|
|
1783
|
+
`Required: ${def.required ? "Yes" : "No"}`,
|
|
1784
|
+
`Spec reference: https://modelcontextprotocol.io/specification/2025-11-25/${def.specRef}`,
|
|
1785
|
+
"",
|
|
1786
|
+
def.description,
|
|
1787
|
+
"",
|
|
1788
|
+
`Fix: ${def.recommendation}`
|
|
1789
|
+
].join("\n")
|
|
1790
|
+
}
|
|
1791
|
+
]
|
|
1792
|
+
};
|
|
1793
|
+
}
|
|
1794
|
+
);
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
// src/mcp/server.ts
|
|
1798
|
+
var require2 = createRequire2(import.meta.url);
|
|
1799
|
+
var { version } = require2("../../package.json");
|
|
1800
|
+
function createComplianceServer() {
|
|
1801
|
+
const server = new McpServer({ name: "mcp-compliance", version });
|
|
1802
|
+
registerTools(server);
|
|
1803
|
+
return server;
|
|
1804
|
+
}
|
|
1805
|
+
async function startServer() {
|
|
1806
|
+
const server = createComplianceServer();
|
|
1807
|
+
const transport = new StdioServerTransport();
|
|
1808
|
+
await server.connect(transport);
|
|
1809
|
+
}
|
|
1810
|
+
var isDirectRun = process.argv[1]?.endsWith("mcp/server.js") || process.argv[1]?.endsWith("mcp\\server.js");
|
|
1811
|
+
if (isDirectRun) {
|
|
1812
|
+
startServer().catch((err) => {
|
|
1813
|
+
console.error("MCP server error:", err);
|
|
1814
|
+
process.exit(1);
|
|
1815
|
+
});
|
|
1816
|
+
}
|
|
1817
|
+
|
|
988
1818
|
// src/reporter.ts
|
|
989
1819
|
import chalk from "chalk";
|
|
990
1820
|
var CATEGORY_LABELS = {
|
|
@@ -1027,8 +1857,16 @@ function testLine(t) {
|
|
|
1027
1857
|
const icon = t.passed ? chalk.green(" PASS") : chalk.red(" FAIL");
|
|
1028
1858
|
const req = t.required ? chalk.dim(" (required)") : "";
|
|
1029
1859
|
const dur = chalk.dim(` ${t.durationMs}ms`);
|
|
1030
|
-
|
|
1860
|
+
let line = `${icon} ${t.name}${req}${dur}
|
|
1031
1861
|
${chalk.dim(` ${t.details}`)}`;
|
|
1862
|
+
if (!t.passed) {
|
|
1863
|
+
const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
|
|
1864
|
+
if (def?.recommendation) {
|
|
1865
|
+
line += `
|
|
1866
|
+
${chalk.cyan(` Fix: ${def.recommendation}`)}`;
|
|
1867
|
+
}
|
|
1868
|
+
}
|
|
1869
|
+
return line;
|
|
1032
1870
|
}
|
|
1033
1871
|
function formatTerminal(report) {
|
|
1034
1872
|
const lines = [];
|
|
@@ -1037,11 +1875,19 @@ function formatTerminal(report) {
|
|
|
1037
1875
|
lines.push(chalk.dim(`Spec: ${report.specVersion} | Tool: v${report.toolVersion} | ${report.timestamp}`));
|
|
1038
1876
|
lines.push(chalk.dim(`URL: ${report.url}`));
|
|
1039
1877
|
if (report.serverInfo.name) {
|
|
1040
|
-
lines.push(
|
|
1878
|
+
lines.push(
|
|
1879
|
+
chalk.dim(
|
|
1880
|
+
`Server: ${report.serverInfo.name} v${report.serverInfo.version || "?"} (protocol ${report.serverInfo.protocolVersion || "?"})`
|
|
1881
|
+
)
|
|
1882
|
+
);
|
|
1041
1883
|
}
|
|
1042
1884
|
lines.push("");
|
|
1043
|
-
lines.push(
|
|
1044
|
-
|
|
1885
|
+
lines.push(
|
|
1886
|
+
` Grade: ${gradeColor(report.grade)} Score: ${chalk.bold(String(report.score))}% Overall: ${overallColor(report.overall)}`
|
|
1887
|
+
);
|
|
1888
|
+
lines.push(
|
|
1889
|
+
` Tests: ${chalk.green(String(report.summary.passed))} passed / ${chalk.red(String(report.summary.failed))} failed / ${report.summary.total} total`
|
|
1890
|
+
);
|
|
1045
1891
|
lines.push(` Required: ${report.summary.requiredPassed}/${report.summary.required} passed`);
|
|
1046
1892
|
const grouped = {};
|
|
1047
1893
|
for (const t of report.tests) {
|
|
@@ -1067,13 +1913,25 @@ function formatTerminal(report) {
|
|
|
1067
1913
|
lines.push(chalk.dim(` Capabilities: ${declared.join(", ")}`));
|
|
1068
1914
|
}
|
|
1069
1915
|
if (report.toolCount > 0) {
|
|
1070
|
-
lines.push(
|
|
1916
|
+
lines.push(
|
|
1917
|
+
chalk.dim(
|
|
1918
|
+
` Tools (${report.toolCount}): ${report.toolNames.slice(0, 10).join(", ")}${report.toolCount > 10 ? "..." : ""}`
|
|
1919
|
+
)
|
|
1920
|
+
);
|
|
1071
1921
|
}
|
|
1072
1922
|
if (report.resourceCount > 0) {
|
|
1073
|
-
lines.push(
|
|
1923
|
+
lines.push(
|
|
1924
|
+
chalk.dim(
|
|
1925
|
+
` Resources (${report.resourceCount}): ${report.resourceNames.slice(0, 10).join(", ")}${report.resourceCount > 10 ? "..." : ""}`
|
|
1926
|
+
)
|
|
1927
|
+
);
|
|
1074
1928
|
}
|
|
1075
1929
|
if (report.promptCount > 0) {
|
|
1076
|
-
lines.push(
|
|
1930
|
+
lines.push(
|
|
1931
|
+
chalk.dim(
|
|
1932
|
+
` Prompts (${report.promptCount}): ${report.promptNames.slice(0, 10).join(", ")}${report.promptCount > 10 ? "..." : ""}`
|
|
1933
|
+
)
|
|
1934
|
+
);
|
|
1077
1935
|
}
|
|
1078
1936
|
if (report.warnings.length > 0) {
|
|
1079
1937
|
lines.push("");
|
|
@@ -1091,131 +1949,80 @@ function formatTerminal(report) {
|
|
|
1091
1949
|
function formatJson(report) {
|
|
1092
1950
|
return JSON.stringify(report, null, 2);
|
|
1093
1951
|
}
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
for (const w of report.warnings) {
|
|
1126
|
-
summary.push(` - ${w}`);
|
|
1952
|
+
function formatSarif(report) {
|
|
1953
|
+
const SPEC_BASE2 = `https://modelcontextprotocol.io/specification/${report.specVersion}`;
|
|
1954
|
+
const rules = report.tests.map((t) => {
|
|
1955
|
+
const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
|
|
1956
|
+
return {
|
|
1957
|
+
id: t.id,
|
|
1958
|
+
name: t.name,
|
|
1959
|
+
shortDescription: { text: t.name },
|
|
1960
|
+
fullDescription: { text: def?.description || t.details },
|
|
1961
|
+
helpUri: t.specRef || `${SPEC_BASE2}/basic`,
|
|
1962
|
+
properties: {
|
|
1963
|
+
category: t.category,
|
|
1964
|
+
required: t.required
|
|
1965
|
+
}
|
|
1966
|
+
};
|
|
1967
|
+
});
|
|
1968
|
+
const results = report.tests.filter((t) => !t.passed).map((t) => {
|
|
1969
|
+
const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
|
|
1970
|
+
return {
|
|
1971
|
+
ruleId: t.id,
|
|
1972
|
+
level: t.required ? "error" : "warning",
|
|
1973
|
+
message: {
|
|
1974
|
+
text: def?.recommendation ? `${t.details}. Fix: ${def.recommendation}` : t.details
|
|
1975
|
+
},
|
|
1976
|
+
locations: [
|
|
1977
|
+
{
|
|
1978
|
+
physicalLocation: {
|
|
1979
|
+
artifactLocation: {
|
|
1980
|
+
uri: report.url,
|
|
1981
|
+
uriBaseId: "MCP_SERVER"
|
|
1982
|
+
}
|
|
1127
1983
|
}
|
|
1128
1984
|
}
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
Full report:
|
|
1135
|
-
${JSON.stringify(report, null, 2)}` }
|
|
1136
|
-
]
|
|
1137
|
-
};
|
|
1138
|
-
} catch (err) {
|
|
1139
|
-
return {
|
|
1140
|
-
content: [{ type: "text", text: `Error running compliance test: ${err.message}` }],
|
|
1141
|
-
isError: true
|
|
1142
|
-
};
|
|
1143
|
-
}
|
|
1144
|
-
}
|
|
1145
|
-
);
|
|
1146
|
-
server.tool(
|
|
1147
|
-
"mcp_compliance_badge",
|
|
1148
|
-
"Get the badge markdown embed code for an MCP server. Runs the compliance test suite first to determine the grade.",
|
|
1149
|
-
{
|
|
1150
|
-
url: z.string().url().describe("The MCP server URL to test")
|
|
1151
|
-
},
|
|
1152
|
-
async ({ url }) => {
|
|
1153
|
-
try {
|
|
1154
|
-
const report = await runComplianceSuite(url);
|
|
1155
|
-
const badge = report.badge;
|
|
1156
|
-
return {
|
|
1157
|
-
content: [{
|
|
1158
|
-
type: "text",
|
|
1159
|
-
text: [
|
|
1160
|
-
`Grade: ${report.grade} (${report.score}%)`,
|
|
1161
|
-
"",
|
|
1162
|
-
"Markdown:",
|
|
1163
|
-
badge.markdown,
|
|
1164
|
-
"",
|
|
1165
|
-
"HTML:",
|
|
1166
|
-
badge.html
|
|
1167
|
-
].join("\n")
|
|
1168
|
-
}]
|
|
1169
|
-
};
|
|
1170
|
-
} catch (err) {
|
|
1171
|
-
return {
|
|
1172
|
-
content: [{ type: "text", text: `Error: ${err.message}` }],
|
|
1173
|
-
isError: true
|
|
1174
|
-
};
|
|
1985
|
+
],
|
|
1986
|
+
properties: {
|
|
1987
|
+
category: t.category,
|
|
1988
|
+
durationMs: t.durationMs
|
|
1175
1989
|
}
|
|
1176
|
-
}
|
|
1177
|
-
);
|
|
1178
|
-
|
|
1179
|
-
"
|
|
1180
|
-
"
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1990
|
+
};
|
|
1991
|
+
});
|
|
1992
|
+
const sarif = {
|
|
1993
|
+
$schema: "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json",
|
|
1994
|
+
version: "2.1.0",
|
|
1995
|
+
runs: [
|
|
1996
|
+
{
|
|
1997
|
+
tool: {
|
|
1998
|
+
driver: {
|
|
1999
|
+
name: "mcp-compliance",
|
|
2000
|
+
version: report.toolVersion,
|
|
2001
|
+
informationUri: "https://github.com/YawLabs/mcp-compliance",
|
|
2002
|
+
rules
|
|
2003
|
+
}
|
|
2004
|
+
},
|
|
2005
|
+
results,
|
|
2006
|
+
invocations: [
|
|
2007
|
+
{
|
|
2008
|
+
executionSuccessful: report.overall !== "fail",
|
|
2009
|
+
properties: {
|
|
2010
|
+
grade: report.grade,
|
|
2011
|
+
score: report.score,
|
|
2012
|
+
overall: report.overall,
|
|
2013
|
+
specVersion: report.specVersion
|
|
2014
|
+
}
|
|
2015
|
+
}
|
|
2016
|
+
]
|
|
1197
2017
|
}
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
text: [
|
|
1202
|
-
`Test: ${def.id}`,
|
|
1203
|
-
`Name: ${def.name}`,
|
|
1204
|
-
`Category: ${def.category}`,
|
|
1205
|
-
`Required: ${def.required ? "Yes" : "No"}`,
|
|
1206
|
-
`Spec reference: https://modelcontextprotocol.io/specification/2025-11-25/${def.specRef}`,
|
|
1207
|
-
"",
|
|
1208
|
-
def.description
|
|
1209
|
-
].join("\n")
|
|
1210
|
-
}]
|
|
1211
|
-
};
|
|
1212
|
-
}
|
|
1213
|
-
);
|
|
2018
|
+
]
|
|
2019
|
+
};
|
|
2020
|
+
return JSON.stringify(sarif, null, 2);
|
|
1214
2021
|
}
|
|
1215
2022
|
|
|
1216
2023
|
// src/index.ts
|
|
1217
|
-
var
|
|
1218
|
-
var { version } =
|
|
2024
|
+
var require3 = createRequire3(import.meta.url);
|
|
2025
|
+
var { version: version2 } = require3("../package.json");
|
|
1219
2026
|
function parseHeaderArg(value, prev) {
|
|
1220
2027
|
const idx = value.indexOf(":");
|
|
1221
2028
|
if (idx === -1) {
|
|
@@ -1230,59 +2037,63 @@ function parseList(value) {
|
|
|
1230
2037
|
return value.split(",").map((s) => s.trim()).filter(Boolean);
|
|
1231
2038
|
}
|
|
1232
2039
|
var program = new Command();
|
|
1233
|
-
program.name("mcp-compliance").description("Test MCP servers for spec compliance").version(
|
|
1234
|
-
program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal or
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
2040
|
+
program.name("mcp-compliance").description("Test MCP servers for spec compliance").version(version2);
|
|
2041
|
+
program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal, json, or sarif", "terminal").option("--strict", "Exit with code 1 on any required test failure (for CI)").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").option("--retries <n>", "Number of retries for failed tests", "0").option("--only <items>", "Only run tests matching these categories or test IDs (comma-separated)", parseList).option("--skip <items>", "Skip tests matching these categories or test IDs (comma-separated)", parseList).option("--verbose", "Print each test result as it runs").action(
|
|
2042
|
+
async (url, opts) => {
|
|
2043
|
+
try {
|
|
2044
|
+
const headers = { ...opts.header };
|
|
2045
|
+
if (opts.auth) headers.Authorization = opts.auth;
|
|
2046
|
+
if (opts.format === "terminal") {
|
|
2047
|
+
console.log(chalk2.dim(`
|
|
1240
2048
|
Testing ${url}...
|
|
1241
2049
|
`));
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
2050
|
+
}
|
|
2051
|
+
const report = await runComplianceSuite(url, {
|
|
2052
|
+
headers,
|
|
2053
|
+
timeout: Number.parseInt(opts.timeout, 10) || 15e3,
|
|
2054
|
+
retries: Number.parseInt(opts.retries, 10) || 0,
|
|
2055
|
+
only: opts.only,
|
|
2056
|
+
skip: opts.skip,
|
|
2057
|
+
onProgress: opts.verbose ? (testId, passed, details) => {
|
|
2058
|
+
const icon = passed ? chalk2.green("PASS") : chalk2.red("FAIL");
|
|
2059
|
+
console.log(` ${icon} ${testId} \u2014 ${details}`);
|
|
2060
|
+
} : void 0
|
|
2061
|
+
});
|
|
2062
|
+
if (opts.verbose && opts.format === "terminal") {
|
|
2063
|
+
console.log("");
|
|
2064
|
+
}
|
|
2065
|
+
if (opts.format === "json") {
|
|
2066
|
+
console.log(formatJson(report));
|
|
2067
|
+
} else if (opts.format === "sarif") {
|
|
2068
|
+
console.log(formatSarif(report));
|
|
2069
|
+
} else {
|
|
2070
|
+
console.log(formatTerminal(report));
|
|
2071
|
+
}
|
|
2072
|
+
if (opts.strict && report.overall === "fail") {
|
|
2073
|
+
process.exit(1);
|
|
2074
|
+
}
|
|
2075
|
+
} catch (err) {
|
|
2076
|
+
if (opts.format === "json" || opts.format === "sarif") {
|
|
2077
|
+
console.error(JSON.stringify({ error: err.message }));
|
|
2078
|
+
} else {
|
|
2079
|
+
console.error(chalk2.red(`
|
|
1270
2080
|
Error: ${err.message}
|
|
1271
2081
|
`));
|
|
2082
|
+
}
|
|
2083
|
+
process.exit(1);
|
|
1272
2084
|
}
|
|
1273
|
-
process.exit(1);
|
|
1274
2085
|
}
|
|
1275
|
-
|
|
2086
|
+
);
|
|
1276
2087
|
program.command("badge").description("Run tests and output just the badge markdown embed code").argument("<url>", "MCP server URL to test").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").action(async (url, opts) => {
|
|
1277
2088
|
try {
|
|
1278
2089
|
const headers = { ...opts.header };
|
|
1279
|
-
if (opts.auth) headers
|
|
2090
|
+
if (opts.auth) headers.Authorization = opts.auth;
|
|
1280
2091
|
console.log(chalk2.dim(`
|
|
1281
2092
|
Testing ${url}...
|
|
1282
2093
|
`));
|
|
1283
2094
|
const report = await runComplianceSuite(url, {
|
|
1284
2095
|
headers,
|
|
1285
|
-
timeout: parseInt(opts.timeout, 10) || 15e3
|
|
2096
|
+
timeout: Number.parseInt(opts.timeout, 10) || 15e3
|
|
1286
2097
|
});
|
|
1287
2098
|
console.log(`Grade: ${report.grade} (${report.score}%)
|
|
1288
2099
|
`);
|
|
@@ -1296,9 +2107,6 @@ Error: ${err.message}
|
|
|
1296
2107
|
}
|
|
1297
2108
|
});
|
|
1298
2109
|
program.command("mcp").description("Start the MCP compliance server (stdio transport)").action(async () => {
|
|
1299
|
-
|
|
1300
|
-
registerTools(server);
|
|
1301
|
-
const transport = new StdioServerTransport();
|
|
1302
|
-
await server.connect(transport);
|
|
2110
|
+
await startServer();
|
|
1303
2111
|
});
|
|
1304
2112
|
program.parse();
|