@yawlabs/mcp-compliance 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -19
- package/dist/chunk-U66YZGE5.js +1578 -0
- package/dist/index.js +1348 -646
- package/dist/mcp/server.js +163 -111
- package/dist/runner.d.ts +4 -4
- package/dist/runner.js +1 -1
- package/package.json +10 -3
- package/dist/chunk-SP24UFRC.js +0 -987
package/dist/index.js
CHANGED
|
@@ -1,13 +1,37 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { Command } from "commander";
|
|
5
|
-
import chalk2 from "chalk";
|
|
6
4
|
import { createRequire as createRequire2 } from "module";
|
|
5
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
6
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
7
|
+
import chalk2 from "chalk";
|
|
8
|
+
import { Command } from "commander";
|
|
9
|
+
|
|
10
|
+
// src/mcp/tools.ts
|
|
11
|
+
import { z } from "zod";
|
|
7
12
|
|
|
8
13
|
// src/runner.ts
|
|
9
|
-
import { request } from "undici";
|
|
10
14
|
import { createRequire } from "module";
|
|
15
|
+
import { request } from "undici";
|
|
16
|
+
|
|
17
|
+
// src/badge.ts
|
|
18
|
+
function generateBadge(url) {
|
|
19
|
+
let parsed;
|
|
20
|
+
try {
|
|
21
|
+
parsed = new URL(url);
|
|
22
|
+
} catch {
|
|
23
|
+
parsed = new URL("https://unknown");
|
|
24
|
+
}
|
|
25
|
+
const encoded = encodeURIComponent(parsed.href);
|
|
26
|
+
const imageUrl = `https://mcp.hosting/api/compliance/${encoded}/badge`;
|
|
27
|
+
const reportUrl = `https://mcp.hosting/compliance/${encoded}`;
|
|
28
|
+
return {
|
|
29
|
+
imageUrl,
|
|
30
|
+
reportUrl,
|
|
31
|
+
markdown: `[](${reportUrl})`,
|
|
32
|
+
html: `<a href="${reportUrl}"><img src="${imageUrl}" alt="MCP Compliant"></a>`
|
|
33
|
+
};
|
|
34
|
+
}
|
|
11
35
|
|
|
12
36
|
// src/grader.ts
|
|
13
37
|
function computeGrade(score) {
|
|
@@ -44,77 +68,359 @@ function computeScore(tests) {
|
|
|
44
68
|
};
|
|
45
69
|
}
|
|
46
70
|
|
|
47
|
-
// src/badge.ts
|
|
48
|
-
function generateBadge(url) {
|
|
49
|
-
let parsed;
|
|
50
|
-
try {
|
|
51
|
-
parsed = new URL(url);
|
|
52
|
-
} catch {
|
|
53
|
-
parsed = new URL("https://unknown");
|
|
54
|
-
}
|
|
55
|
-
const encoded = encodeURIComponent(parsed.href);
|
|
56
|
-
const imageUrl = `https://mcp.hosting/api/compliance/${encoded}/badge`;
|
|
57
|
-
const reportUrl = `https://mcp.hosting/compliance/${encoded}`;
|
|
58
|
-
return {
|
|
59
|
-
imageUrl,
|
|
60
|
-
reportUrl,
|
|
61
|
-
markdown: `[](${reportUrl})`,
|
|
62
|
-
html: `<a href="${reportUrl}"><img src="${imageUrl}" alt="MCP Compliant"></a>`
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
|
|
66
71
|
// src/types.ts
|
|
67
72
|
var TEST_DEFINITIONS = [
|
|
68
73
|
// ── Transport (7 tests) ──────────────────────────────────────────
|
|
69
|
-
{
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
74
|
+
{
|
|
75
|
+
id: "transport-post",
|
|
76
|
+
name: "HTTP POST accepted",
|
|
77
|
+
category: "transport",
|
|
78
|
+
required: true,
|
|
79
|
+
specRef: "basic/transports#streamable-http",
|
|
80
|
+
description: "Verifies the server accepts HTTP POST requests and returns a 2xx status code. This is the fundamental transport requirement for Streamable HTTP MCP servers."
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
id: "transport-content-type",
|
|
84
|
+
name: "Responds with JSON or SSE",
|
|
85
|
+
category: "transport",
|
|
86
|
+
required: true,
|
|
87
|
+
specRef: "basic/transports#streamable-http",
|
|
88
|
+
description: "Checks that the server responds with Content-Type application/json or text/event-stream. MCP servers must use one of these two content types."
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
id: "transport-notification-202",
|
|
92
|
+
name: "Notification returns 202 Accepted",
|
|
93
|
+
category: "transport",
|
|
94
|
+
required: false,
|
|
95
|
+
specRef: "basic/transports#streamable-http",
|
|
96
|
+
description: "Verifies that sending a JSON-RPC notification (no id field) returns HTTP 202 Accepted with no body. Per spec, servers MUST return 202 for notifications."
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
id: "transport-session-id",
|
|
100
|
+
name: "Enforces MCP-Session-Id after init",
|
|
101
|
+
category: "transport",
|
|
102
|
+
required: false,
|
|
103
|
+
specRef: "basic/transports#streamable-http",
|
|
104
|
+
description: "Tests that the server returns HTTP 400 when MCP-Session-Id header is missing on requests after initialization (when the server issued a session ID)."
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
id: "transport-get",
|
|
108
|
+
name: "GET returns SSE stream or 405",
|
|
109
|
+
category: "transport",
|
|
110
|
+
required: false,
|
|
111
|
+
specRef: "basic/transports#streamable-http",
|
|
112
|
+
description: "Tests the GET endpoint for server-initiated messages. Server should return text/event-stream or 405 Method Not Allowed."
|
|
113
|
+
},
|
|
114
|
+
{
|
|
115
|
+
id: "transport-delete",
|
|
116
|
+
name: "DELETE accepted or returns 405",
|
|
117
|
+
category: "transport",
|
|
118
|
+
required: false,
|
|
119
|
+
specRef: "basic/transports#streamable-http",
|
|
120
|
+
description: "Tests the DELETE endpoint for session termination. Server should accept the request or return 405 Method Not Allowed."
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
id: "transport-batch-reject",
|
|
124
|
+
name: "Rejects JSON-RPC batch requests",
|
|
125
|
+
category: "transport",
|
|
126
|
+
required: true,
|
|
127
|
+
specRef: "basic/transports#streamable-http",
|
|
128
|
+
description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests."
|
|
129
|
+
},
|
|
76
130
|
// ── Lifecycle (10 tests) ─────────────────────────────────────────
|
|
77
|
-
{
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
{
|
|
86
|
-
|
|
131
|
+
{
|
|
132
|
+
id: "lifecycle-init",
|
|
133
|
+
name: "Initialize handshake",
|
|
134
|
+
category: "lifecycle",
|
|
135
|
+
required: true,
|
|
136
|
+
specRef: "basic/lifecycle#initialization",
|
|
137
|
+
description: "Tests the initialize handshake by sending an initialize request with client capabilities. The server must return a result with protocolVersion."
|
|
138
|
+
},
|
|
139
|
+
{
|
|
140
|
+
id: "lifecycle-proto-version",
|
|
141
|
+
name: "Returns valid protocol version",
|
|
142
|
+
category: "lifecycle",
|
|
143
|
+
required: true,
|
|
144
|
+
specRef: "basic/lifecycle#version-negotiation",
|
|
145
|
+
description: "Validates that the protocolVersion returned by the server matches the YYYY-MM-DD date format required by the spec."
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
id: "lifecycle-server-info",
|
|
149
|
+
name: "Includes serverInfo",
|
|
150
|
+
category: "lifecycle",
|
|
151
|
+
required: false,
|
|
152
|
+
specRef: "basic/lifecycle#initialization",
|
|
153
|
+
description: "Checks that the server includes a serverInfo object with at least a name field in its initialize response. While recommended, this is not strictly required."
|
|
154
|
+
},
|
|
155
|
+
{
|
|
156
|
+
id: "lifecycle-capabilities",
|
|
157
|
+
name: "Returns capabilities object",
|
|
158
|
+
category: "lifecycle",
|
|
159
|
+
required: true,
|
|
160
|
+
specRef: "basic/lifecycle#capability-negotiation",
|
|
161
|
+
description: "Verifies the server returns a capabilities object in its initialize response. An empty object is valid (no optional features declared)."
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
id: "lifecycle-jsonrpc",
|
|
165
|
+
name: "Response is valid JSON-RPC 2.0",
|
|
166
|
+
category: "lifecycle",
|
|
167
|
+
required: true,
|
|
168
|
+
specRef: "basic",
|
|
169
|
+
description: 'Validates that the initialize response is a proper JSON-RPC 2.0 message with jsonrpc="2.0", an id field, and either a result or error field.'
|
|
170
|
+
},
|
|
171
|
+
{
|
|
172
|
+
id: "lifecycle-ping",
|
|
173
|
+
name: "Responds to ping",
|
|
174
|
+
category: "lifecycle",
|
|
175
|
+
required: true,
|
|
176
|
+
specRef: "basic/utilities#ping",
|
|
177
|
+
description: "Tests that the server responds to the ping method with an empty result object. This is a required utility method."
|
|
178
|
+
},
|
|
179
|
+
{
|
|
180
|
+
id: "lifecycle-instructions",
|
|
181
|
+
name: "Instructions field is valid",
|
|
182
|
+
category: "lifecycle",
|
|
183
|
+
required: false,
|
|
184
|
+
specRef: "basic/lifecycle#initialization",
|
|
185
|
+
description: "If the server includes an instructions field in the initialize response, validates it is a string. Instructions provide guidance for how the client should interact with the server."
|
|
186
|
+
},
|
|
187
|
+
{
|
|
188
|
+
id: "lifecycle-id-match",
|
|
189
|
+
name: "Response ID matches request ID",
|
|
190
|
+
category: "lifecycle",
|
|
191
|
+
required: true,
|
|
192
|
+
specRef: "basic",
|
|
193
|
+
description: "Verifies that the JSON-RPC response id matches the request id sent by the client. This is a fundamental JSON-RPC 2.0 requirement."
|
|
194
|
+
},
|
|
195
|
+
{
|
|
196
|
+
id: "lifecycle-logging",
|
|
197
|
+
name: "logging/setLevel accepted",
|
|
198
|
+
category: "lifecycle",
|
|
199
|
+
required: false,
|
|
200
|
+
specRef: "server/utilities#logging",
|
|
201
|
+
description: "If the server declares logging capability, tests that logging/setLevel method is accepted with a valid log level."
|
|
202
|
+
},
|
|
203
|
+
{
|
|
204
|
+
id: "lifecycle-completions",
|
|
205
|
+
name: "completion/complete accepted",
|
|
206
|
+
category: "lifecycle",
|
|
207
|
+
required: false,
|
|
208
|
+
specRef: "server/utilities#completion",
|
|
209
|
+
description: "If the server declares completions capability, tests that the completion/complete method is accepted."
|
|
210
|
+
},
|
|
87
211
|
// ── Tools (4 tests) ──────────────────────────────────────────────
|
|
88
|
-
{
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
212
|
+
{
|
|
213
|
+
id: "tools-list",
|
|
214
|
+
name: "tools/list returns valid response",
|
|
215
|
+
category: "tools",
|
|
216
|
+
required: false,
|
|
217
|
+
specRef: "server/tools#listing-tools",
|
|
218
|
+
description: "Calls tools/list and validates it returns an array of tool definitions. Required if the server declares tools capability."
|
|
219
|
+
},
|
|
220
|
+
{
|
|
221
|
+
id: "tools-call",
|
|
222
|
+
name: "tools/call responds correctly",
|
|
223
|
+
category: "tools",
|
|
224
|
+
required: false,
|
|
225
|
+
specRef: "server/tools#calling-tools",
|
|
226
|
+
description: "Calls the first tool with empty arguments and verifies the response format. Accepts both successful results and InvalidParams errors."
|
|
227
|
+
},
|
|
228
|
+
{
|
|
229
|
+
id: "tools-pagination",
|
|
230
|
+
name: "tools/list supports pagination",
|
|
231
|
+
category: "tools",
|
|
232
|
+
required: false,
|
|
233
|
+
specRef: "server/tools#listing-tools",
|
|
234
|
+
description: "Tests cursor-based pagination on tools/list. Validates nextCursor is a string if present and that fetching the next page returns a valid response."
|
|
235
|
+
},
|
|
236
|
+
{
|
|
237
|
+
id: "tools-content-types",
|
|
238
|
+
name: "Tool content items have valid types",
|
|
239
|
+
category: "tools",
|
|
240
|
+
required: false,
|
|
241
|
+
specRef: "server/tools#calling-tools",
|
|
242
|
+
description: "Validates that content items returned by tools/call have a recognized type field (text, image, audio, resource, resource_link)."
|
|
243
|
+
},
|
|
92
244
|
// ── Resources (5 tests) ──────────────────────────────────────────
|
|
93
|
-
{
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
245
|
+
{
|
|
246
|
+
id: "resources-list",
|
|
247
|
+
name: "resources/list returns valid response",
|
|
248
|
+
category: "resources",
|
|
249
|
+
required: false,
|
|
250
|
+
specRef: "server/resources#listing-resources",
|
|
251
|
+
description: "Calls resources/list and validates it returns an array. Required if the server declares resources capability."
|
|
252
|
+
},
|
|
253
|
+
{
|
|
254
|
+
id: "resources-read",
|
|
255
|
+
name: "resources/read returns content",
|
|
256
|
+
category: "resources",
|
|
257
|
+
required: false,
|
|
258
|
+
specRef: "server/resources#reading-resources",
|
|
259
|
+
description: "Reads the first resource and validates the response contains a contents array with proper uri and text/blob fields."
|
|
260
|
+
},
|
|
261
|
+
{
|
|
262
|
+
id: "resources-templates",
|
|
263
|
+
name: "resources/templates/list returns valid response",
|
|
264
|
+
category: "resources",
|
|
265
|
+
required: false,
|
|
266
|
+
specRef: "server/resources#resource-templates",
|
|
267
|
+
description: "Tests the resource templates endpoint. Accepts Method not found (-32601) since templates are optional."
|
|
268
|
+
},
|
|
269
|
+
{
|
|
270
|
+
id: "resources-pagination",
|
|
271
|
+
name: "resources/list supports pagination",
|
|
272
|
+
category: "resources",
|
|
273
|
+
required: false,
|
|
274
|
+
specRef: "server/resources#listing-resources",
|
|
275
|
+
description: "Tests cursor-based pagination on resources/list. Validates nextCursor is a string if present and that fetching the next page works."
|
|
276
|
+
},
|
|
277
|
+
{
|
|
278
|
+
id: "resources-subscribe",
|
|
279
|
+
name: "Resource subscribe/unsubscribe",
|
|
280
|
+
category: "resources",
|
|
281
|
+
required: false,
|
|
282
|
+
specRef: "server/resources#subscriptions",
|
|
283
|
+
description: "If the server declares resources.subscribe capability, tests that resources/subscribe and resources/unsubscribe methods are accepted."
|
|
284
|
+
},
|
|
98
285
|
// ── Prompts (3 tests) ────────────────────────────────────────────
|
|
99
|
-
{
|
|
100
|
-
|
|
101
|
-
|
|
286
|
+
{
|
|
287
|
+
id: "prompts-list",
|
|
288
|
+
name: "prompts/list returns valid response",
|
|
289
|
+
category: "prompts",
|
|
290
|
+
required: false,
|
|
291
|
+
specRef: "server/prompts#listing-prompts",
|
|
292
|
+
description: "Calls prompts/list and validates it returns an array. Required if the server declares prompts capability."
|
|
293
|
+
},
|
|
294
|
+
{
|
|
295
|
+
id: "prompts-get",
|
|
296
|
+
name: "prompts/get returns valid messages",
|
|
297
|
+
category: "prompts",
|
|
298
|
+
required: false,
|
|
299
|
+
specRef: "server/prompts#getting-a-prompt",
|
|
300
|
+
description: "Gets the first prompt and validates the response contains a messages array with proper role and content fields."
|
|
301
|
+
},
|
|
302
|
+
{
|
|
303
|
+
id: "prompts-pagination",
|
|
304
|
+
name: "prompts/list supports pagination",
|
|
305
|
+
category: "prompts",
|
|
306
|
+
required: false,
|
|
307
|
+
specRef: "server/prompts#listing-prompts",
|
|
308
|
+
description: "Tests cursor-based pagination on prompts/list. Validates nextCursor is a string if present and that fetching the next page works."
|
|
309
|
+
},
|
|
102
310
|
// ── Error Handling (8 tests) ─────────────────────────────────────
|
|
103
|
-
{
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
311
|
+
{
|
|
312
|
+
id: "error-unknown-method",
|
|
313
|
+
name: "Returns JSON-RPC error for unknown method",
|
|
314
|
+
category: "errors",
|
|
315
|
+
required: true,
|
|
316
|
+
specRef: "basic",
|
|
317
|
+
description: "Sends an unknown method and verifies the server returns a JSON-RPC error. The spec requires error code -32601 (Method not found)."
|
|
318
|
+
},
|
|
319
|
+
{
|
|
320
|
+
id: "error-method-code",
|
|
321
|
+
name: "Uses correct JSON-RPC error code for unknown method",
|
|
322
|
+
category: "errors",
|
|
323
|
+
required: false,
|
|
324
|
+
specRef: "basic",
|
|
325
|
+
description: "Checks the error code is specifically -32601 (Method not found) for unknown methods, as required by JSON-RPC 2.0."
|
|
326
|
+
},
|
|
327
|
+
{
|
|
328
|
+
id: "error-invalid-jsonrpc",
|
|
329
|
+
name: "Handles malformed JSON-RPC",
|
|
330
|
+
category: "errors",
|
|
331
|
+
required: true,
|
|
332
|
+
specRef: "basic",
|
|
333
|
+
description: "Sends a malformed JSON-RPC message (missing required fields) and verifies the server returns an error or 4xx status."
|
|
334
|
+
},
|
|
335
|
+
{
|
|
336
|
+
id: "error-invalid-json",
|
|
337
|
+
name: "Handles invalid JSON body",
|
|
338
|
+
category: "errors",
|
|
339
|
+
required: false,
|
|
340
|
+
specRef: "basic",
|
|
341
|
+
description: "Sends invalid JSON and verifies the server returns a parse error (-32700) or 4xx status code."
|
|
342
|
+
},
|
|
343
|
+
{
|
|
344
|
+
id: "error-missing-params",
|
|
345
|
+
name: "Returns error for tools/call without name",
|
|
346
|
+
category: "errors",
|
|
347
|
+
required: false,
|
|
348
|
+
specRef: "server/tools#error-handling",
|
|
349
|
+
description: "Calls tools/call with an empty params object (missing required name field) and verifies an error is returned."
|
|
350
|
+
},
|
|
351
|
+
{
|
|
352
|
+
id: "error-parse-code",
|
|
353
|
+
name: "Returns -32700 for invalid JSON",
|
|
354
|
+
category: "errors",
|
|
355
|
+
required: false,
|
|
356
|
+
specRef: "basic",
|
|
357
|
+
description: "Checks that the server returns the specific JSON-RPC error code -32700 (Parse error) when receiving invalid JSON, as required by the JSON-RPC 2.0 specification."
|
|
358
|
+
},
|
|
359
|
+
{
|
|
360
|
+
id: "error-invalid-request-code",
|
|
361
|
+
name: "Returns -32600 for invalid request",
|
|
362
|
+
category: "errors",
|
|
363
|
+
required: false,
|
|
364
|
+
specRef: "basic",
|
|
365
|
+
description: "Checks that the server returns the specific JSON-RPC error code -32600 (Invalid Request) for malformed JSON-RPC messages missing required fields."
|
|
366
|
+
},
|
|
367
|
+
{
|
|
368
|
+
id: "tools-call-unknown",
|
|
369
|
+
name: "Returns error for unknown tool name",
|
|
370
|
+
category: "errors",
|
|
371
|
+
required: false,
|
|
372
|
+
specRef: "server/tools#error-handling",
|
|
373
|
+
description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response."
|
|
374
|
+
},
|
|
111
375
|
// ── Schema Validation (6 tests) ──────────────────────────────────
|
|
112
|
-
{
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
376
|
+
{
|
|
377
|
+
id: "tools-schema",
|
|
378
|
+
name: "All tools have name and inputSchema",
|
|
379
|
+
category: "schema",
|
|
380
|
+
required: false,
|
|
381
|
+
specRef: "server/tools#data-types",
|
|
382
|
+
description: 'Validates every tool has a valid name (1-128 chars, alphanumeric/underscore/hyphen/dot) and a required inputSchema of type "object".'
|
|
383
|
+
},
|
|
384
|
+
{
|
|
385
|
+
id: "tools-annotations",
|
|
386
|
+
name: "Tool annotations are valid",
|
|
387
|
+
category: "schema",
|
|
388
|
+
required: false,
|
|
389
|
+
specRef: "server/tools#annotations",
|
|
390
|
+
description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans; title should be a string."
|
|
391
|
+
},
|
|
392
|
+
{
|
|
393
|
+
id: "tools-title-field",
|
|
394
|
+
name: "Tools include title field",
|
|
395
|
+
category: "schema",
|
|
396
|
+
required: false,
|
|
397
|
+
specRef: "server/tools#data-types",
|
|
398
|
+
description: "Checks if tools include the optional title field for human-readable display names. Added in spec version 2025-11-25."
|
|
399
|
+
},
|
|
400
|
+
{
|
|
401
|
+
id: "tools-output-schema",
|
|
402
|
+
name: "Tools with outputSchema are valid",
|
|
403
|
+
category: "schema",
|
|
404
|
+
required: false,
|
|
405
|
+
specRef: "server/tools#structured-content",
|
|
406
|
+
description: 'If tools declare an outputSchema, validates it is a valid JSON Schema object with type "object". Used for structured output validation.'
|
|
407
|
+
},
|
|
408
|
+
{
|
|
409
|
+
id: "prompts-schema",
|
|
410
|
+
name: "Prompts have name field",
|
|
411
|
+
category: "schema",
|
|
412
|
+
required: false,
|
|
413
|
+
specRef: "server/prompts#data-types",
|
|
414
|
+
description: "Validates every prompt has a name and that any arguments array contains items with name fields."
|
|
415
|
+
},
|
|
416
|
+
{
|
|
417
|
+
id: "resources-schema",
|
|
418
|
+
name: "Resources have uri and name",
|
|
419
|
+
category: "schema",
|
|
420
|
+
required: false,
|
|
421
|
+
specRef: "server/resources#data-types",
|
|
422
|
+
description: "Validates every resource has a valid URI (parseable as a URL) and a name field."
|
|
423
|
+
}
|
|
118
424
|
];
|
|
119
425
|
|
|
120
426
|
// src/runner.ts
|
|
@@ -165,7 +471,7 @@ async function mcpRequest(backendUrl, method, params, nextId, extraHeaders, time
|
|
|
165
471
|
});
|
|
166
472
|
const headers = {
|
|
167
473
|
"Content-Type": "application/json",
|
|
168
|
-
|
|
474
|
+
Accept: "application/json, text/event-stream",
|
|
169
475
|
...extraHeaders
|
|
170
476
|
};
|
|
171
477
|
const res = await request(backendUrl, {
|
|
@@ -200,7 +506,7 @@ async function mcpRequest(backendUrl, method, params, nextId, extraHeaders, time
|
|
|
200
506
|
async function mcpNotification(backendUrl, method, params, extraHeaders, timeout) {
|
|
201
507
|
const headers = {
|
|
202
508
|
"Content-Type": "application/json",
|
|
203
|
-
|
|
509
|
+
Accept: "application/json, text/event-stream",
|
|
204
510
|
...extraHeaders
|
|
205
511
|
};
|
|
206
512
|
const res = await request(backendUrl, {
|
|
@@ -252,7 +558,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
252
558
|
}
|
|
253
559
|
return true;
|
|
254
560
|
}
|
|
255
|
-
|
|
561
|
+
const serverInfo = {
|
|
256
562
|
protocolVersion: null,
|
|
257
563
|
name: null,
|
|
258
564
|
version: null,
|
|
@@ -290,94 +596,129 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
290
596
|
});
|
|
291
597
|
options.onProgress?.(id, lastResult.passed, lastResult.details);
|
|
292
598
|
}
|
|
293
|
-
await test(
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
signal: AbortSignal.timeout(timeout)
|
|
311
|
-
});
|
|
312
|
-
await res.body.text();
|
|
313
|
-
const rawCt = res.headers["content-type"];
|
|
314
|
-
const ct = (Array.isArray(rawCt) ? rawCt[0] : rawCt || "").toLowerCase();
|
|
315
|
-
const valid = ct.includes("application/json") || ct.includes("text/event-stream");
|
|
316
|
-
return { passed: valid, details: `Content-Type: ${ct}` };
|
|
317
|
-
});
|
|
318
|
-
await test("transport-get", "GET returns SSE stream or 405", "transport", false, "basic/transports#streamable-http", async () => {
|
|
319
|
-
const res = await request(backendUrl, {
|
|
320
|
-
method: "GET",
|
|
321
|
-
headers: { "Accept": "text/event-stream", ...userHeaders },
|
|
322
|
-
signal: AbortSignal.timeout(timeout)
|
|
323
|
-
});
|
|
324
|
-
await res.body.text();
|
|
325
|
-
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
326
|
-
if (res.statusCode === 405) {
|
|
327
|
-
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
328
|
-
}
|
|
329
|
-
if (ct.includes("text/event-stream")) {
|
|
330
|
-
return { passed: true, details: "Returns text/event-stream for SSE" };
|
|
331
|
-
}
|
|
332
|
-
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
333
|
-
return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
|
|
334
|
-
}
|
|
335
|
-
return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
|
|
336
|
-
});
|
|
337
|
-
await test("transport-delete", "DELETE accepted or returns 405", "transport", false, "basic/transports#streamable-http", async () => {
|
|
338
|
-
const res = await request(backendUrl, {
|
|
339
|
-
method: "DELETE",
|
|
340
|
-
headers: { ...userHeaders },
|
|
341
|
-
signal: AbortSignal.timeout(timeout)
|
|
342
|
-
});
|
|
343
|
-
await res.body.text();
|
|
344
|
-
if (res.statusCode === 405) {
|
|
345
|
-
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
599
|
+
await test(
|
|
600
|
+
"transport-post",
|
|
601
|
+
"HTTP POST accepted",
|
|
602
|
+
"transport",
|
|
603
|
+
true,
|
|
604
|
+
"basic/transports#streamable-http",
|
|
605
|
+
async () => {
|
|
606
|
+
const res = await request(backendUrl, {
|
|
607
|
+
method: "POST",
|
|
608
|
+
headers: { "Content-Type": "application/json", Accept: "application/json, text/event-stream", ...userHeaders },
|
|
609
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 99901, method: "ping" }),
|
|
610
|
+
signal: AbortSignal.timeout(timeout)
|
|
611
|
+
});
|
|
612
|
+
await res.body.text();
|
|
613
|
+
const passed = res.statusCode >= 200 && res.statusCode < 300;
|
|
614
|
+
const note = res.statusCode === 401 || res.statusCode === 403 ? " (auth required)" : "";
|
|
615
|
+
return { passed, details: `HTTP ${res.statusCode}${note}` };
|
|
346
616
|
}
|
|
347
|
-
|
|
348
|
-
|
|
617
|
+
);
|
|
618
|
+
await test(
|
|
619
|
+
"transport-content-type",
|
|
620
|
+
"Responds with JSON or SSE",
|
|
621
|
+
"transport",
|
|
622
|
+
true,
|
|
623
|
+
"basic/transports#streamable-http",
|
|
624
|
+
async () => {
|
|
625
|
+
const res = await request(backendUrl, {
|
|
626
|
+
method: "POST",
|
|
627
|
+
headers: { "Content-Type": "application/json", Accept: "application/json, text/event-stream", ...userHeaders },
|
|
628
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 99902, method: "ping" }),
|
|
629
|
+
signal: AbortSignal.timeout(timeout)
|
|
630
|
+
});
|
|
631
|
+
await res.body.text();
|
|
632
|
+
const rawCt = res.headers["content-type"];
|
|
633
|
+
const ct = (Array.isArray(rawCt) ? rawCt[0] : rawCt || "").toLowerCase();
|
|
634
|
+
const valid = ct.includes("application/json") || ct.includes("text/event-stream");
|
|
635
|
+
return { passed: valid, details: `Content-Type: ${ct}` };
|
|
349
636
|
}
|
|
350
|
-
|
|
351
|
-
|
|
637
|
+
);
|
|
638
|
+
await test(
|
|
639
|
+
"transport-get",
|
|
640
|
+
"GET returns SSE stream or 405",
|
|
641
|
+
"transport",
|
|
642
|
+
false,
|
|
643
|
+
"basic/transports#streamable-http",
|
|
644
|
+
async () => {
|
|
645
|
+
const res = await request(backendUrl, {
|
|
646
|
+
method: "GET",
|
|
647
|
+
headers: { Accept: "text/event-stream", ...userHeaders },
|
|
648
|
+
signal: AbortSignal.timeout(timeout)
|
|
649
|
+
});
|
|
650
|
+
await res.body.text();
|
|
651
|
+
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
652
|
+
if (res.statusCode === 405) {
|
|
653
|
+
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
654
|
+
}
|
|
655
|
+
if (ct.includes("text/event-stream")) {
|
|
656
|
+
return { passed: true, details: "Returns text/event-stream for SSE" };
|
|
657
|
+
}
|
|
658
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
659
|
+
return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
|
|
660
|
+
}
|
|
661
|
+
return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
|
|
352
662
|
}
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
663
|
+
);
|
|
664
|
+
await test(
|
|
665
|
+
"transport-delete",
|
|
666
|
+
"DELETE accepted or returns 405",
|
|
667
|
+
"transport",
|
|
668
|
+
false,
|
|
669
|
+
"basic/transports#streamable-http",
|
|
670
|
+
async () => {
|
|
671
|
+
const res = await request(backendUrl, {
|
|
672
|
+
method: "DELETE",
|
|
673
|
+
headers: { ...userHeaders },
|
|
674
|
+
signal: AbortSignal.timeout(timeout)
|
|
675
|
+
});
|
|
676
|
+
await res.body.text();
|
|
677
|
+
if (res.statusCode === 405) {
|
|
678
|
+
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
679
|
+
}
|
|
680
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
681
|
+
return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
|
|
682
|
+
}
|
|
683
|
+
if (res.statusCode === 400 || res.statusCode === 404) {
|
|
684
|
+
return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
|
|
685
|
+
}
|
|
686
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
368
687
|
}
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
688
|
+
);
|
|
689
|
+
await test(
|
|
690
|
+
"transport-batch-reject",
|
|
691
|
+
"Rejects JSON-RPC batch requests",
|
|
692
|
+
"transport",
|
|
693
|
+
true,
|
|
694
|
+
"basic/transports#streamable-http",
|
|
695
|
+
async () => {
|
|
696
|
+
const res = await request(backendUrl, {
|
|
697
|
+
method: "POST",
|
|
698
|
+
headers: { "Content-Type": "application/json", Accept: "application/json, text/event-stream", ...userHeaders },
|
|
699
|
+
body: JSON.stringify([
|
|
700
|
+
{ jsonrpc: "2.0", id: 99903, method: "ping" },
|
|
701
|
+
{ jsonrpc: "2.0", id: 99904, method: "ping" }
|
|
702
|
+
]),
|
|
703
|
+
signal: AbortSignal.timeout(timeout)
|
|
704
|
+
});
|
|
705
|
+
const text = await res.body.text();
|
|
706
|
+
if (res.statusCode >= 400 && res.statusCode < 500) {
|
|
707
|
+
return { passed: true, details: `HTTP ${res.statusCode} (batch rejected)` };
|
|
373
708
|
}
|
|
374
|
-
|
|
375
|
-
|
|
709
|
+
try {
|
|
710
|
+
const body = JSON.parse(text);
|
|
711
|
+
if (body?.error) {
|
|
712
|
+
return { passed: true, details: `JSON-RPC error: ${body.error.code} \u2014 ${body.error.message}` };
|
|
713
|
+
}
|
|
714
|
+
if (Array.isArray(body)) {
|
|
715
|
+
return { passed: false, details: "Server processed batch request (MCP forbids batch)" };
|
|
716
|
+
}
|
|
717
|
+
} catch {
|
|
376
718
|
}
|
|
377
|
-
|
|
719
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error or 4xx for batch request` };
|
|
378
720
|
}
|
|
379
|
-
|
|
380
|
-
});
|
|
721
|
+
);
|
|
381
722
|
let initRes = null;
|
|
382
723
|
try {
|
|
383
724
|
initRes = await rpc("initialize", {
|
|
@@ -401,35 +742,69 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
401
742
|
await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders(), timeout);
|
|
402
743
|
} catch {
|
|
403
744
|
}
|
|
404
|
-
await test(
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
warnings.push(`Server negotiated protocol version ${version2} (latest is ${SPEC_VERSION})`);
|
|
745
|
+
await test(
|
|
746
|
+
"lifecycle-init",
|
|
747
|
+
"Initialize handshake",
|
|
748
|
+
"lifecycle",
|
|
749
|
+
true,
|
|
750
|
+
"basic/lifecycle#initialization",
|
|
751
|
+
async () => {
|
|
752
|
+
if (!initRes) return { passed: false, details: "Initialize request failed" };
|
|
753
|
+
const result = initRes.body?.result;
|
|
754
|
+
if (!result) return { passed: false, details: "No result in response" };
|
|
755
|
+
return { passed: !!result.protocolVersion, details: `Protocol: ${result.protocolVersion || "missing"}` };
|
|
416
756
|
}
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
757
|
+
);
|
|
758
|
+
await test(
|
|
759
|
+
"lifecycle-proto-version",
|
|
760
|
+
"Returns valid protocol version",
|
|
761
|
+
"lifecycle",
|
|
762
|
+
true,
|
|
763
|
+
"basic/lifecycle#version-negotiation",
|
|
764
|
+
async () => {
|
|
765
|
+
const version2 = initRes?.body?.result?.protocolVersion;
|
|
766
|
+
if (!version2) return { passed: false, details: "No protocolVersion" };
|
|
767
|
+
const valid = /^\d{4}-\d{2}-\d{2}$/.test(version2);
|
|
768
|
+
if (valid && version2 !== SPEC_VERSION) {
|
|
769
|
+
warnings.push(`Server negotiated protocol version ${version2} (latest is ${SPEC_VERSION})`);
|
|
770
|
+
}
|
|
771
|
+
return { passed: valid, details: `Version: ${version2}` };
|
|
772
|
+
}
|
|
773
|
+
);
|
|
774
|
+
await test(
|
|
775
|
+
"lifecycle-server-info",
|
|
776
|
+
"Includes serverInfo",
|
|
777
|
+
"lifecycle",
|
|
778
|
+
false,
|
|
779
|
+
"basic/lifecycle#initialization",
|
|
780
|
+
async () => {
|
|
781
|
+
const info = initRes?.body?.result?.serverInfo;
|
|
782
|
+
return { passed: !!info?.name, details: info ? `${info.name} v${info.version || "?"}` : "Missing serverInfo" };
|
|
783
|
+
}
|
|
784
|
+
);
|
|
785
|
+
await test(
|
|
786
|
+
"lifecycle-capabilities",
|
|
787
|
+
"Returns capabilities object",
|
|
788
|
+
"lifecycle",
|
|
789
|
+
true,
|
|
790
|
+
"basic/lifecycle#capability-negotiation",
|
|
791
|
+
async () => {
|
|
792
|
+
const caps = initRes?.body?.result?.capabilities;
|
|
793
|
+
if (!caps || typeof caps !== "object") return { passed: false, details: "No capabilities object in response" };
|
|
794
|
+
const declared = Object.keys(caps).filter((k) => caps[k] !== void 0);
|
|
795
|
+
return {
|
|
796
|
+
passed: true,
|
|
797
|
+
details: declared.length > 0 ? `Capabilities: ${declared.join(", ")}` : "Empty capabilities (valid)"
|
|
798
|
+
};
|
|
799
|
+
}
|
|
800
|
+
);
|
|
429
801
|
await test("lifecycle-jsonrpc", "Response is valid JSON-RPC 2.0", "lifecycle", true, "basic", async () => {
|
|
430
802
|
const body = initRes?.body;
|
|
431
803
|
const valid = body?.jsonrpc === "2.0" && body?.id !== void 0 && (body?.result !== void 0 || body?.error !== void 0);
|
|
432
|
-
return {
|
|
804
|
+
return {
|
|
805
|
+
passed: valid,
|
|
806
|
+
details: valid ? "Valid JSON-RPC 2.0 response" : `Missing fields: jsonrpc=${body?.jsonrpc}, id=${body?.id}`
|
|
807
|
+
};
|
|
433
808
|
});
|
|
434
809
|
await test("lifecycle-ping", "Responds to ping", "lifecycle", true, "basic/utilities#ping", async () => {
|
|
435
810
|
const res = await rpc("ping");
|
|
@@ -438,149 +813,223 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
438
813
|
if (body?.result !== void 0) return { passed: true, details: "Ping responded successfully" };
|
|
439
814
|
return { passed: false, details: "No result in ping response" };
|
|
440
815
|
});
|
|
441
|
-
await test(
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
const
|
|
449
|
-
return { passed:
|
|
816
|
+
await test(
|
|
817
|
+
"lifecycle-instructions",
|
|
818
|
+
"Instructions field is valid",
|
|
819
|
+
"lifecycle",
|
|
820
|
+
false,
|
|
821
|
+
"basic/lifecycle#initialization",
|
|
822
|
+
async () => {
|
|
823
|
+
const result = initRes?.body?.result;
|
|
824
|
+
if (!result) return { passed: false, details: "No init result" };
|
|
825
|
+
if (result.instructions === void 0) {
|
|
826
|
+
return { passed: true, details: "No instructions field (optional)" };
|
|
827
|
+
}
|
|
828
|
+
if (typeof result.instructions === "string") {
|
|
829
|
+
const preview = result.instructions.length > 80 ? result.instructions.slice(0, 80) + "..." : result.instructions;
|
|
830
|
+
return { passed: true, details: `Instructions: "${preview}"` };
|
|
831
|
+
}
|
|
832
|
+
return { passed: false, details: `instructions should be a string, got ${typeof result.instructions}` };
|
|
450
833
|
}
|
|
451
|
-
|
|
452
|
-
});
|
|
834
|
+
);
|
|
453
835
|
await test("lifecycle-id-match", "Response ID matches request ID", "lifecycle", true, "basic", async () => {
|
|
454
836
|
const res = await rpc("ping");
|
|
455
837
|
const body = res.body;
|
|
456
838
|
if (body?.id === void 0) return { passed: false, details: "No id in response" };
|
|
457
839
|
const match = body.id === res.requestId;
|
|
458
|
-
return {
|
|
840
|
+
return {
|
|
841
|
+
passed: match,
|
|
842
|
+
details: match ? `Request id=${res.requestId}, response id=${body.id} (match)` : `Request id=${res.requestId}, response id=${body.id} (MISMATCH)`
|
|
843
|
+
};
|
|
459
844
|
});
|
|
460
845
|
const hasLogging = !!serverInfo.capabilities.logging;
|
|
461
|
-
await test(
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
846
|
+
await test(
|
|
847
|
+
"lifecycle-logging",
|
|
848
|
+
"logging/setLevel accepted",
|
|
849
|
+
"lifecycle",
|
|
850
|
+
hasLogging,
|
|
851
|
+
"server/utilities#logging",
|
|
852
|
+
async () => {
|
|
853
|
+
if (!hasLogging) return { passed: true, details: "Server does not declare logging capability (skipped)" };
|
|
854
|
+
const res = await rpc("logging/setLevel", { level: "info" });
|
|
855
|
+
if (res.body?.error) {
|
|
856
|
+
return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
|
|
857
|
+
}
|
|
858
|
+
return { passed: true, details: "logging/setLevel accepted" };
|
|
466
859
|
}
|
|
467
|
-
|
|
468
|
-
});
|
|
860
|
+
);
|
|
469
861
|
const hasCompletions = !!serverInfo.capabilities.completions;
|
|
470
|
-
await test(
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
if (
|
|
478
|
-
|
|
862
|
+
await test(
|
|
863
|
+
"lifecycle-completions",
|
|
864
|
+
"completion/complete accepted",
|
|
865
|
+
"lifecycle",
|
|
866
|
+
hasCompletions,
|
|
867
|
+
"server/utilities#completion",
|
|
868
|
+
async () => {
|
|
869
|
+
if (!hasCompletions) return { passed: true, details: "Server does not declare completions capability (skipped)" };
|
|
870
|
+
const res = await rpc("completion/complete", {
|
|
871
|
+
ref: { type: "ref/prompt", name: "__test__" },
|
|
872
|
+
argument: { name: "test", value: "" }
|
|
873
|
+
});
|
|
874
|
+
if (res.body?.error) {
|
|
875
|
+
if (res.body.error.code === -32602) {
|
|
876
|
+
return { passed: true, details: "InvalidParams for test ref (acceptable)" };
|
|
877
|
+
}
|
|
878
|
+
return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
|
|
479
879
|
}
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
return { passed: true, details:
|
|
485
|
-
}
|
|
486
|
-
return { passed: true, details: "completion/complete accepted" };
|
|
487
|
-
});
|
|
488
|
-
await test("transport-notification-202", "Notification returns 202 Accepted", "transport", false, "basic/transports#streamable-http", async () => {
|
|
489
|
-
const res = await request(backendUrl, {
|
|
490
|
-
method: "POST",
|
|
491
|
-
headers: {
|
|
492
|
-
"Content-Type": "application/json",
|
|
493
|
-
"Accept": "application/json, text/event-stream",
|
|
494
|
-
...buildHeaders()
|
|
495
|
-
},
|
|
496
|
-
body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/cancelled", params: { requestId: "nonexistent", reason: "compliance test" } }),
|
|
497
|
-
signal: AbortSignal.timeout(timeout)
|
|
498
|
-
});
|
|
499
|
-
await res.body.text();
|
|
500
|
-
if (res.statusCode === 202) {
|
|
501
|
-
return { passed: true, details: "HTTP 202 Accepted (correct)" };
|
|
502
|
-
}
|
|
503
|
-
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
504
|
-
return { passed: true, details: `HTTP ${res.statusCode} (accepted, but 202 is preferred)` };
|
|
505
|
-
}
|
|
506
|
-
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected 202 Accepted for notifications` };
|
|
507
|
-
});
|
|
508
|
-
await test("transport-session-id", "Enforces MCP-Session-Id after init", "transport", false, "basic/transports#streamable-http", async () => {
|
|
509
|
-
if (!sessionId) {
|
|
510
|
-
warnings.push("Server did not issue MCP-Session-Id header");
|
|
511
|
-
return { passed: true, details: "Server did not issue session ID (test not applicable)" };
|
|
512
|
-
}
|
|
513
|
-
const headersWithout = { ...userHeaders };
|
|
514
|
-
if (negotiatedProtocolVersion) headersWithout["mcp-protocol-version"] = negotiatedProtocolVersion;
|
|
515
|
-
const res = await mcpRequest(backendUrl, "ping", void 0, createIdCounter(99910), headersWithout, timeout);
|
|
516
|
-
if (res.statusCode === 400) {
|
|
517
|
-
return { passed: true, details: "HTTP 400 for missing session ID (correct)" };
|
|
880
|
+
const values = res.body?.result?.completion?.values;
|
|
881
|
+
if (Array.isArray(values)) {
|
|
882
|
+
return { passed: true, details: `Returned ${values.length} completion(s)` };
|
|
883
|
+
}
|
|
884
|
+
return { passed: true, details: "completion/complete accepted" };
|
|
518
885
|
}
|
|
519
|
-
|
|
520
|
-
|
|
886
|
+
);
|
|
887
|
+
await test(
|
|
888
|
+
"transport-notification-202",
|
|
889
|
+
"Notification returns 202 Accepted",
|
|
890
|
+
"transport",
|
|
891
|
+
false,
|
|
892
|
+
"basic/transports#streamable-http",
|
|
893
|
+
async () => {
|
|
894
|
+
const res = await request(backendUrl, {
|
|
895
|
+
method: "POST",
|
|
896
|
+
headers: {
|
|
897
|
+
"Content-Type": "application/json",
|
|
898
|
+
Accept: "application/json, text/event-stream",
|
|
899
|
+
...buildHeaders()
|
|
900
|
+
},
|
|
901
|
+
body: JSON.stringify({
|
|
902
|
+
jsonrpc: "2.0",
|
|
903
|
+
method: "notifications/cancelled",
|
|
904
|
+
params: { requestId: "nonexistent", reason: "compliance test" }
|
|
905
|
+
}),
|
|
906
|
+
signal: AbortSignal.timeout(timeout)
|
|
907
|
+
});
|
|
908
|
+
await res.body.text();
|
|
909
|
+
if (res.statusCode === 202) {
|
|
910
|
+
return { passed: true, details: "HTTP 202 Accepted (correct)" };
|
|
911
|
+
}
|
|
912
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
913
|
+
return { passed: true, details: `HTTP ${res.statusCode} (accepted, but 202 is preferred)` };
|
|
914
|
+
}
|
|
915
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected 202 Accepted for notifications` };
|
|
521
916
|
}
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
return { passed: true, details: `${toolCount} tool(s): ${toolNames.slice(0, 5).join(", ")}${toolCount > 5 ? "..." : ""}` };
|
|
534
|
-
});
|
|
535
|
-
await test("tools-schema", "All tools have name and inputSchema", "schema", hasTools, "server/tools#data-types", async () => {
|
|
536
|
-
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
537
|
-
const issues = [];
|
|
538
|
-
for (const tool of tools) {
|
|
539
|
-
if (!tool.name) {
|
|
540
|
-
issues.push("Tool missing name");
|
|
541
|
-
continue;
|
|
917
|
+
);
|
|
918
|
+
await test(
|
|
919
|
+
"transport-session-id",
|
|
920
|
+
"Enforces MCP-Session-Id after init",
|
|
921
|
+
"transport",
|
|
922
|
+
false,
|
|
923
|
+
"basic/transports#streamable-http",
|
|
924
|
+
async () => {
|
|
925
|
+
if (!sessionId) {
|
|
926
|
+
warnings.push("Server did not issue MCP-Session-Id header");
|
|
927
|
+
return { passed: true, details: "Server did not issue session ID (test not applicable)" };
|
|
542
928
|
}
|
|
543
|
-
|
|
544
|
-
|
|
929
|
+
const headersWithout = { ...userHeaders };
|
|
930
|
+
if (negotiatedProtocolVersion) headersWithout["mcp-protocol-version"] = negotiatedProtocolVersion;
|
|
931
|
+
const res = await mcpRequest(backendUrl, "ping", void 0, createIdCounter(99910), headersWithout, timeout);
|
|
932
|
+
if (res.statusCode === 400) {
|
|
933
|
+
return { passed: true, details: "HTTP 400 for missing session ID (correct)" };
|
|
545
934
|
}
|
|
546
|
-
if (
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
} else if (tool.inputSchema.type !== "object") {
|
|
552
|
-
issues.push(`${tool.name}: inputSchema.type must be "object" (got "${tool.inputSchema.type || "undefined"}")`);
|
|
935
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
936
|
+
return {
|
|
937
|
+
passed: false,
|
|
938
|
+
details: `HTTP ${res.statusCode} \u2014 server should return 400 when session ID is missing`
|
|
939
|
+
};
|
|
553
940
|
}
|
|
941
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
554
942
|
}
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
await test(
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
if (
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
943
|
+
);
|
|
944
|
+
const hasTools = !!serverInfo.capabilities.tools;
|
|
945
|
+
let cachedToolsList = null;
|
|
946
|
+
await test(
|
|
947
|
+
"tools-list",
|
|
948
|
+
"tools/list returns valid response",
|
|
949
|
+
"tools",
|
|
950
|
+
hasTools,
|
|
951
|
+
"server/tools#listing-tools",
|
|
952
|
+
async () => {
|
|
953
|
+
const res = await rpc("tools/list");
|
|
954
|
+
const tools = res.body?.result?.tools;
|
|
955
|
+
if (!Array.isArray(tools)) return { passed: false, details: "No tools array in result" };
|
|
956
|
+
cachedToolsList = tools;
|
|
957
|
+
toolCount = tools.length;
|
|
958
|
+
toolNames = tools.map((t) => t.name).filter(Boolean);
|
|
959
|
+
return {
|
|
960
|
+
passed: true,
|
|
961
|
+
details: `${toolCount} tool(s): ${toolNames.slice(0, 5).join(", ")}${toolCount > 5 ? "..." : ""}`
|
|
962
|
+
};
|
|
963
|
+
}
|
|
964
|
+
);
|
|
965
|
+
await test(
|
|
966
|
+
"tools-schema",
|
|
967
|
+
"All tools have name and inputSchema",
|
|
968
|
+
"schema",
|
|
969
|
+
hasTools,
|
|
970
|
+
"server/tools#data-types",
|
|
971
|
+
async () => {
|
|
972
|
+
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
973
|
+
const issues = [];
|
|
974
|
+
for (const tool of tools) {
|
|
975
|
+
if (!tool.name) {
|
|
976
|
+
issues.push("Tool missing name");
|
|
977
|
+
continue;
|
|
978
|
+
}
|
|
979
|
+
if (tool.name.length > 128 || !/^[A-Za-z0-9_.\-]+$/.test(tool.name)) {
|
|
980
|
+
issues.push(`${tool.name}: name format invalid`);
|
|
981
|
+
}
|
|
982
|
+
if (!tool.description) warnings.push(`Tool "${tool.name}" missing description`);
|
|
983
|
+
if (!tool.inputSchema) {
|
|
984
|
+
issues.push(`${tool.name}: missing inputSchema (required)`);
|
|
985
|
+
} else if (typeof tool.inputSchema !== "object" || tool.inputSchema === null) {
|
|
986
|
+
issues.push(`${tool.name}: inputSchema must be a valid JSON Schema object`);
|
|
987
|
+
} else if (tool.inputSchema.type !== "object") {
|
|
988
|
+
issues.push(
|
|
989
|
+
`${tool.name}: inputSchema.type must be "object" (got "${tool.inputSchema.type || "undefined"}")`
|
|
990
|
+
);
|
|
575
991
|
}
|
|
576
992
|
}
|
|
577
|
-
|
|
578
|
-
|
|
993
|
+
const detail = issues.length === 0 ? "All tools have valid schemas" : issues.join("; ");
|
|
994
|
+
return { passed: issues.length === 0, details: detail };
|
|
995
|
+
}
|
|
996
|
+
);
|
|
997
|
+
await test(
|
|
998
|
+
"tools-annotations",
|
|
999
|
+
"Tool annotations are valid",
|
|
1000
|
+
"schema",
|
|
1001
|
+
false,
|
|
1002
|
+
"server/tools#annotations",
|
|
1003
|
+
async () => {
|
|
1004
|
+
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
1005
|
+
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1006
|
+
const issues = [];
|
|
1007
|
+
let annotatedCount = 0;
|
|
1008
|
+
for (const tool of tools) {
|
|
1009
|
+
const ann = tool.annotations;
|
|
1010
|
+
if (!ann) continue;
|
|
1011
|
+
annotatedCount++;
|
|
1012
|
+
if (typeof ann !== "object" || ann === null) {
|
|
1013
|
+
issues.push(`${tool.name}: annotations must be an object`);
|
|
1014
|
+
continue;
|
|
1015
|
+
}
|
|
1016
|
+
const boolFields = ["readOnlyHint", "destructiveHint", "idempotentHint", "openWorldHint"];
|
|
1017
|
+
for (const field of boolFields) {
|
|
1018
|
+
if (ann[field] !== void 0 && typeof ann[field] !== "boolean") {
|
|
1019
|
+
issues.push(`${tool.name}: annotations.${field} should be boolean, got ${typeof ann[field]}`);
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
if (ann.title !== void 0 && typeof ann.title !== "string") {
|
|
1023
|
+
issues.push(`${tool.name}: annotations.title should be string`);
|
|
1024
|
+
}
|
|
579
1025
|
}
|
|
1026
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1027
|
+
return {
|
|
1028
|
+
passed: true,
|
|
1029
|
+
details: annotatedCount > 0 ? `${annotatedCount} tool(s) with valid annotations` : "No tools have annotations (optional)"
|
|
1030
|
+
};
|
|
580
1031
|
}
|
|
581
|
-
|
|
582
|
-
return { passed: true, details: annotatedCount > 0 ? `${annotatedCount} tool(s) with valid annotations` : "No tools have annotations (optional)" };
|
|
583
|
-
});
|
|
1032
|
+
);
|
|
584
1033
|
await test("tools-title-field", "Tools include title field", "schema", false, "server/tools#data-types", async () => {
|
|
585
1034
|
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
586
1035
|
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
@@ -597,211 +1046,319 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
597
1046
|
}
|
|
598
1047
|
return { passed: true, details: `${withTitle.length}/${tools.length} tool(s) have title field` };
|
|
599
1048
|
});
|
|
600
|
-
await test(
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
if (
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
1049
|
+
await test(
|
|
1050
|
+
"tools-output-schema",
|
|
1051
|
+
"Tools with outputSchema are valid",
|
|
1052
|
+
"schema",
|
|
1053
|
+
false,
|
|
1054
|
+
"server/tools#structured-content",
|
|
1055
|
+
async () => {
|
|
1056
|
+
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
1057
|
+
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1058
|
+
const issues = [];
|
|
1059
|
+
let withSchema = 0;
|
|
1060
|
+
for (const tool of tools) {
|
|
1061
|
+
if (tool.outputSchema === void 0) continue;
|
|
1062
|
+
withSchema++;
|
|
1063
|
+
if (typeof tool.outputSchema !== "object" || tool.outputSchema === null) {
|
|
1064
|
+
issues.push(`${tool.name}: outputSchema must be a JSON Schema object`);
|
|
1065
|
+
} else if (tool.outputSchema.type !== "object") {
|
|
1066
|
+
issues.push(
|
|
1067
|
+
`${tool.name}: outputSchema.type must be "object" (got "${tool.outputSchema.type || "undefined"}")`
|
|
1068
|
+
);
|
|
1069
|
+
}
|
|
612
1070
|
}
|
|
1071
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1072
|
+
return {
|
|
1073
|
+
passed: true,
|
|
1074
|
+
details: withSchema > 0 ? `${withSchema} tool(s) with valid outputSchema` : "No tools have outputSchema (optional)"
|
|
1075
|
+
};
|
|
613
1076
|
}
|
|
614
|
-
|
|
615
|
-
return { passed: true, details: withSchema > 0 ? `${withSchema} tool(s) with valid outputSchema` : "No tools have outputSchema (optional)" };
|
|
616
|
-
});
|
|
1077
|
+
);
|
|
617
1078
|
if (toolNames.length > 0) {
|
|
618
|
-
await test(
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
1079
|
+
await test(
|
|
1080
|
+
"tools-call",
|
|
1081
|
+
"tools/call responds correctly",
|
|
1082
|
+
"tools",
|
|
1083
|
+
false,
|
|
1084
|
+
"server/tools#calling-tools",
|
|
1085
|
+
async () => {
|
|
1086
|
+
const res = await rpc("tools/call", { name: toolNames[0], arguments: {} });
|
|
1087
|
+
const result = res.body?.result;
|
|
1088
|
+
const error = res.body?.error;
|
|
1089
|
+
if (error) {
|
|
1090
|
+
const code = error.code;
|
|
1091
|
+
if (code === -32602 || code === -32600) {
|
|
1092
|
+
return { passed: true, details: `Invalid params error (acceptable): code ${code}` };
|
|
1093
|
+
}
|
|
1094
|
+
return { passed: true, details: `Protocol error: code ${code} \u2014 ${error.message}` };
|
|
626
1095
|
}
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
return { passed: false, details: "Response missing content array" };
|
|
638
|
-
});
|
|
639
|
-
await test("tools-content-types", "Tool content items have valid types", "tools", false, "server/tools#calling-tools", async () => {
|
|
640
|
-
const res = await rpc("tools/call", { name: toolNames[0], arguments: {} });
|
|
641
|
-
const result = res.body?.result;
|
|
642
|
-
const error = res.body?.error;
|
|
643
|
-
if (error) {
|
|
644
|
-
return { passed: true, details: `Tool returned error (content types not applicable): code ${error.code}` };
|
|
645
|
-
}
|
|
646
|
-
const content = result?.content;
|
|
647
|
-
if (!Array.isArray(content) || content.length === 0) {
|
|
648
|
-
return { passed: true, details: "No content items to validate" };
|
|
1096
|
+
if (result?.content && Array.isArray(result.content)) {
|
|
1097
|
+
const badItems = result.content.filter((c) => !c.type);
|
|
1098
|
+
if (badItems.length > 0)
|
|
1099
|
+
return { passed: false, details: `${badItems.length} content item(s) missing 'type' field` };
|
|
1100
|
+
return { passed: true, details: `Returned ${result.content.length} content item(s)` };
|
|
1101
|
+
}
|
|
1102
|
+
if (result?.isError && result?.content && Array.isArray(result.content)) {
|
|
1103
|
+
return { passed: true, details: "Tool returned execution error with content (valid)" };
|
|
1104
|
+
}
|
|
1105
|
+
return { passed: false, details: "Response missing content array" };
|
|
649
1106
|
}
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
1107
|
+
);
|
|
1108
|
+
await test(
|
|
1109
|
+
"tools-content-types",
|
|
1110
|
+
"Tool content items have valid types",
|
|
1111
|
+
"tools",
|
|
1112
|
+
false,
|
|
1113
|
+
"server/tools#calling-tools",
|
|
1114
|
+
async () => {
|
|
1115
|
+
const res = await rpc("tools/call", { name: toolNames[0], arguments: {} });
|
|
1116
|
+
const result = res.body?.result;
|
|
1117
|
+
const error = res.body?.error;
|
|
1118
|
+
if (error) {
|
|
1119
|
+
return { passed: true, details: `Tool returned error (content types not applicable): code ${error.code}` };
|
|
659
1120
|
}
|
|
1121
|
+
const content = result?.content;
|
|
1122
|
+
if (!Array.isArray(content) || content.length === 0) {
|
|
1123
|
+
return { passed: true, details: "No content items to validate" };
|
|
1124
|
+
}
|
|
1125
|
+
const issues = [];
|
|
1126
|
+
const types = /* @__PURE__ */ new Set();
|
|
1127
|
+
for (const item of content) {
|
|
1128
|
+
if (!item.type) {
|
|
1129
|
+
issues.push("Content item missing type field");
|
|
1130
|
+
} else if (!VALID_CONTENT_TYPES.includes(item.type)) {
|
|
1131
|
+
issues.push(`Unknown content type: "${item.type}"`);
|
|
1132
|
+
} else {
|
|
1133
|
+
types.add(item.type);
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1137
|
+
return { passed: true, details: `Content types: ${[...types].join(", ")}` };
|
|
660
1138
|
}
|
|
661
|
-
|
|
662
|
-
return { passed: true, details: `Content types: ${[...types].join(", ")}` };
|
|
663
|
-
});
|
|
1139
|
+
);
|
|
664
1140
|
}
|
|
665
1141
|
if (hasTools) {
|
|
666
|
-
await test(
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
if (
|
|
678
|
-
|
|
1142
|
+
await test(
|
|
1143
|
+
"tools-pagination",
|
|
1144
|
+
"tools/list supports pagination",
|
|
1145
|
+
"tools",
|
|
1146
|
+
false,
|
|
1147
|
+
"server/tools#listing-tools",
|
|
1148
|
+
async () => {
|
|
1149
|
+
const res = await rpc("tools/list");
|
|
1150
|
+
const result = res.body?.result;
|
|
1151
|
+
if (!result) return { passed: false, details: "No result from tools/list" };
|
|
1152
|
+
if (!Array.isArray(result.tools)) return { passed: false, details: "No tools array" };
|
|
1153
|
+
if (result.nextCursor !== void 0) {
|
|
1154
|
+
if (typeof result.nextCursor !== "string") {
|
|
1155
|
+
return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
|
|
1156
|
+
}
|
|
1157
|
+
const nextRes = await rpc("tools/list", { cursor: result.nextCursor });
|
|
1158
|
+
const nextResult = nextRes.body?.result;
|
|
1159
|
+
if (!nextResult || !Array.isArray(nextResult.tools)) {
|
|
1160
|
+
return { passed: false, details: "Next page failed to return tools array" };
|
|
1161
|
+
}
|
|
1162
|
+
return {
|
|
1163
|
+
passed: true,
|
|
1164
|
+
details: `Pagination works: page 1 had ${result.tools.length} tools, page 2 had ${nextResult.tools.length} tools`
|
|
1165
|
+
};
|
|
679
1166
|
}
|
|
680
|
-
return { passed: true, details:
|
|
1167
|
+
return { passed: true, details: `${result.tools.length} tool(s), no nextCursor (single page)` };
|
|
681
1168
|
}
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
1169
|
+
);
|
|
1170
|
+
await test(
|
|
1171
|
+
"tools-call-unknown",
|
|
1172
|
+
"Returns error for unknown tool name",
|
|
1173
|
+
"errors",
|
|
1174
|
+
false,
|
|
1175
|
+
"server/tools#error-handling",
|
|
1176
|
+
async () => {
|
|
1177
|
+
const res = await rpc("tools/call", { name: "__nonexistent_tool_compliance_test__", arguments: {} });
|
|
1178
|
+
const error = res.body?.error;
|
|
1179
|
+
const isError = res.body?.result?.isError;
|
|
1180
|
+
if (error) return { passed: true, details: `Error code: ${error.code} \u2014 ${error.message}` };
|
|
1181
|
+
if (isError) return { passed: true, details: "Tool execution error with isError=true (valid)" };
|
|
1182
|
+
return { passed: false, details: "No error returned for nonexistent tool" };
|
|
1183
|
+
}
|
|
1184
|
+
);
|
|
692
1185
|
}
|
|
693
1186
|
const hasResources = !!serverInfo.capabilities.resources;
|
|
694
1187
|
const hasSubscribe = !!serverInfo.capabilities.resources?.subscribe;
|
|
695
1188
|
if (hasResources) {
|
|
696
1189
|
let cachedResourcesList = null;
|
|
697
|
-
await test(
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
1190
|
+
await test(
|
|
1191
|
+
"resources-list",
|
|
1192
|
+
"resources/list returns valid response",
|
|
1193
|
+
"resources",
|
|
1194
|
+
true,
|
|
1195
|
+
"server/resources#listing-resources",
|
|
1196
|
+
async () => {
|
|
1197
|
+
const res = await rpc("resources/list");
|
|
1198
|
+
const resources = res.body?.result?.resources;
|
|
1199
|
+
if (!Array.isArray(resources)) return { passed: false, details: "No resources array" };
|
|
1200
|
+
cachedResourcesList = resources;
|
|
1201
|
+
resourceCount = resources.length;
|
|
1202
|
+
resourceNames = resources.map((r) => r.name).filter(Boolean);
|
|
1203
|
+
return { passed: true, details: `${resourceCount} resource(s)` };
|
|
1204
|
+
}
|
|
1205
|
+
);
|
|
1206
|
+
await test(
|
|
1207
|
+
"resources-schema",
|
|
1208
|
+
"Resources have uri and name",
|
|
1209
|
+
"schema",
|
|
1210
|
+
true,
|
|
1211
|
+
"server/resources#data-types",
|
|
1212
|
+
async () => {
|
|
1213
|
+
const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
|
|
1214
|
+
const issues = [];
|
|
1215
|
+
for (const r of resources) {
|
|
1216
|
+
if (!r.uri) issues.push("Resource missing uri");
|
|
1217
|
+
else {
|
|
1218
|
+
try {
|
|
1219
|
+
new URL(r.uri);
|
|
1220
|
+
} catch {
|
|
1221
|
+
issues.push(`${r.uri}: invalid URI format`);
|
|
1222
|
+
}
|
|
716
1223
|
}
|
|
1224
|
+
if (!r.name) issues.push(`${r.uri || "?"}: missing name`);
|
|
1225
|
+
if (!r.description) warnings.push(`Resource "${r.name || r.uri}" missing description`);
|
|
1226
|
+
if (!r.mimeType) warnings.push(`Resource "${r.name || r.uri}" missing mimeType`);
|
|
717
1227
|
}
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
1228
|
+
return {
|
|
1229
|
+
passed: issues.length === 0,
|
|
1230
|
+
details: issues.length === 0 ? "All resources valid" : issues.join("; ")
|
|
1231
|
+
};
|
|
721
1232
|
}
|
|
722
|
-
|
|
723
|
-
});
|
|
1233
|
+
);
|
|
724
1234
|
if (resourceCount > 0) {
|
|
725
|
-
await test(
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
1235
|
+
await test(
|
|
1236
|
+
"resources-read",
|
|
1237
|
+
"resources/read returns content",
|
|
1238
|
+
"resources",
|
|
1239
|
+
false,
|
|
1240
|
+
"server/resources#reading-resources",
|
|
1241
|
+
async () => {
|
|
1242
|
+
const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
|
|
1243
|
+
const firstUri = resources[0]?.uri;
|
|
1244
|
+
if (!firstUri) return { passed: false, details: "No resource URI to test" };
|
|
1245
|
+
const readRes = await rpc("resources/read", { uri: firstUri });
|
|
1246
|
+
const contents = readRes.body?.result?.contents;
|
|
1247
|
+
if (!Array.isArray(contents)) return { passed: false, details: "No contents array" };
|
|
1248
|
+
const issues = [];
|
|
1249
|
+
for (const c of contents) {
|
|
1250
|
+
if (!c.uri) issues.push("Content item missing uri");
|
|
1251
|
+
if (!c.text && !c.blob) issues.push(`Content item for ${c.uri || "?"} missing both text and blob`);
|
|
1252
|
+
}
|
|
1253
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1254
|
+
return { passed: true, details: `Read ${contents.length} content item(s) from ${firstUri}` };
|
|
1255
|
+
}
|
|
1256
|
+
);
|
|
1257
|
+
}
|
|
1258
|
+
await test(
|
|
1259
|
+
"resources-templates",
|
|
1260
|
+
"resources/templates/list returns valid response",
|
|
1261
|
+
"resources",
|
|
1262
|
+
false,
|
|
1263
|
+
"server/resources#resource-templates",
|
|
1264
|
+
async () => {
|
|
1265
|
+
const res = await rpc("resources/templates/list");
|
|
1266
|
+
const error = res.body?.error;
|
|
1267
|
+
if (error) {
|
|
1268
|
+
if (error.code === -32601) return { passed: true, details: "Method not supported (acceptable)" };
|
|
1269
|
+
return { passed: false, details: `Error: ${error.message}` };
|
|
1270
|
+
}
|
|
1271
|
+
const templates = res.body?.result?.resourceTemplates;
|
|
1272
|
+
if (!Array.isArray(templates)) return { passed: false, details: "No resourceTemplates array" };
|
|
732
1273
|
const issues = [];
|
|
733
|
-
for (const
|
|
734
|
-
if (!
|
|
735
|
-
if (!
|
|
1274
|
+
for (const t of templates) {
|
|
1275
|
+
if (!t.uriTemplate) issues.push("Template missing uriTemplate");
|
|
1276
|
+
if (!t.name) issues.push(`${t.uriTemplate || "?"}: missing name`);
|
|
736
1277
|
}
|
|
737
1278
|
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
738
|
-
return { passed: true, details:
|
|
739
|
-
});
|
|
740
|
-
}
|
|
741
|
-
await test("resources-templates", "resources/templates/list returns valid response", "resources", false, "server/resources#resource-templates", async () => {
|
|
742
|
-
const res = await rpc("resources/templates/list");
|
|
743
|
-
const error = res.body?.error;
|
|
744
|
-
if (error) {
|
|
745
|
-
if (error.code === -32601) return { passed: true, details: "Method not supported (acceptable)" };
|
|
746
|
-
return { passed: false, details: `Error: ${error.message}` };
|
|
747
|
-
}
|
|
748
|
-
const templates = res.body?.result?.resourceTemplates;
|
|
749
|
-
if (!Array.isArray(templates)) return { passed: false, details: "No resourceTemplates array" };
|
|
750
|
-
const issues = [];
|
|
751
|
-
for (const t of templates) {
|
|
752
|
-
if (!t.uriTemplate) issues.push("Template missing uriTemplate");
|
|
753
|
-
if (!t.name) issues.push(`${t.uriTemplate || "?"}: missing name`);
|
|
1279
|
+
return { passed: true, details: `${templates.length} resource template(s)` };
|
|
754
1280
|
}
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
}
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
1281
|
+
);
|
|
1282
|
+
await test(
|
|
1283
|
+
"resources-pagination",
|
|
1284
|
+
"resources/list supports pagination",
|
|
1285
|
+
"resources",
|
|
1286
|
+
false,
|
|
1287
|
+
"server/resources#listing-resources",
|
|
1288
|
+
async () => {
|
|
1289
|
+
const res = await rpc("resources/list");
|
|
1290
|
+
const result = res.body?.result;
|
|
1291
|
+
if (!result) return { passed: false, details: "No result from resources/list" };
|
|
1292
|
+
if (!Array.isArray(result.resources)) return { passed: false, details: "No resources array" };
|
|
1293
|
+
if (result.nextCursor !== void 0) {
|
|
1294
|
+
if (typeof result.nextCursor !== "string") {
|
|
1295
|
+
return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
|
|
1296
|
+
}
|
|
1297
|
+
const nextRes = await rpc("resources/list", { cursor: result.nextCursor });
|
|
1298
|
+
const nextResult = nextRes.body?.result;
|
|
1299
|
+
if (!nextResult || !Array.isArray(nextResult.resources)) {
|
|
1300
|
+
return { passed: false, details: "Next page failed to return resources array" };
|
|
1301
|
+
}
|
|
1302
|
+
return {
|
|
1303
|
+
passed: true,
|
|
1304
|
+
details: `Pagination works: page 1 had ${result.resources.length}, page 2 had ${nextResult.resources.length}`
|
|
1305
|
+
};
|
|
771
1306
|
}
|
|
772
|
-
return { passed: true, details:
|
|
1307
|
+
return { passed: true, details: `${result.resources.length} resource(s), no nextCursor (single page)` };
|
|
773
1308
|
}
|
|
774
|
-
|
|
775
|
-
});
|
|
1309
|
+
);
|
|
776
1310
|
if (hasSubscribe && resourceCount > 0) {
|
|
777
|
-
await test(
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
1311
|
+
await test(
|
|
1312
|
+
"resources-subscribe",
|
|
1313
|
+
"Resource subscribe/unsubscribe",
|
|
1314
|
+
"resources",
|
|
1315
|
+
true,
|
|
1316
|
+
"server/resources#subscriptions",
|
|
1317
|
+
async () => {
|
|
1318
|
+
const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
|
|
1319
|
+
const firstUri = resources[0]?.uri;
|
|
1320
|
+
if (!firstUri) return { passed: false, details: "No resource URI for subscribe test" };
|
|
1321
|
+
const subRes = await rpc("resources/subscribe", { uri: firstUri });
|
|
1322
|
+
if (subRes.body?.error) {
|
|
1323
|
+
return {
|
|
1324
|
+
passed: false,
|
|
1325
|
+
details: `Subscribe error: ${subRes.body.error.code} \u2014 ${subRes.body.error.message}`
|
|
1326
|
+
};
|
|
1327
|
+
}
|
|
1328
|
+
const unsubRes = await rpc("resources/unsubscribe", { uri: firstUri });
|
|
1329
|
+
if (unsubRes.body?.error) {
|
|
1330
|
+
return {
|
|
1331
|
+
passed: false,
|
|
1332
|
+
details: `Unsubscribe error: ${unsubRes.body.error.code} \u2014 ${unsubRes.body.error.message}`
|
|
1333
|
+
};
|
|
1334
|
+
}
|
|
1335
|
+
return { passed: true, details: `Subscribe/unsubscribe for ${firstUri} succeeded` };
|
|
788
1336
|
}
|
|
789
|
-
|
|
790
|
-
});
|
|
1337
|
+
);
|
|
791
1338
|
}
|
|
792
1339
|
}
|
|
793
1340
|
const hasPrompts = !!serverInfo.capabilities.prompts;
|
|
794
1341
|
if (hasPrompts) {
|
|
795
1342
|
let cachedPromptsList = null;
|
|
796
|
-
await test(
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
1343
|
+
await test(
|
|
1344
|
+
"prompts-list",
|
|
1345
|
+
"prompts/list returns valid response",
|
|
1346
|
+
"prompts",
|
|
1347
|
+
true,
|
|
1348
|
+
"server/prompts#listing-prompts",
|
|
1349
|
+
async () => {
|
|
1350
|
+
const res = await rpc("prompts/list");
|
|
1351
|
+
const prompts = res.body?.result?.prompts;
|
|
1352
|
+
if (!Array.isArray(prompts)) return { passed: false, details: "No prompts array" };
|
|
1353
|
+
cachedPromptsList = prompts;
|
|
1354
|
+
promptCount = prompts.length;
|
|
1355
|
+
promptNames = prompts.map((p) => p.name).filter(Boolean);
|
|
1356
|
+
return {
|
|
1357
|
+
passed: true,
|
|
1358
|
+
details: `${promptCount} prompt(s): ${promptNames.slice(0, 5).join(", ")}${promptCount > 5 ? "..." : ""}`
|
|
1359
|
+
};
|
|
1360
|
+
}
|
|
1361
|
+
);
|
|
805
1362
|
await test("prompts-schema", "Prompts have name field", "schema", true, "server/prompts#data-types", async () => {
|
|
806
1363
|
const prompts = cachedPromptsList ?? (await rpc("prompts/list")).body?.result?.prompts ?? [];
|
|
807
1364
|
const issues = [];
|
|
@@ -818,39 +1375,56 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
818
1375
|
return { passed: issues.length === 0, details: issues.length === 0 ? "All prompts valid" : issues.join("; ") };
|
|
819
1376
|
});
|
|
820
1377
|
if (promptNames.length > 0) {
|
|
821
|
-
await test(
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
if (
|
|
1378
|
+
await test(
|
|
1379
|
+
"prompts-get",
|
|
1380
|
+
"prompts/get returns valid messages",
|
|
1381
|
+
"prompts",
|
|
1382
|
+
false,
|
|
1383
|
+
"server/prompts#getting-a-prompt",
|
|
1384
|
+
async () => {
|
|
1385
|
+
const res = await rpc("prompts/get", { name: promptNames[0] });
|
|
1386
|
+
const error = res.body?.error;
|
|
1387
|
+
if (error) return { passed: true, details: `Error (may need arguments): code ${error.code}` };
|
|
1388
|
+
const messages = res.body?.result?.messages;
|
|
1389
|
+
if (!Array.isArray(messages)) return { passed: false, details: "No messages array in result" };
|
|
1390
|
+
const issues = [];
|
|
1391
|
+
for (const msg of messages) {
|
|
1392
|
+
if (!msg.role || !["user", "assistant"].includes(msg.role)) issues.push(`Invalid role: ${msg.role}`);
|
|
1393
|
+
if (!msg.content) issues.push("Message missing content");
|
|
1394
|
+
}
|
|
1395
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1396
|
+
return { passed: true, details: `${messages.length} message(s) from ${promptNames[0]}` };
|
|
831
1397
|
}
|
|
832
|
-
|
|
833
|
-
return { passed: true, details: `${messages.length} message(s) from ${promptNames[0]}` };
|
|
834
|
-
});
|
|
1398
|
+
);
|
|
835
1399
|
}
|
|
836
|
-
await test(
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
if (
|
|
848
|
-
|
|
1400
|
+
await test(
|
|
1401
|
+
"prompts-pagination",
|
|
1402
|
+
"prompts/list supports pagination",
|
|
1403
|
+
"prompts",
|
|
1404
|
+
false,
|
|
1405
|
+
"server/prompts#listing-prompts",
|
|
1406
|
+
async () => {
|
|
1407
|
+
const res = await rpc("prompts/list");
|
|
1408
|
+
const result = res.body?.result;
|
|
1409
|
+
if (!result) return { passed: false, details: "No result from prompts/list" };
|
|
1410
|
+
if (!Array.isArray(result.prompts)) return { passed: false, details: "No prompts array" };
|
|
1411
|
+
if (result.nextCursor !== void 0) {
|
|
1412
|
+
if (typeof result.nextCursor !== "string") {
|
|
1413
|
+
return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
|
|
1414
|
+
}
|
|
1415
|
+
const nextRes = await rpc("prompts/list", { cursor: result.nextCursor });
|
|
1416
|
+
const nextResult = nextRes.body?.result;
|
|
1417
|
+
if (!nextResult || !Array.isArray(nextResult.prompts)) {
|
|
1418
|
+
return { passed: false, details: "Next page failed to return prompts array" };
|
|
1419
|
+
}
|
|
1420
|
+
return {
|
|
1421
|
+
passed: true,
|
|
1422
|
+
details: `Pagination works: page 1 had ${result.prompts.length}, page 2 had ${nextResult.prompts.length}`
|
|
1423
|
+
};
|
|
849
1424
|
}
|
|
850
|
-
return { passed: true, details:
|
|
1425
|
+
return { passed: true, details: `${result.prompts.length} prompt(s), no nextCursor (single page)` };
|
|
851
1426
|
}
|
|
852
|
-
|
|
853
|
-
});
|
|
1427
|
+
);
|
|
854
1428
|
}
|
|
855
1429
|
await test("error-unknown-method", "Returns JSON-RPC error for unknown method", "errors", true, "basic", async () => {
|
|
856
1430
|
const res = await rpc("nonexistent/method");
|
|
@@ -862,16 +1436,23 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
862
1436
|
details: `Error code: ${error.code}${correctCode ? " (correct: Method not found)" : " (expected -32601)"} \u2014 ${error.message}`
|
|
863
1437
|
};
|
|
864
1438
|
});
|
|
865
|
-
await test(
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
1439
|
+
await test(
|
|
1440
|
+
"error-method-code",
|
|
1441
|
+
"Uses correct JSON-RPC error code for unknown method",
|
|
1442
|
+
"errors",
|
|
1443
|
+
false,
|
|
1444
|
+
"basic",
|
|
1445
|
+
async () => {
|
|
1446
|
+
const res = await rpc("nonexistent/method");
|
|
1447
|
+
const error = res.body?.error;
|
|
1448
|
+
if (!error) return { passed: false, details: "No error returned" };
|
|
1449
|
+
return { passed: error.code === -32601, details: `Expected -32601, got ${error.code}` };
|
|
1450
|
+
}
|
|
1451
|
+
);
|
|
871
1452
|
await test("error-invalid-jsonrpc", "Handles malformed JSON-RPC", "errors", true, "basic", async () => {
|
|
872
1453
|
const res = await request(backendUrl, {
|
|
873
1454
|
method: "POST",
|
|
874
|
-
headers: { "Content-Type": "application/json",
|
|
1455
|
+
headers: { "Content-Type": "application/json", Accept: "application/json, text/event-stream", ...buildHeaders() },
|
|
875
1456
|
body: JSON.stringify({ not: "a valid jsonrpc message" }),
|
|
876
1457
|
signal: AbortSignal.timeout(timeout)
|
|
877
1458
|
});
|
|
@@ -880,17 +1461,21 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
880
1461
|
const body = JSON.parse(text);
|
|
881
1462
|
if (body?.error) {
|
|
882
1463
|
const correctCode = body.error.code === -32600;
|
|
883
|
-
return {
|
|
1464
|
+
return {
|
|
1465
|
+
passed: true,
|
|
1466
|
+
details: `Error code: ${body.error.code}${correctCode ? " (correct: Invalid Request)" : ""} \u2014 ${body.error.message}`
|
|
1467
|
+
};
|
|
884
1468
|
}
|
|
885
1469
|
} catch {
|
|
886
1470
|
}
|
|
887
|
-
if (res.statusCode >= 400 && res.statusCode < 500)
|
|
1471
|
+
if (res.statusCode >= 400 && res.statusCode < 500)
|
|
1472
|
+
return { passed: true, details: `HTTP ${res.statusCode} (acceptable)` };
|
|
888
1473
|
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected JSON-RPC error or 4xx status` };
|
|
889
1474
|
});
|
|
890
1475
|
await test("error-invalid-json", "Handles invalid JSON body", "errors", false, "basic", async () => {
|
|
891
1476
|
const res = await request(backendUrl, {
|
|
892
1477
|
method: "POST",
|
|
893
|
-
headers: { "Content-Type": "application/json",
|
|
1478
|
+
headers: { "Content-Type": "application/json", Accept: "application/json, text/event-stream", ...buildHeaders() },
|
|
894
1479
|
body: "{this is not valid json!!!",
|
|
895
1480
|
signal: AbortSignal.timeout(timeout)
|
|
896
1481
|
});
|
|
@@ -900,24 +1485,35 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
900
1485
|
if (body?.error) return { passed: true, details: `Error code: ${body.error.code} \u2014 ${body.error.message}` };
|
|
901
1486
|
} catch {
|
|
902
1487
|
}
|
|
903
|
-
if (res.statusCode >= 400 && res.statusCode < 500)
|
|
1488
|
+
if (res.statusCode >= 400 && res.statusCode < 500)
|
|
1489
|
+
return { passed: true, details: `HTTP ${res.statusCode} (acceptable)` };
|
|
904
1490
|
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected parse error or 4xx status` };
|
|
905
1491
|
});
|
|
906
|
-
await test(
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
1492
|
+
await test(
|
|
1493
|
+
"error-missing-params",
|
|
1494
|
+
"Returns error for tools/call without name",
|
|
1495
|
+
"errors",
|
|
1496
|
+
false,
|
|
1497
|
+
"server/tools#error-handling",
|
|
1498
|
+
async () => {
|
|
1499
|
+
const res = await rpc("tools/call", {});
|
|
1500
|
+
const error = res.body?.error;
|
|
1501
|
+
const isError = res.body?.result?.isError;
|
|
1502
|
+
if (error) {
|
|
1503
|
+
const correctCode = error.code === -32602;
|
|
1504
|
+
return {
|
|
1505
|
+
passed: true,
|
|
1506
|
+
details: `Error code: ${error.code}${correctCode ? " (correct: Invalid params)" : ""} \u2014 ${error.message}`
|
|
1507
|
+
};
|
|
1508
|
+
}
|
|
1509
|
+
if (isError) return { passed: true, details: "Tool execution error (valid)" };
|
|
1510
|
+
return { passed: false, details: "No error for tools/call without name" };
|
|
913
1511
|
}
|
|
914
|
-
|
|
915
|
-
return { passed: false, details: "No error for tools/call without name" };
|
|
916
|
-
});
|
|
1512
|
+
);
|
|
917
1513
|
await test("error-parse-code", "Returns -32700 for invalid JSON", "errors", false, "basic", async () => {
|
|
918
1514
|
const res = await request(backendUrl, {
|
|
919
1515
|
method: "POST",
|
|
920
|
-
headers: { "Content-Type": "application/json",
|
|
1516
|
+
headers: { "Content-Type": "application/json", Accept: "application/json, text/event-stream", ...buildHeaders() },
|
|
921
1517
|
body: "<<<not json>>>",
|
|
922
1518
|
signal: AbortSignal.timeout(timeout)
|
|
923
1519
|
});
|
|
@@ -940,7 +1536,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
940
1536
|
await test("error-invalid-request-code", "Returns -32600 for invalid request", "errors", false, "basic", async () => {
|
|
941
1537
|
const res = await request(backendUrl, {
|
|
942
1538
|
method: "POST",
|
|
943
|
-
headers: { "Content-Type": "application/json",
|
|
1539
|
+
headers: { "Content-Type": "application/json", Accept: "application/json, text/event-stream", ...buildHeaders() },
|
|
944
1540
|
body: JSON.stringify({ jsonrpc: "2.0", id: 99999 }),
|
|
945
1541
|
signal: AbortSignal.timeout(timeout)
|
|
946
1542
|
});
|
|
@@ -985,6 +1581,171 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
985
1581
|
};
|
|
986
1582
|
}
|
|
987
1583
|
|
|
1584
|
+
// src/mcp/tools.ts
|
|
1585
|
+
function registerTools(server) {
|
|
1586
|
+
server.tool(
|
|
1587
|
+
"mcp_compliance_test",
|
|
1588
|
+
"Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all 43 tests covering transport, lifecycle, tools, resources, prompts, errors, and schema validation.",
|
|
1589
|
+
{
|
|
1590
|
+
url: z.string().url().describe("The MCP server URL to test (must be HTTP or HTTPS)"),
|
|
1591
|
+
auth: z.string().optional().describe('Authorization header value (e.g., "Bearer tok123")'),
|
|
1592
|
+
headers: z.record(z.string()).optional().describe('Additional headers to include on all requests (e.g., {"X-Api-Key": "abc"})'),
|
|
1593
|
+
timeout: z.number().optional().describe("Request timeout in milliseconds (default: 15000)"),
|
|
1594
|
+
retries: z.number().optional().describe("Number of retries for failed tests (default: 0)"),
|
|
1595
|
+
only: z.array(z.string()).optional().describe("Only run tests matching these categories or test IDs"),
|
|
1596
|
+
skip: z.array(z.string()).optional().describe("Skip tests matching these categories or test IDs")
|
|
1597
|
+
},
|
|
1598
|
+
{
|
|
1599
|
+
title: "Run MCP Compliance Tests",
|
|
1600
|
+
readOnlyHint: true,
|
|
1601
|
+
destructiveHint: false,
|
|
1602
|
+
idempotentHint: true,
|
|
1603
|
+
openWorldHint: true
|
|
1604
|
+
},
|
|
1605
|
+
async ({ url, auth, headers: extraHeaders, timeout, retries, only, skip }) => {
|
|
1606
|
+
try {
|
|
1607
|
+
const headers = { ...extraHeaders };
|
|
1608
|
+
if (auth) headers.Authorization = auth;
|
|
1609
|
+
const report = await runComplianceSuite(url, {
|
|
1610
|
+
headers: Object.keys(headers).length > 0 ? headers : void 0,
|
|
1611
|
+
timeout,
|
|
1612
|
+
retries,
|
|
1613
|
+
only,
|
|
1614
|
+
skip
|
|
1615
|
+
});
|
|
1616
|
+
const summary = [
|
|
1617
|
+
`Grade: ${report.grade} (${report.score}%)`,
|
|
1618
|
+
`Overall: ${report.overall}`,
|
|
1619
|
+
`Tests: ${report.summary.passed}/${report.summary.total} passed (${report.summary.requiredPassed}/${report.summary.required} required)`,
|
|
1620
|
+
"",
|
|
1621
|
+
...report.tests.map(
|
|
1622
|
+
(t) => `${t.passed ? "PASS" : "FAIL"} ${t.name}${t.required ? " (required)" : ""} \u2014 ${t.details}`
|
|
1623
|
+
)
|
|
1624
|
+
];
|
|
1625
|
+
if (report.serverInfo.name) {
|
|
1626
|
+
summary.unshift(`Server: ${report.serverInfo.name} v${report.serverInfo.version || "?"}`);
|
|
1627
|
+
}
|
|
1628
|
+
if (report.warnings.length > 0) {
|
|
1629
|
+
summary.push("", `Warnings (${report.warnings.length}):`);
|
|
1630
|
+
for (const w of report.warnings) {
|
|
1631
|
+
summary.push(` - ${w}`);
|
|
1632
|
+
}
|
|
1633
|
+
}
|
|
1634
|
+
return {
|
|
1635
|
+
content: [
|
|
1636
|
+
{ type: "text", text: summary.join("\n") },
|
|
1637
|
+
{ type: "text", text: `
|
|
1638
|
+
|
|
1639
|
+
Full report:
|
|
1640
|
+
${JSON.stringify(report, null, 2)}` }
|
|
1641
|
+
]
|
|
1642
|
+
};
|
|
1643
|
+
} catch (err) {
|
|
1644
|
+
return {
|
|
1645
|
+
content: [{ type: "text", text: `Error running compliance test: ${err.message}` }],
|
|
1646
|
+
isError: true
|
|
1647
|
+
};
|
|
1648
|
+
}
|
|
1649
|
+
}
|
|
1650
|
+
);
|
|
1651
|
+
server.tool(
|
|
1652
|
+
"mcp_compliance_badge",
|
|
1653
|
+
"Get the badge markdown embed code for an MCP server. Runs the compliance test suite first to determine the grade.",
|
|
1654
|
+
{
|
|
1655
|
+
url: z.string().url().describe("The MCP server URL to test"),
|
|
1656
|
+
auth: z.string().optional().describe('Authorization header value (e.g., "Bearer tok123")'),
|
|
1657
|
+
headers: z.record(z.string()).optional().describe("Additional headers to include on all requests"),
|
|
1658
|
+
timeout: z.number().optional().describe("Request timeout in milliseconds (default: 15000)")
|
|
1659
|
+
},
|
|
1660
|
+
{
|
|
1661
|
+
title: "Get Compliance Badge",
|
|
1662
|
+
readOnlyHint: true,
|
|
1663
|
+
destructiveHint: false,
|
|
1664
|
+
idempotentHint: true,
|
|
1665
|
+
openWorldHint: true
|
|
1666
|
+
},
|
|
1667
|
+
async ({ url, auth, headers: extraHeaders, timeout }) => {
|
|
1668
|
+
try {
|
|
1669
|
+
const headers = { ...extraHeaders };
|
|
1670
|
+
if (auth) headers.Authorization = auth;
|
|
1671
|
+
const report = await runComplianceSuite(url, {
|
|
1672
|
+
headers: Object.keys(headers).length > 0 ? headers : void 0,
|
|
1673
|
+
timeout
|
|
1674
|
+
});
|
|
1675
|
+
const badge = report.badge;
|
|
1676
|
+
return {
|
|
1677
|
+
content: [
|
|
1678
|
+
{
|
|
1679
|
+
type: "text",
|
|
1680
|
+
text: [
|
|
1681
|
+
`Grade: ${report.grade} (${report.score}%)`,
|
|
1682
|
+
"",
|
|
1683
|
+
"Markdown:",
|
|
1684
|
+
badge.markdown,
|
|
1685
|
+
"",
|
|
1686
|
+
"HTML:",
|
|
1687
|
+
badge.html
|
|
1688
|
+
].join("\n")
|
|
1689
|
+
}
|
|
1690
|
+
]
|
|
1691
|
+
};
|
|
1692
|
+
} catch (err) {
|
|
1693
|
+
return {
|
|
1694
|
+
content: [{ type: "text", text: `Error: ${err.message}` }],
|
|
1695
|
+
isError: true
|
|
1696
|
+
};
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
);
|
|
1700
|
+
server.tool(
|
|
1701
|
+
"mcp_compliance_explain",
|
|
1702
|
+
"Explain what a specific compliance test ID checks and why it matters.",
|
|
1703
|
+
{
|
|
1704
|
+
testId: z.string().describe('The test ID to explain (e.g., "transport-post", "lifecycle-init", "tools-schema")')
|
|
1705
|
+
},
|
|
1706
|
+
{
|
|
1707
|
+
title: "Explain Compliance Test",
|
|
1708
|
+
readOnlyHint: true,
|
|
1709
|
+
destructiveHint: false,
|
|
1710
|
+
idempotentHint: true,
|
|
1711
|
+
openWorldHint: false
|
|
1712
|
+
},
|
|
1713
|
+
async ({ testId }) => {
|
|
1714
|
+
const def = TEST_DEFINITIONS.find((t) => t.id === testId);
|
|
1715
|
+
if (!def) {
|
|
1716
|
+
return {
|
|
1717
|
+
content: [
|
|
1718
|
+
{
|
|
1719
|
+
type: "text",
|
|
1720
|
+
text: `Unknown test ID: "${testId}"
|
|
1721
|
+
|
|
1722
|
+
Valid test IDs:
|
|
1723
|
+
${TEST_DEFINITIONS.map((t) => t.id).join(", ")}`
|
|
1724
|
+
}
|
|
1725
|
+
],
|
|
1726
|
+
isError: true
|
|
1727
|
+
};
|
|
1728
|
+
}
|
|
1729
|
+
return {
|
|
1730
|
+
content: [
|
|
1731
|
+
{
|
|
1732
|
+
type: "text",
|
|
1733
|
+
text: [
|
|
1734
|
+
`Test: ${def.id}`,
|
|
1735
|
+
`Name: ${def.name}`,
|
|
1736
|
+
`Category: ${def.category}`,
|
|
1737
|
+
`Required: ${def.required ? "Yes" : "No"}`,
|
|
1738
|
+
`Spec reference: https://modelcontextprotocol.io/specification/2025-11-25/${def.specRef}`,
|
|
1739
|
+
"",
|
|
1740
|
+
def.description
|
|
1741
|
+
].join("\n")
|
|
1742
|
+
}
|
|
1743
|
+
]
|
|
1744
|
+
};
|
|
1745
|
+
}
|
|
1746
|
+
);
|
|
1747
|
+
}
|
|
1748
|
+
|
|
988
1749
|
// src/reporter.ts
|
|
989
1750
|
import chalk from "chalk";
|
|
990
1751
|
var CATEGORY_LABELS = {
|
|
@@ -1037,11 +1798,19 @@ function formatTerminal(report) {
|
|
|
1037
1798
|
lines.push(chalk.dim(`Spec: ${report.specVersion} | Tool: v${report.toolVersion} | ${report.timestamp}`));
|
|
1038
1799
|
lines.push(chalk.dim(`URL: ${report.url}`));
|
|
1039
1800
|
if (report.serverInfo.name) {
|
|
1040
|
-
lines.push(
|
|
1801
|
+
lines.push(
|
|
1802
|
+
chalk.dim(
|
|
1803
|
+
`Server: ${report.serverInfo.name} v${report.serverInfo.version || "?"} (protocol ${report.serverInfo.protocolVersion || "?"})`
|
|
1804
|
+
)
|
|
1805
|
+
);
|
|
1041
1806
|
}
|
|
1042
1807
|
lines.push("");
|
|
1043
|
-
lines.push(
|
|
1044
|
-
|
|
1808
|
+
lines.push(
|
|
1809
|
+
` Grade: ${gradeColor(report.grade)} Score: ${chalk.bold(String(report.score))}% Overall: ${overallColor(report.overall)}`
|
|
1810
|
+
);
|
|
1811
|
+
lines.push(
|
|
1812
|
+
` Tests: ${chalk.green(String(report.summary.passed))} passed / ${chalk.red(String(report.summary.failed))} failed / ${report.summary.total} total`
|
|
1813
|
+
);
|
|
1045
1814
|
lines.push(` Required: ${report.summary.requiredPassed}/${report.summary.required} passed`);
|
|
1046
1815
|
const grouped = {};
|
|
1047
1816
|
for (const t of report.tests) {
|
|
@@ -1067,13 +1836,25 @@ function formatTerminal(report) {
|
|
|
1067
1836
|
lines.push(chalk.dim(` Capabilities: ${declared.join(", ")}`));
|
|
1068
1837
|
}
|
|
1069
1838
|
if (report.toolCount > 0) {
|
|
1070
|
-
lines.push(
|
|
1839
|
+
lines.push(
|
|
1840
|
+
chalk.dim(
|
|
1841
|
+
` Tools (${report.toolCount}): ${report.toolNames.slice(0, 10).join(", ")}${report.toolCount > 10 ? "..." : ""}`
|
|
1842
|
+
)
|
|
1843
|
+
);
|
|
1071
1844
|
}
|
|
1072
1845
|
if (report.resourceCount > 0) {
|
|
1073
|
-
lines.push(
|
|
1846
|
+
lines.push(
|
|
1847
|
+
chalk.dim(
|
|
1848
|
+
` Resources (${report.resourceCount}): ${report.resourceNames.slice(0, 10).join(", ")}${report.resourceCount > 10 ? "..." : ""}`
|
|
1849
|
+
)
|
|
1850
|
+
);
|
|
1074
1851
|
}
|
|
1075
1852
|
if (report.promptCount > 0) {
|
|
1076
|
-
lines.push(
|
|
1853
|
+
lines.push(
|
|
1854
|
+
chalk.dim(
|
|
1855
|
+
` Prompts (${report.promptCount}): ${report.promptNames.slice(0, 10).join(", ")}${report.promptCount > 10 ? "..." : ""}`
|
|
1856
|
+
)
|
|
1857
|
+
);
|
|
1077
1858
|
}
|
|
1078
1859
|
if (report.warnings.length > 0) {
|
|
1079
1860
|
lines.push("");
|
|
@@ -1093,9 +1874,6 @@ function formatJson(report) {
|
|
|
1093
1874
|
}
|
|
1094
1875
|
|
|
1095
1876
|
// src/index.ts
|
|
1096
|
-
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
1097
|
-
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
1098
|
-
import { z } from "zod";
|
|
1099
1877
|
var require2 = createRequire2(import.meta.url);
|
|
1100
1878
|
var { version } = require2("../package.json");
|
|
1101
1879
|
function parseHeaderArg(value, prev) {
|
|
@@ -1113,58 +1891,60 @@ function parseList(value) {
|
|
|
1113
1891
|
}
|
|
1114
1892
|
var program = new Command();
|
|
1115
1893
|
program.name("mcp-compliance").description("Test MCP servers for spec compliance").version(version);
|
|
1116
|
-
program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal or json", "terminal").option("--strict", "Exit with code 1 on any required test failure (for CI)").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").option("--retries <n>", "Number of retries for failed tests", "0").option("--only <items>", "Only run tests matching these categories or test IDs (comma-separated)", parseList).option("--skip <items>", "Skip tests matching these categories or test IDs (comma-separated)", parseList).option("--verbose", "Print each test result as it runs").action(
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1894
|
+
program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal or json", "terminal").option("--strict", "Exit with code 1 on any required test failure (for CI)").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").option("--retries <n>", "Number of retries for failed tests", "0").option("--only <items>", "Only run tests matching these categories or test IDs (comma-separated)", parseList).option("--skip <items>", "Skip tests matching these categories or test IDs (comma-separated)", parseList).option("--verbose", "Print each test result as it runs").action(
|
|
1895
|
+
async (url, opts) => {
|
|
1896
|
+
try {
|
|
1897
|
+
const headers = { ...opts.header };
|
|
1898
|
+
if (opts.auth) headers.Authorization = opts.auth;
|
|
1899
|
+
if (opts.format === "terminal") {
|
|
1900
|
+
console.log(chalk2.dim(`
|
|
1122
1901
|
Testing ${url}...
|
|
1123
1902
|
`));
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1903
|
+
}
|
|
1904
|
+
const report = await runComplianceSuite(url, {
|
|
1905
|
+
headers,
|
|
1906
|
+
timeout: Number.parseInt(opts.timeout, 10) || 15e3,
|
|
1907
|
+
retries: Number.parseInt(opts.retries, 10) || 0,
|
|
1908
|
+
only: opts.only,
|
|
1909
|
+
skip: opts.skip,
|
|
1910
|
+
onProgress: opts.verbose ? (testId, passed, details) => {
|
|
1911
|
+
const icon = passed ? chalk2.green("PASS") : chalk2.red("FAIL");
|
|
1912
|
+
console.log(` ${icon} ${testId} \u2014 ${details}`);
|
|
1913
|
+
} : void 0
|
|
1914
|
+
});
|
|
1915
|
+
if (opts.verbose && opts.format === "terminal") {
|
|
1916
|
+
console.log("");
|
|
1917
|
+
}
|
|
1918
|
+
if (opts.format === "json") {
|
|
1919
|
+
console.log(formatJson(report));
|
|
1920
|
+
} else {
|
|
1921
|
+
console.log(formatTerminal(report));
|
|
1922
|
+
}
|
|
1923
|
+
if (opts.strict && report.overall === "fail") {
|
|
1924
|
+
process.exit(1);
|
|
1925
|
+
}
|
|
1926
|
+
} catch (err) {
|
|
1927
|
+
if (opts.format === "json") {
|
|
1928
|
+
console.error(JSON.stringify({ error: err.message }));
|
|
1929
|
+
} else {
|
|
1930
|
+
console.error(chalk2.red(`
|
|
1152
1931
|
Error: ${err.message}
|
|
1153
1932
|
`));
|
|
1933
|
+
}
|
|
1934
|
+
process.exit(1);
|
|
1154
1935
|
}
|
|
1155
|
-
process.exit(1);
|
|
1156
1936
|
}
|
|
1157
|
-
|
|
1937
|
+
);
|
|
1158
1938
|
program.command("badge").description("Run tests and output just the badge markdown embed code").argument("<url>", "MCP server URL to test").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").action(async (url, opts) => {
|
|
1159
1939
|
try {
|
|
1160
1940
|
const headers = { ...opts.header };
|
|
1161
|
-
if (opts.auth) headers
|
|
1941
|
+
if (opts.auth) headers.Authorization = opts.auth;
|
|
1162
1942
|
console.log(chalk2.dim(`
|
|
1163
1943
|
Testing ${url}...
|
|
1164
1944
|
`));
|
|
1165
1945
|
const report = await runComplianceSuite(url, {
|
|
1166
1946
|
headers,
|
|
1167
|
-
timeout: parseInt(opts.timeout, 10) || 15e3
|
|
1947
|
+
timeout: Number.parseInt(opts.timeout, 10) || 15e3
|
|
1168
1948
|
});
|
|
1169
1949
|
console.log(`Grade: ${report.grade} (${report.score}%)
|
|
1170
1950
|
`);
|
|
@@ -1179,85 +1959,7 @@ Error: ${err.message}
|
|
|
1179
1959
|
});
|
|
1180
1960
|
program.command("mcp").description("Start the MCP compliance server (stdio transport)").action(async () => {
|
|
1181
1961
|
const server = new McpServer({ name: "mcp-compliance", version });
|
|
1182
|
-
server
|
|
1183
|
-
"mcp_compliance_test",
|
|
1184
|
-
"Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all 43 tests covering transport, lifecycle, tools, resources, prompts, errors, and schema validation.",
|
|
1185
|
-
{ url: z.string().url().describe("The MCP server URL to test (must be HTTP or HTTPS)") },
|
|
1186
|
-
async ({ url }) => {
|
|
1187
|
-
try {
|
|
1188
|
-
const report = await runComplianceSuite(url);
|
|
1189
|
-
const summary = [
|
|
1190
|
-
`Grade: ${report.grade} (${report.score}%)`,
|
|
1191
|
-
`Overall: ${report.overall}`,
|
|
1192
|
-
`Tests: ${report.summary.passed}/${report.summary.total} passed (${report.summary.requiredPassed}/${report.summary.required} required)`,
|
|
1193
|
-
"",
|
|
1194
|
-
...report.tests.map(
|
|
1195
|
-
(t) => `${t.passed ? "PASS" : "FAIL"} ${t.name}${t.required ? " (required)" : ""} \u2014 ${t.details}`
|
|
1196
|
-
)
|
|
1197
|
-
];
|
|
1198
|
-
if (report.serverInfo.name) {
|
|
1199
|
-
summary.unshift(`Server: ${report.serverInfo.name} v${report.serverInfo.version || "?"}`);
|
|
1200
|
-
}
|
|
1201
|
-
if (report.warnings.length > 0) {
|
|
1202
|
-
summary.push("", `Warnings (${report.warnings.length}):`);
|
|
1203
|
-
for (const w of report.warnings) summary.push(` - ${w}`);
|
|
1204
|
-
}
|
|
1205
|
-
return {
|
|
1206
|
-
content: [
|
|
1207
|
-
{ type: "text", text: summary.join("\n") },
|
|
1208
|
-
{ type: "text", text: `
|
|
1209
|
-
|
|
1210
|
-
Full report:
|
|
1211
|
-
${JSON.stringify(report, null, 2)}` }
|
|
1212
|
-
]
|
|
1213
|
-
};
|
|
1214
|
-
} catch (err) {
|
|
1215
|
-
return { content: [{ type: "text", text: `Error running compliance test: ${err.message}` }], isError: true };
|
|
1216
|
-
}
|
|
1217
|
-
}
|
|
1218
|
-
);
|
|
1219
|
-
server.tool(
|
|
1220
|
-
"mcp_compliance_badge",
|
|
1221
|
-
"Get the badge markdown embed code for an MCP server. Runs the compliance test suite first to determine the grade.",
|
|
1222
|
-
{ url: z.string().url().describe("The MCP server URL to test") },
|
|
1223
|
-
async ({ url }) => {
|
|
1224
|
-
try {
|
|
1225
|
-
const report = await runComplianceSuite(url);
|
|
1226
|
-
const badge = report.badge;
|
|
1227
|
-
return {
|
|
1228
|
-
content: [{
|
|
1229
|
-
type: "text",
|
|
1230
|
-
text: [`Grade: ${report.grade} (${report.score}%)`, "", "Markdown:", badge.markdown, "", "HTML:", badge.html].join("\n")
|
|
1231
|
-
}]
|
|
1232
|
-
};
|
|
1233
|
-
} catch (err) {
|
|
1234
|
-
return { content: [{ type: "text", text: `Error: ${err.message}` }], isError: true };
|
|
1235
|
-
}
|
|
1236
|
-
}
|
|
1237
|
-
);
|
|
1238
|
-
server.tool(
|
|
1239
|
-
"mcp_compliance_explain",
|
|
1240
|
-
"Explain what a specific compliance test ID checks and why it matters.",
|
|
1241
|
-
{ testId: z.string().describe('The test ID to explain (e.g., "transport-post", "lifecycle-init", "tools-schema")') },
|
|
1242
|
-
async ({ testId }) => {
|
|
1243
|
-
const def = TEST_DEFINITIONS.find((t) => t.id === testId);
|
|
1244
|
-
if (!def) {
|
|
1245
|
-
return {
|
|
1246
|
-
content: [{ type: "text", text: `Unknown test ID: "${testId}"
|
|
1247
|
-
|
|
1248
|
-
Valid test IDs:
|
|
1249
|
-
${TEST_DEFINITIONS.map((t) => t.id).join(", ")}` }],
|
|
1250
|
-
isError: true
|
|
1251
|
-
};
|
|
1252
|
-
}
|
|
1253
|
-
return {
|
|
1254
|
-
content: [{
|
|
1255
|
-
type: "text",
|
|
1256
|
-
text: [`Test: ${def.id}`, `Name: ${def.name}`, `Category: ${def.category}`, `Required: ${def.required ? "Yes" : "No"}`, `Spec reference: https://modelcontextprotocol.io/specification/2025-11-25/${def.specRef}`, "", def.description].join("\n")
|
|
1257
|
-
}]
|
|
1258
|
-
};
|
|
1259
|
-
}
|
|
1260
|
-
);
|
|
1962
|
+
registerTools(server);
|
|
1261
1963
|
const transport = new StdioServerTransport();
|
|
1262
1964
|
await server.connect(transport);
|
|
1263
1965
|
});
|