@yawlabs/mcp-compliance 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -9
- package/dist/{chunk-Z7VLPYIO.js → chunk-SELO4TOW.js} +1376 -27
- package/dist/index.js +1387 -35
- package/dist/mcp/server.js +5 -5
- package/dist/runner.d.ts +2 -2
- package/dist/runner.js +1 -1
- package/package.json +5 -1
package/dist/index.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// src/index.ts
|
|
4
4
|
import { createRequire as createRequire3 } from "module";
|
|
5
5
|
import chalk2 from "chalk";
|
|
6
|
-
import { Command } from "commander";
|
|
6
|
+
import { Command, Option } from "commander";
|
|
7
7
|
|
|
8
8
|
// src/mcp/server.ts
|
|
9
9
|
import { createRequire as createRequire2 } from "module";
|
|
@@ -71,7 +71,7 @@ function computeScore(tests) {
|
|
|
71
71
|
|
|
72
72
|
// src/types.ts
|
|
73
73
|
var TEST_DEFINITIONS = [
|
|
74
|
-
// ── Transport (
|
|
74
|
+
// ── Transport (13 tests) ─────────────────────────────────────────
|
|
75
75
|
{
|
|
76
76
|
id: "transport-post",
|
|
77
77
|
name: "HTTP POST accepted",
|
|
@@ -96,8 +96,8 @@ var TEST_DEFINITIONS = [
|
|
|
96
96
|
category: "transport",
|
|
97
97
|
required: false,
|
|
98
98
|
specRef: "basic/transports#streamable-http",
|
|
99
|
-
description: "Verifies that sending a JSON-RPC notification (no id field) returns HTTP 202 Accepted with no body. Per spec, servers MUST return 202
|
|
100
|
-
recommendation: "Detect JSON-RPC messages without an id field and return HTTP 202 with an empty body. Do not
|
|
99
|
+
description: "Verifies that sending a JSON-RPC notification (no id field) returns exactly HTTP 202 Accepted with no body. Per spec, servers MUST return 202 \u2014 not 200 or 204.",
|
|
100
|
+
recommendation: "Detect JSON-RPC messages without an id field and return HTTP 202 with an empty body. Do not return 200 or 204 \u2014 the spec requires exactly 202 Accepted."
|
|
101
101
|
},
|
|
102
102
|
{
|
|
103
103
|
id: "transport-session-id",
|
|
@@ -108,6 +108,24 @@ var TEST_DEFINITIONS = [
|
|
|
108
108
|
description: "Tests that the server returns HTTP 400 when MCP-Session-Id header is missing on requests after initialization (when the server issued a session ID).",
|
|
109
109
|
recommendation: "If your server issues an MCP-Session-Id header in the initialize response, reject subsequent requests that omit this header with HTTP 400."
|
|
110
110
|
},
|
|
111
|
+
{
|
|
112
|
+
id: "transport-session-invalid",
|
|
113
|
+
name: "Returns 404 for unknown session ID",
|
|
114
|
+
category: "transport",
|
|
115
|
+
required: false,
|
|
116
|
+
specRef: "basic/transports#streamable-http",
|
|
117
|
+
description: "Sends a request with a fabricated MCP-Session-Id and verifies the server returns HTTP 404. Per spec, servers managing sessions MUST return 404 for unrecognized session IDs.",
|
|
118
|
+
recommendation: "Return HTTP 404 (Not Found) for requests with an MCP-Session-Id that does not match any active session. Do not return 400 \u2014 that is for missing session IDs."
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
id: "transport-content-type-reject",
|
|
122
|
+
name: "Rejects non-JSON request Content-Type",
|
|
123
|
+
category: "transport",
|
|
124
|
+
required: false,
|
|
125
|
+
specRef: "basic/transports#streamable-http",
|
|
126
|
+
description: "Sends a POST with Content-Type: text/plain instead of application/json and verifies the server rejects it with a 4xx status.",
|
|
127
|
+
recommendation: "Validate the Content-Type header on incoming POST requests. Reject requests that are not application/json with HTTP 415 (Unsupported Media Type) or 400."
|
|
128
|
+
},
|
|
111
129
|
{
|
|
112
130
|
id: "transport-get",
|
|
113
131
|
name: "GET returns SSE stream or 405",
|
|
@@ -162,7 +180,16 @@ var TEST_DEFINITIONS = [
|
|
|
162
180
|
description: "Sends multiple JSON-RPC requests in parallel and verifies the server responds to all with correct matching IDs. Tests that the server can handle concurrent connections.",
|
|
163
181
|
recommendation: "Ensure your server can handle multiple simultaneous requests. Each response must include the correct id matching the request. Use async handlers or connection pooling."
|
|
164
182
|
},
|
|
165
|
-
|
|
183
|
+
{
|
|
184
|
+
id: "transport-sse-event-field",
|
|
185
|
+
name: "SSE responses include event: message",
|
|
186
|
+
category: "transport",
|
|
187
|
+
required: false,
|
|
188
|
+
specRef: "basic/transports#streamable-http",
|
|
189
|
+
description: "Sends a request with Accept: text/event-stream and checks that SSE responses include the event: message field. Per spec, servers MUST set event: message for JSON-RPC messages in SSE streams.",
|
|
190
|
+
recommendation: 'Include "event: message" before each "data:" line in your SSE responses. This is required by the MCP spec for JSON-RPC messages sent over SSE.'
|
|
191
|
+
},
|
|
192
|
+
// ── Lifecycle (15 tests) ─────────────────────────────────────────
|
|
166
193
|
{
|
|
167
194
|
id: "lifecycle-init",
|
|
168
195
|
name: "Initialize handshake",
|
|
@@ -235,6 +262,33 @@ var TEST_DEFINITIONS = [
|
|
|
235
262
|
description: "Verifies that the JSON-RPC response id matches the request id sent by the client. This is a fundamental JSON-RPC 2.0 requirement.",
|
|
236
263
|
recommendation: "Copy the id field from the request into the response. This is a core JSON-RPC 2.0 requirement. Check that your framework does not modify or discard the request ID."
|
|
237
264
|
},
|
|
265
|
+
{
|
|
266
|
+
id: "lifecycle-string-id",
|
|
267
|
+
name: "Supports string request IDs",
|
|
268
|
+
category: "lifecycle",
|
|
269
|
+
required: false,
|
|
270
|
+
specRef: "basic",
|
|
271
|
+
description: "Sends a request with a string id instead of a number. JSON-RPC 2.0 allows both string and number IDs. The server must echo back the exact string id in the response.",
|
|
272
|
+
recommendation: "Ensure your JSON-RPC implementation supports both string and number request IDs. Echo the id back exactly as received, preserving its type."
|
|
273
|
+
},
|
|
274
|
+
{
|
|
275
|
+
id: "lifecycle-version-negotiate",
|
|
276
|
+
name: "Handles unknown protocol version",
|
|
277
|
+
category: "lifecycle",
|
|
278
|
+
required: false,
|
|
279
|
+
specRef: "basic/lifecycle#version-negotiation",
|
|
280
|
+
description: 'Sends an initialize request with a future protocol version ("2099-01-01") and verifies the server either negotiates down to a version it supports or returns an error.',
|
|
281
|
+
recommendation: "When the client requests an unsupported protocol version, respond with the closest version your server supports. Do not blindly accept unknown versions."
|
|
282
|
+
},
|
|
283
|
+
{
|
|
284
|
+
id: "lifecycle-reinit-reject",
|
|
285
|
+
name: "Rejects second initialize request",
|
|
286
|
+
category: "lifecycle",
|
|
287
|
+
required: false,
|
|
288
|
+
specRef: "basic/lifecycle#initialization",
|
|
289
|
+
description: "Sends a second initialize request within the same session. Per spec, the client MUST NOT send initialize more than once. The server should reject it.",
|
|
290
|
+
recommendation: "Track initialization state per session. Reject duplicate initialize requests with a JSON-RPC error or HTTP 4xx. Do not reset session state on re-initialization."
|
|
291
|
+
},
|
|
238
292
|
{
|
|
239
293
|
id: "lifecycle-logging",
|
|
240
294
|
name: "logging/setLevel accepted",
|
|
@@ -264,12 +318,12 @@ var TEST_DEFINITIONS = [
|
|
|
264
318
|
},
|
|
265
319
|
{
|
|
266
320
|
id: "lifecycle-progress",
|
|
267
|
-
name: "
|
|
321
|
+
name: "Handles progress notifications gracefully",
|
|
268
322
|
category: "lifecycle",
|
|
269
323
|
required: false,
|
|
270
324
|
specRef: "basic/utilities#progress",
|
|
271
|
-
description: "
|
|
272
|
-
recommendation: "Accept notifications
|
|
325
|
+
description: "Sends a notifications/progress to the server and verifies it does not error. Note: per spec, progress flows from server to client during long-running requests. This test validates the server handles unexpected notifications gracefully.",
|
|
326
|
+
recommendation: "Accept unknown notifications without returning an error. The server should not crash or return a non-2xx status for notifications it does not recognize."
|
|
273
327
|
},
|
|
274
328
|
// ── Tools (4 tests) ──────────────────────────────────────────────
|
|
275
329
|
{
|
|
@@ -278,7 +332,7 @@ var TEST_DEFINITIONS = [
|
|
|
278
332
|
category: "tools",
|
|
279
333
|
required: false,
|
|
280
334
|
specRef: "server/tools#listing-tools",
|
|
281
|
-
description: "Calls tools/list and validates it returns an array of tool definitions.
|
|
335
|
+
description: "Calls tools/list and validates it returns an array of tool definitions. Dynamically required at runtime if the server declares tools capability.",
|
|
282
336
|
recommendation: "Implement the tools/list handler to return { tools: [...] } with an array of tool definition objects. Each tool needs at least a name and inputSchema."
|
|
283
337
|
},
|
|
284
338
|
{
|
|
@@ -315,7 +369,7 @@ var TEST_DEFINITIONS = [
|
|
|
315
369
|
category: "resources",
|
|
316
370
|
required: false,
|
|
317
371
|
specRef: "server/resources#listing-resources",
|
|
318
|
-
description: "Calls resources/list and validates it returns an array.
|
|
372
|
+
description: "Calls resources/list and validates it returns an array. Dynamically required at runtime if the server declares resources capability.",
|
|
319
373
|
recommendation: "Implement resources/list to return { resources: [...] } with an array of resource objects. Each resource needs at least a uri and name."
|
|
320
374
|
},
|
|
321
375
|
{
|
|
@@ -361,7 +415,7 @@ var TEST_DEFINITIONS = [
|
|
|
361
415
|
category: "prompts",
|
|
362
416
|
required: false,
|
|
363
417
|
specRef: "server/prompts#listing-prompts",
|
|
364
|
-
description: "Calls prompts/list and validates it returns an array.
|
|
418
|
+
description: "Calls prompts/list and validates it returns an array. Dynamically required at runtime if the server declares prompts capability.",
|
|
365
419
|
recommendation: "Implement prompts/list to return { prompts: [...] } with an array of prompt objects. Each prompt needs at least a name field."
|
|
366
420
|
},
|
|
367
421
|
{
|
|
@@ -382,7 +436,7 @@ var TEST_DEFINITIONS = [
|
|
|
382
436
|
description: "Tests cursor-based pagination on prompts/list. Validates nextCursor is a string if present and that fetching the next page works.",
|
|
383
437
|
recommendation: "If you return nextCursor in prompts/list, ensure it is a string and that passing it back as cursor in the next request returns valid results."
|
|
384
438
|
},
|
|
385
|
-
// ── Error Handling (
|
|
439
|
+
// ── Error Handling (10 tests) ────────────────────────────────────
|
|
386
440
|
{
|
|
387
441
|
id: "error-unknown-method",
|
|
388
442
|
name: "Returns JSON-RPC error for unknown method",
|
|
@@ -455,6 +509,24 @@ var TEST_DEFINITIONS = [
|
|
|
455
509
|
description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response.",
|
|
456
510
|
recommendation: "Return a JSON-RPC error or set isError: true when tools/call receives an unrecognized tool name. Do not return an empty success response."
|
|
457
511
|
},
|
|
512
|
+
{
|
|
513
|
+
id: "error-capability-gated",
|
|
514
|
+
name: "Rejects methods for undeclared capabilities",
|
|
515
|
+
category: "errors",
|
|
516
|
+
required: false,
|
|
517
|
+
specRef: "basic/lifecycle#capability-negotiation",
|
|
518
|
+
description: "Calls list methods (tools/list, resources/list, prompts/list) for capabilities the server did NOT declare, and verifies the server returns an error instead of success.",
|
|
519
|
+
recommendation: "Return a JSON-RPC error (e.g., -32601 Method not found) for methods associated with capabilities not declared in your initialize response."
|
|
520
|
+
},
|
|
521
|
+
{
|
|
522
|
+
id: "error-invalid-cursor",
|
|
523
|
+
name: "Handles invalid pagination cursor gracefully",
|
|
524
|
+
category: "errors",
|
|
525
|
+
required: false,
|
|
526
|
+
specRef: "basic",
|
|
527
|
+
description: "Sends a garbage pagination cursor to a list method and verifies the server handles it gracefully \u2014 either returning an error or ignoring the invalid cursor.",
|
|
528
|
+
recommendation: "Validate pagination cursors before use. Return a JSON-RPC error for unrecognized cursors, or treat invalid cursors as a request for the first page."
|
|
529
|
+
},
|
|
458
530
|
// ── Schema Validation (6 tests) ──────────────────────────────────
|
|
459
531
|
{
|
|
460
532
|
id: "tools-schema",
|
|
@@ -471,8 +543,8 @@ var TEST_DEFINITIONS = [
|
|
|
471
543
|
category: "schema",
|
|
472
544
|
required: false,
|
|
473
545
|
specRef: "server/tools#annotations",
|
|
474
|
-
description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans
|
|
475
|
-
recommendation: "If you include annotations on tools, ensure readOnlyHint, destructiveHint, idempotentHint, and openWorldHint are booleans.
|
|
546
|
+
description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans.",
|
|
547
|
+
recommendation: "If you include annotations on tools, ensure readOnlyHint, destructiveHint, idempotentHint, and openWorldHint are booleans. Note: title belongs on the Tool object, not inside annotations."
|
|
476
548
|
},
|
|
477
549
|
{
|
|
478
550
|
id: "tools-title-field",
|
|
@@ -509,22 +581,275 @@ var TEST_DEFINITIONS = [
|
|
|
509
581
|
specRef: "server/resources#data-types",
|
|
510
582
|
description: "Validates every resource has a valid URI (parseable as a URL) and a name field.",
|
|
511
583
|
recommendation: "Ensure every resource has a valid, parseable URI and a name field. Add description and mimeType for better client integration."
|
|
584
|
+
},
|
|
585
|
+
// ── Security: Auth & Transport (9 tests) ─────────────────────────
|
|
586
|
+
{
|
|
587
|
+
id: "security-auth-required",
|
|
588
|
+
name: "Rejects unauthenticated requests",
|
|
589
|
+
category: "security",
|
|
590
|
+
required: false,
|
|
591
|
+
specRef: "basic/authorization",
|
|
592
|
+
description: "Sends a request without an Authorization header and verifies the server returns HTTP 401. Servers exposed over the network should require authentication.",
|
|
593
|
+
recommendation: "Implement authentication on your MCP endpoint. Return HTTP 401 Unauthorized for requests without valid credentials. Use OAuth 2.1 or Bearer tokens as recommended by the MCP spec."
|
|
594
|
+
},
|
|
595
|
+
{
|
|
596
|
+
id: "security-auth-malformed",
|
|
597
|
+
name: "Rejects malformed auth credentials",
|
|
598
|
+
category: "security",
|
|
599
|
+
required: false,
|
|
600
|
+
specRef: "basic/authorization",
|
|
601
|
+
description: "Sends a request with a malformed Authorization header (garbage value) and verifies the server returns HTTP 401 or 403. Servers must validate auth tokens, not just check for presence.",
|
|
602
|
+
recommendation: "Validate the format and signature of Authorization header values. Reject malformed or invalid tokens with HTTP 401. Do not treat any non-empty Authorization header as valid."
|
|
603
|
+
},
|
|
604
|
+
{
|
|
605
|
+
id: "security-tls-required",
|
|
606
|
+
name: "Enforces HTTPS/TLS",
|
|
607
|
+
category: "security",
|
|
608
|
+
required: false,
|
|
609
|
+
specRef: "basic/authorization",
|
|
610
|
+
description: "If the server URL uses HTTPS, attempts an HTTP (plaintext) connection and verifies it is rejected or redirected. Production MCP servers should not accept plaintext connections.",
|
|
611
|
+
recommendation: "Configure your server to reject HTTP connections or redirect to HTTPS. Use TLS 1.2 or higher. The MCP spec requires HTTPS for production deployments."
|
|
612
|
+
},
|
|
613
|
+
{
|
|
614
|
+
id: "security-session-entropy",
|
|
615
|
+
name: "Session IDs are high-entropy",
|
|
616
|
+
category: "security",
|
|
617
|
+
required: false,
|
|
618
|
+
specRef: "basic/transports#streamable-http",
|
|
619
|
+
description: "Analyzes the MCP-Session-Id returned by the server. Session IDs should be cryptographically random and not sequential or predictable.",
|
|
620
|
+
recommendation: "Generate session IDs using a cryptographically secure random source (e.g., crypto.randomUUID()). Session IDs should be at least 128 bits of entropy. Do not use sequential counters or timestamps."
|
|
621
|
+
},
|
|
622
|
+
{
|
|
623
|
+
id: "security-session-not-auth",
|
|
624
|
+
name: "Session ID does not bypass auth",
|
|
625
|
+
category: "security",
|
|
626
|
+
required: false,
|
|
627
|
+
specRef: "basic/transports#streamable-http",
|
|
628
|
+
description: "Verifies that presenting a valid MCP-Session-Id without an Authorization header is still rejected. Per spec, servers MUST NOT use sessions for authentication.",
|
|
629
|
+
recommendation: "Always validate the Authorization header independently of the MCP-Session-Id. Sessions are for request routing, not authentication. Reject requests that lack valid auth even if they have a valid session ID."
|
|
630
|
+
},
|
|
631
|
+
{
|
|
632
|
+
id: "security-oauth-metadata",
|
|
633
|
+
name: "Protected Resource Metadata endpoint exists",
|
|
634
|
+
category: "security",
|
|
635
|
+
required: false,
|
|
636
|
+
specRef: "basic/authorization",
|
|
637
|
+
description: "Checks for a Protected Resource Metadata (RFC 9728) endpoint at /.well-known/oauth-protected-resource. Per MCP 2025-11-25, the MCP server publishes PRM with a resource identifier and authorization_servers array. Falls back to legacy /.well-known/oauth-authorization-server with a warning.",
|
|
638
|
+
recommendation: "Publish a Protected Resource Metadata document at /.well-known/oauth-protected-resource on your server's origin. Include 'resource' (your server's URL) and 'authorization_servers' (array of OAuth AS URLs). See RFC 9728."
|
|
639
|
+
},
|
|
640
|
+
{
|
|
641
|
+
id: "security-token-in-uri",
|
|
642
|
+
name: "Rejects auth tokens in query string",
|
|
643
|
+
category: "security",
|
|
644
|
+
required: false,
|
|
645
|
+
specRef: "basic/authorization",
|
|
646
|
+
description: "Sends a request with the auth token in the URL query string instead of the Authorization header. The MCP spec forbids transmitting credentials in URIs.",
|
|
647
|
+
recommendation: "Never accept authentication tokens from URL query parameters. Tokens in URIs are logged by proxies, appear in browser history, and leak via the Referer header. Only accept tokens in the Authorization header."
|
|
648
|
+
},
|
|
649
|
+
{
|
|
650
|
+
id: "security-cors-headers",
|
|
651
|
+
name: "CORS headers are restrictive",
|
|
652
|
+
category: "security",
|
|
653
|
+
required: false,
|
|
654
|
+
specRef: "basic/transports#streamable-http",
|
|
655
|
+
description: "If the server returns CORS headers, verifies that Access-Control-Allow-Origin is not set to wildcard (*). Wildcard CORS on an authenticated API allows cross-origin credential theft.",
|
|
656
|
+
recommendation: 'Set Access-Control-Allow-Origin to specific trusted origins, not "*". If CORS is not needed (server-to-server only), do not send CORS headers at all.'
|
|
657
|
+
},
|
|
658
|
+
{
|
|
659
|
+
id: "security-origin-validation",
|
|
660
|
+
name: "Validates Origin header on requests",
|
|
661
|
+
category: "security",
|
|
662
|
+
required: false,
|
|
663
|
+
specRef: "basic/transports#streamable-http",
|
|
664
|
+
description: "Sends a request with a suspicious Origin header (https://evil-rebinding-attack.example.com) and verifies the server rejects it. Per spec, servers MUST validate the Origin header to prevent DNS rebinding attacks.",
|
|
665
|
+
recommendation: "Validate the Origin header on all incoming requests. Reject requests from untrusted origins with HTTP 403. Maintain an allowlist of permitted origins."
|
|
666
|
+
},
|
|
667
|
+
// ── Security: Input Validation (6 tests) ─────────────────────────
|
|
668
|
+
{
|
|
669
|
+
id: "security-command-injection",
|
|
670
|
+
name: "Resists command injection in tool params",
|
|
671
|
+
category: "security",
|
|
672
|
+
required: false,
|
|
673
|
+
specRef: "server/tools#calling-tools",
|
|
674
|
+
description: "Calls each tool with OS command injection payloads in string parameters (e.g., '; cat /etc/passwd', '$(whoami)'). Verifies the server does not execute injected commands.",
|
|
675
|
+
recommendation: "Never pass tool argument values directly to shell commands. Use parameterized APIs, execFile() instead of exec(), or allowlists. Sanitize all user-provided input before use in system calls."
|
|
676
|
+
},
|
|
677
|
+
{
|
|
678
|
+
id: "security-sql-injection",
|
|
679
|
+
name: "Resists SQL injection in tool params",
|
|
680
|
+
category: "security",
|
|
681
|
+
required: false,
|
|
682
|
+
specRef: "server/tools#calling-tools",
|
|
683
|
+
description: `Calls each tool with SQL injection payloads in string parameters (e.g., "' OR 1=1 --"). Verifies the server does not return database errors or unexpected data.`,
|
|
684
|
+
recommendation: "Use parameterized queries or prepared statements for all database operations. Never concatenate user input into SQL strings. Return generic error messages that do not reveal database structure."
|
|
685
|
+
},
|
|
686
|
+
{
|
|
687
|
+
id: "security-path-traversal",
|
|
688
|
+
name: "Resists path traversal in tool params",
|
|
689
|
+
category: "security",
|
|
690
|
+
required: false,
|
|
691
|
+
specRef: "server/tools#calling-tools",
|
|
692
|
+
description: "Calls each tool with path traversal payloads in string parameters (e.g., '../../etc/passwd', '..\\\\..\\\\windows\\\\system.ini'). Verifies the server does not expose files outside its intended scope.",
|
|
693
|
+
recommendation: "Validate and sanitize file paths. Use path.resolve() and verify the result is within the allowed directory. Reject paths containing '..' segments. Use a chroot or sandboxed filesystem for file operations."
|
|
694
|
+
},
|
|
695
|
+
{
|
|
696
|
+
id: "security-ssrf-internal",
|
|
697
|
+
name: "Resists SSRF to internal networks",
|
|
698
|
+
category: "security",
|
|
699
|
+
required: false,
|
|
700
|
+
specRef: "server/tools#calling-tools",
|
|
701
|
+
description: "For tools that accept URL parameters, submits internal IP addresses (169.254.169.254, 127.0.0.1, 10.0.0.0/8) and cloud metadata endpoints. Verifies the server blocks requests to internal networks.",
|
|
702
|
+
recommendation: "Validate and restrict URLs in tool parameters. Block requests to private IP ranges (10.x, 172.16-31.x, 192.168.x, 127.x, 169.254.x), link-local addresses, and cloud metadata endpoints. Use an allowlist of permitted domains."
|
|
703
|
+
},
|
|
704
|
+
{
|
|
705
|
+
id: "security-oversized-input",
|
|
706
|
+
name: "Handles oversized inputs gracefully",
|
|
707
|
+
category: "security",
|
|
708
|
+
required: false,
|
|
709
|
+
specRef: "server/tools#calling-tools",
|
|
710
|
+
description: "Sends a tools/call request with an extremely large argument value (1MB+ string). Verifies the server rejects it with an error instead of crashing or consuming excessive resources.",
|
|
711
|
+
recommendation: "Implement request body size limits. Return HTTP 413 or JSON-RPC error for oversized payloads. Set explicit maxBodyLength in your HTTP server configuration."
|
|
712
|
+
},
|
|
713
|
+
{
|
|
714
|
+
id: "security-extra-params",
|
|
715
|
+
name: "Rejects or ignores extra tool params",
|
|
716
|
+
category: "security",
|
|
717
|
+
required: false,
|
|
718
|
+
specRef: "server/tools#calling-tools",
|
|
719
|
+
description: "Calls a tool with unexpected additional parameters beyond what the schema defines. Verifies the server either rejects them (strict) or silently ignores them (permissive) without errors.",
|
|
720
|
+
recommendation: "Use JSON Schema validation with additionalProperties: false to reject unexpected parameters, or strip unknown properties before processing. Do not pass unvalidated properties to internal functions."
|
|
721
|
+
},
|
|
722
|
+
// ── Security: Tool Integrity (4 tests) ───────────────────────────
|
|
723
|
+
{
|
|
724
|
+
id: "security-tool-schema-defined",
|
|
725
|
+
name: "All tools define inputSchema",
|
|
726
|
+
category: "security",
|
|
727
|
+
required: false,
|
|
728
|
+
specRef: "server/tools#data-types",
|
|
729
|
+
description: "Verifies all tools have an inputSchema with type 'object'. Tools without schemas cannot have their inputs validated, creating an injection risk.",
|
|
730
|
+
recommendation: "Define a complete JSON Schema (inputSchema with type: 'object') for every tool. Specify all expected properties, their types, and constraints. This enables input validation and prevents parameter injection."
|
|
731
|
+
},
|
|
732
|
+
{
|
|
733
|
+
id: "security-tool-rug-pull",
|
|
734
|
+
name: "Tool definitions are stable across calls",
|
|
735
|
+
category: "security",
|
|
736
|
+
required: false,
|
|
737
|
+
specRef: "server/tools#listing-tools",
|
|
738
|
+
description: "Calls tools/list twice and compares the results. Tool definitions should not change between calls within the same session, which could indicate a rug-pull attack.",
|
|
739
|
+
recommendation: "Ensure tools/list returns consistent results within a session. If tools change dynamically, send a tools/list_changed notification. Never silently alter tool definitions \u2014 this is a known MCP attack vector (tool poisoning)."
|
|
740
|
+
},
|
|
741
|
+
{
|
|
742
|
+
id: "security-tool-description-poisoning",
|
|
743
|
+
name: "Tool descriptions free of injection patterns",
|
|
744
|
+
category: "security",
|
|
745
|
+
required: false,
|
|
746
|
+
specRef: "server/tools#data-types",
|
|
747
|
+
description: "Scans all tool names, descriptions, and parameter descriptions for prompt injection patterns: 'ignore previous', 'override', 'system prompt', hidden Unicode characters, and Base64-encoded strings.",
|
|
748
|
+
recommendation: "Review all tool descriptions for prompt injection patterns. Remove any text that attempts to override LLM instructions, references system prompts, or contains hidden characters. Tool descriptions are rendered to LLMs and can be used for prompt injection."
|
|
749
|
+
},
|
|
750
|
+
{
|
|
751
|
+
id: "security-tool-cross-reference",
|
|
752
|
+
name: "Tools do not reference other tools by name",
|
|
753
|
+
category: "security",
|
|
754
|
+
required: false,
|
|
755
|
+
specRef: "server/tools#data-types",
|
|
756
|
+
description: "Checks that tool descriptions do not reference other tool names. Cross-references between tools can be used to manipulate LLM tool selection and create implicit execution chains.",
|
|
757
|
+
recommendation: "Avoid referencing other tool names in tool descriptions. Each tool should be self-contained. If tools have dependencies, document them in server instructions, not in individual tool descriptions."
|
|
758
|
+
},
|
|
759
|
+
// ── Security: Information Disclosure (3 tests) ───────────────────
|
|
760
|
+
{
|
|
761
|
+
id: "security-error-no-stacktrace",
|
|
762
|
+
name: "Error responses do not leak stack traces",
|
|
763
|
+
category: "security",
|
|
764
|
+
required: false,
|
|
765
|
+
specRef: "basic",
|
|
766
|
+
description: "Triggers various error conditions and inspects responses for stack traces, file paths, and internal implementation details. Error responses should not reveal server internals.",
|
|
767
|
+
recommendation: "Sanitize error responses before returning them to clients. Remove stack traces, file paths, database connection strings, and internal IP addresses. Use generic error messages for unexpected failures."
|
|
768
|
+
},
|
|
769
|
+
{
|
|
770
|
+
id: "security-error-no-internal-ip",
|
|
771
|
+
name: "Error responses do not leak internal IPs",
|
|
772
|
+
category: "security",
|
|
773
|
+
required: false,
|
|
774
|
+
specRef: "basic",
|
|
775
|
+
description: "Inspects error response bodies for private IP addresses (10.x, 172.16-31.x, 192.168.x, 127.x) that would reveal internal network topology.",
|
|
776
|
+
recommendation: "Strip internal IP addresses from error responses. Configure your reverse proxy to not forward X-Real-IP or internal addressing. Use a centralized error handler that sanitizes responses."
|
|
777
|
+
},
|
|
778
|
+
{
|
|
779
|
+
id: "security-rate-limiting",
|
|
780
|
+
name: "Rate limiting is enforced",
|
|
781
|
+
category: "security",
|
|
782
|
+
required: false,
|
|
783
|
+
specRef: "basic/transports#streamable-http",
|
|
784
|
+
description: "Sends a burst of rapid requests and checks if the server eventually returns HTTP 429 Too Many Requests. Production servers should implement rate limiting to prevent abuse.",
|
|
785
|
+
recommendation: "Implement rate limiting on your MCP endpoint. Return HTTP 429 with a Retry-After header when limits are exceeded. Consider per-IP, per-token, and per-session rate limits."
|
|
512
786
|
}
|
|
513
787
|
];
|
|
514
788
|
|
|
515
789
|
// src/runner.ts
|
|
790
|
+
var TEST_DEFINITIONS_MAP = new Map(TEST_DEFINITIONS.map((t) => [t.id, t]));
|
|
516
791
|
var _require = createRequire(import.meta.url);
|
|
517
792
|
var { version: TOOL_VERSION } = _require("../package.json");
|
|
518
793
|
var SPEC_VERSION = "2025-11-25";
|
|
519
794
|
var SPEC_BASE = `https://modelcontextprotocol.io/specification/${SPEC_VERSION}`;
|
|
520
795
|
var VALID_CONTENT_TYPES = ["text", "image", "audio", "resource", "resource_link"];
|
|
796
|
+
var INJECTION_PAYLOADS = {
|
|
797
|
+
command: ["; cat /etc/passwd", "$(whoami)", "| ls -la", "`id`", "&& echo pwned"],
|
|
798
|
+
sql: ["' OR 1=1 --", "'; DROP TABLE users; --", "1 UNION SELECT * FROM information_schema.tables--"],
|
|
799
|
+
pathTraversal: ["../../etc/passwd", "..\\..\\windows\\system.ini", "%2e%2e%2f%2e%2e%2fetc%2fpasswd"],
|
|
800
|
+
ssrf: ["http://169.254.169.254/latest/meta-data/", "http://127.0.0.1:22", "http://[::1]/", "http://10.0.0.1/"]
|
|
801
|
+
};
|
|
802
|
+
var STACK_TRACE_PATTERNS = [
|
|
803
|
+
/at\s+\S+\s+\(.*:\d+:\d+\)/i,
|
|
804
|
+
// Node.js: "at Function (file.js:10:5)"
|
|
805
|
+
/Traceback\s+\(most recent/i,
|
|
806
|
+
// Python
|
|
807
|
+
/\.py",\s+line\s+\d+/i,
|
|
808
|
+
// Python file reference
|
|
809
|
+
/\.java:\d+\)/i,
|
|
810
|
+
// Java
|
|
811
|
+
/\.go:\d+/i,
|
|
812
|
+
// Go
|
|
813
|
+
/from\s+\S+\.rb:\d+/i,
|
|
814
|
+
// Ruby
|
|
815
|
+
/\.cs:line\s+\d+/i,
|
|
816
|
+
// C#/.NET
|
|
817
|
+
/#\d+\s+\/.*\.php\(\d+\)/i,
|
|
818
|
+
// PHP
|
|
819
|
+
/panicked\s+at\s+'/i,
|
|
820
|
+
// Rust
|
|
821
|
+
/ENOENT|EACCES|EPERM/,
|
|
822
|
+
// Node.js system errors
|
|
823
|
+
/node_modules\//,
|
|
824
|
+
// Node.js module paths
|
|
825
|
+
/\/usr\/local\/|\/home\//,
|
|
826
|
+
// Unix paths
|
|
827
|
+
/[A-Z]:\\.*\\/,
|
|
828
|
+
// Windows paths
|
|
829
|
+
/password|passwd|secret|credential/i,
|
|
830
|
+
// Sensitive terms
|
|
831
|
+
/jdbc:|mysql:|postgres:|mongodb:/i
|
|
832
|
+
// DB connection strings
|
|
833
|
+
];
|
|
834
|
+
var INTERNAL_IP_PATTERNS = [
|
|
835
|
+
/\b10\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/,
|
|
836
|
+
/\b172\.(1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3}\b/,
|
|
837
|
+
/\b192\.168\.\d{1,3}\.\d{1,3}\b/,
|
|
838
|
+
/\b127\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/,
|
|
839
|
+
/\b::1\b/,
|
|
840
|
+
// IPv6 loopback
|
|
841
|
+
/\bfe80:/i,
|
|
842
|
+
// IPv6 link-local
|
|
843
|
+
/\bf[cd][0-9a-f]{2}:/i
|
|
844
|
+
// IPv6 unique local (fc00::/fd00::)
|
|
845
|
+
];
|
|
521
846
|
function createIdCounter(start = 0) {
|
|
522
847
|
let id = start;
|
|
523
848
|
return () => ++id;
|
|
524
849
|
}
|
|
525
850
|
function parseSSEResponse(text) {
|
|
526
851
|
const lines = text.split("\n");
|
|
527
|
-
let
|
|
852
|
+
let firstJsonRpcResponse = null;
|
|
528
853
|
let currentData = [];
|
|
529
854
|
function flushEvent() {
|
|
530
855
|
if (currentData.length === 0) return;
|
|
@@ -533,8 +858,8 @@ function parseSSEResponse(text) {
|
|
|
533
858
|
if (!data.trim()) return;
|
|
534
859
|
try {
|
|
535
860
|
const parsed = JSON.parse(data);
|
|
536
|
-
if (parsed.jsonrpc === "2.0" && parsed.id !== void 0) {
|
|
537
|
-
|
|
861
|
+
if (!firstJsonRpcResponse && parsed.jsonrpc === "2.0" && parsed.id !== void 0) {
|
|
862
|
+
firstJsonRpcResponse = parsed;
|
|
538
863
|
}
|
|
539
864
|
} catch {
|
|
540
865
|
}
|
|
@@ -548,7 +873,7 @@ function parseSSEResponse(text) {
|
|
|
548
873
|
}
|
|
549
874
|
}
|
|
550
875
|
flushEvent();
|
|
551
|
-
return
|
|
876
|
+
return firstJsonRpcResponse;
|
|
552
877
|
}
|
|
553
878
|
async function mcpRequest(backendUrl, method, params, nextId, extraHeaders, timeout) {
|
|
554
879
|
const id = nextId();
|
|
@@ -758,6 +1083,32 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
758
1083
|
return { passed: valid, details: `Content-Type: ${ct}` };
|
|
759
1084
|
}
|
|
760
1085
|
);
|
|
1086
|
+
await test(
|
|
1087
|
+
"transport-content-type-reject",
|
|
1088
|
+
"Rejects non-JSON request Content-Type",
|
|
1089
|
+
"transport",
|
|
1090
|
+
false,
|
|
1091
|
+
"basic/transports#streamable-http",
|
|
1092
|
+
async () => {
|
|
1093
|
+
const res = await request(backendUrl, {
|
|
1094
|
+
method: "POST",
|
|
1095
|
+
headers: { "Content-Type": "text/plain", Accept: "application/json, text/event-stream", ...userHeaders },
|
|
1096
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 99905, method: "ping" }),
|
|
1097
|
+
signal: AbortSignal.timeout(timeout)
|
|
1098
|
+
});
|
|
1099
|
+
await res.body.text();
|
|
1100
|
+
if (res.statusCode >= 400 && res.statusCode < 500) {
|
|
1101
|
+
return { passed: true, details: `HTTP ${res.statusCode} (incorrect Content-Type rejected)` };
|
|
1102
|
+
}
|
|
1103
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1104
|
+
return {
|
|
1105
|
+
passed: false,
|
|
1106
|
+
details: `HTTP ${res.statusCode} \u2014 server accepted text/plain Content-Type (should require application/json)`
|
|
1107
|
+
};
|
|
1108
|
+
}
|
|
1109
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
1110
|
+
}
|
|
1111
|
+
);
|
|
761
1112
|
await test(
|
|
762
1113
|
"transport-get",
|
|
763
1114
|
"GET returns SSE stream or 405",
|
|
@@ -772,7 +1123,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
772
1123
|
signal: AbortSignal.timeout(timeout)
|
|
773
1124
|
});
|
|
774
1125
|
const body = await res.body.text();
|
|
775
|
-
const
|
|
1126
|
+
const rawCt = res.headers["content-type"];
|
|
1127
|
+
const ct = (Array.isArray(rawCt) ? rawCt[0] : rawCt || "").toLowerCase();
|
|
776
1128
|
if (res.statusCode === 405) {
|
|
777
1129
|
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
778
1130
|
}
|
|
@@ -880,6 +1232,50 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
880
1232
|
return { passed: valid, details: `Version: ${version3}` };
|
|
881
1233
|
}
|
|
882
1234
|
);
|
|
1235
|
+
await test(
|
|
1236
|
+
"lifecycle-version-negotiate",
|
|
1237
|
+
"Handles unknown protocol version",
|
|
1238
|
+
"lifecycle",
|
|
1239
|
+
false,
|
|
1240
|
+
"basic/lifecycle#version-negotiation",
|
|
1241
|
+
async () => {
|
|
1242
|
+
try {
|
|
1243
|
+
const futureRes = await mcpRequest(
|
|
1244
|
+
backendUrl,
|
|
1245
|
+
"initialize",
|
|
1246
|
+
{
|
|
1247
|
+
protocolVersion: "2099-01-01",
|
|
1248
|
+
capabilities: {},
|
|
1249
|
+
clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
|
|
1250
|
+
},
|
|
1251
|
+
createIdCounter(99960),
|
|
1252
|
+
userHeaders,
|
|
1253
|
+
timeout
|
|
1254
|
+
);
|
|
1255
|
+
const result = futureRes.body?.result;
|
|
1256
|
+
const error = futureRes.body?.error;
|
|
1257
|
+
if (error) {
|
|
1258
|
+
return {
|
|
1259
|
+
passed: true,
|
|
1260
|
+
details: `Server rejected unknown version with error: ${error.code} \u2014 ${error.message}`
|
|
1261
|
+
};
|
|
1262
|
+
}
|
|
1263
|
+
if (result?.protocolVersion) {
|
|
1264
|
+
const offered = result.protocolVersion;
|
|
1265
|
+
if (offered === "2099-01-01") {
|
|
1266
|
+
return {
|
|
1267
|
+
passed: false,
|
|
1268
|
+
details: 'Server accepted impossible future version "2099-01-01" \u2014 should offer a version it actually supports'
|
|
1269
|
+
};
|
|
1270
|
+
}
|
|
1271
|
+
return { passed: true, details: `Server negotiated down to ${offered} (correct)` };
|
|
1272
|
+
}
|
|
1273
|
+
return { passed: false, details: "No protocolVersion or error in response" };
|
|
1274
|
+
} catch {
|
|
1275
|
+
return { passed: true, details: "Connection rejected for unknown version (acceptable)" };
|
|
1276
|
+
}
|
|
1277
|
+
}
|
|
1278
|
+
);
|
|
883
1279
|
await test(
|
|
884
1280
|
"lifecycle-server-info",
|
|
885
1281
|
"Includes serverInfo",
|
|
@@ -951,6 +1347,73 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
951
1347
|
details: match ? `Request id=${res.requestId}, response id=${body.id} (match)` : `Request id=${res.requestId}, response id=${body.id} (MISMATCH)`
|
|
952
1348
|
};
|
|
953
1349
|
});
|
|
1350
|
+
await test("lifecycle-string-id", "Supports string request IDs", "lifecycle", false, "basic", async () => {
|
|
1351
|
+
const stringId = "compliance-test-string-id";
|
|
1352
|
+
const body = JSON.stringify({ jsonrpc: "2.0", id: stringId, method: "ping", params: {} });
|
|
1353
|
+
const res = await request(backendUrl, {
|
|
1354
|
+
method: "POST",
|
|
1355
|
+
headers: {
|
|
1356
|
+
"Content-Type": "application/json",
|
|
1357
|
+
Accept: "application/json, text/event-stream",
|
|
1358
|
+
...buildHeaders()
|
|
1359
|
+
},
|
|
1360
|
+
body,
|
|
1361
|
+
signal: AbortSignal.timeout(timeout)
|
|
1362
|
+
});
|
|
1363
|
+
const text = await res.body.text();
|
|
1364
|
+
const rawCtStr = res.headers["content-type"];
|
|
1365
|
+
const ct = (Array.isArray(rawCtStr) ? rawCtStr[0] : rawCtStr || "").toLowerCase();
|
|
1366
|
+
let parsed;
|
|
1367
|
+
if (ct.includes("text/event-stream")) {
|
|
1368
|
+
parsed = parseSSEResponse(text);
|
|
1369
|
+
}
|
|
1370
|
+
if (!parsed) {
|
|
1371
|
+
try {
|
|
1372
|
+
parsed = JSON.parse(text);
|
|
1373
|
+
} catch {
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
if (!parsed) return { passed: false, details: "Could not parse response" };
|
|
1377
|
+
if (parsed.id === stringId) {
|
|
1378
|
+
return { passed: true, details: `String id="${stringId}" echoed back correctly` };
|
|
1379
|
+
}
|
|
1380
|
+
if (parsed.id === void 0) {
|
|
1381
|
+
return { passed: false, details: "No id in response" };
|
|
1382
|
+
}
|
|
1383
|
+
return {
|
|
1384
|
+
passed: false,
|
|
1385
|
+
details: `String id="${stringId}" sent, got back id=${JSON.stringify(parsed.id)} (type: ${typeof parsed.id})`
|
|
1386
|
+
};
|
|
1387
|
+
});
|
|
1388
|
+
await test(
|
|
1389
|
+
"lifecycle-reinit-reject",
|
|
1390
|
+
"Rejects second initialize request",
|
|
1391
|
+
"lifecycle",
|
|
1392
|
+
false,
|
|
1393
|
+
"basic/lifecycle#initialization",
|
|
1394
|
+
async () => {
|
|
1395
|
+
try {
|
|
1396
|
+
const res = await rpc("initialize", {
|
|
1397
|
+
protocolVersion: SPEC_VERSION,
|
|
1398
|
+
capabilities: {},
|
|
1399
|
+
clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
|
|
1400
|
+
});
|
|
1401
|
+
const error = res.body?.error;
|
|
1402
|
+
if (error) {
|
|
1403
|
+
return { passed: true, details: `Re-initialization rejected with error: ${error.code} \u2014 ${error.message}` };
|
|
1404
|
+
}
|
|
1405
|
+
if (res.statusCode >= 400) {
|
|
1406
|
+
return { passed: true, details: `HTTP ${res.statusCode} (re-initialization rejected)` };
|
|
1407
|
+
}
|
|
1408
|
+
return {
|
|
1409
|
+
passed: false,
|
|
1410
|
+
details: `Server accepted second initialize (HTTP ${res.statusCode}) \u2014 should reject duplicate initialization`
|
|
1411
|
+
};
|
|
1412
|
+
} catch {
|
|
1413
|
+
return { passed: true, details: "Connection rejected (acceptable)" };
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
);
|
|
954
1417
|
const hasLogging = !!serverInfo.capabilities.logging;
|
|
955
1418
|
await test(
|
|
956
1419
|
"lifecycle-logging",
|
|
@@ -1025,7 +1488,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1025
1488
|
);
|
|
1026
1489
|
await test(
|
|
1027
1490
|
"lifecycle-progress",
|
|
1028
|
-
"
|
|
1491
|
+
"Handles progress notifications gracefully",
|
|
1029
1492
|
"lifecycle",
|
|
1030
1493
|
false,
|
|
1031
1494
|
"basic/utilities#progress",
|
|
@@ -1038,9 +1501,12 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1038
1501
|
timeout
|
|
1039
1502
|
);
|
|
1040
1503
|
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1041
|
-
return { passed: true, details: `HTTP ${res.statusCode} (
|
|
1504
|
+
return { passed: true, details: `HTTP ${res.statusCode} (notification handled gracefully)` };
|
|
1042
1505
|
}
|
|
1043
|
-
return {
|
|
1506
|
+
return {
|
|
1507
|
+
passed: false,
|
|
1508
|
+
details: `HTTP ${res.statusCode} \u2014 server should accept unknown notifications without error`
|
|
1509
|
+
};
|
|
1044
1510
|
}
|
|
1045
1511
|
);
|
|
1046
1512
|
await test(
|
|
@@ -1082,7 +1548,11 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1082
1548
|
return { passed: true, details: "HTTP 202 Accepted (correct)" };
|
|
1083
1549
|
}
|
|
1084
1550
|
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1085
|
-
|
|
1551
|
+
warnings.push(`Notification returned HTTP ${res.statusCode} instead of spec-required 202 Accepted`);
|
|
1552
|
+
return {
|
|
1553
|
+
passed: false,
|
|
1554
|
+
details: `HTTP ${res.statusCode} \u2014 spec requires 202 Accepted for notifications (MUST)`
|
|
1555
|
+
};
|
|
1086
1556
|
}
|
|
1087
1557
|
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected 202 Accepted for notifications` };
|
|
1088
1558
|
}
|
|
@@ -1113,6 +1583,34 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1113
1583
|
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
1114
1584
|
}
|
|
1115
1585
|
);
|
|
1586
|
+
await test(
|
|
1587
|
+
"transport-session-invalid",
|
|
1588
|
+
"Returns 404 for unknown session ID",
|
|
1589
|
+
"transport",
|
|
1590
|
+
false,
|
|
1591
|
+
"basic/transports#streamable-http",
|
|
1592
|
+
async () => {
|
|
1593
|
+
if (!sessionId) {
|
|
1594
|
+
return { passed: true, details: "Server did not issue session ID (test not applicable)" };
|
|
1595
|
+
}
|
|
1596
|
+
const fakeHeaders = {
|
|
1597
|
+
...userHeaders,
|
|
1598
|
+
"mcp-session-id": "invalid-nonexistent-session-id"
|
|
1599
|
+
};
|
|
1600
|
+
if (negotiatedProtocolVersion) fakeHeaders["mcp-protocol-version"] = negotiatedProtocolVersion;
|
|
1601
|
+
const res = await mcpRequest(backendUrl, "ping", void 0, createIdCounter(99915), fakeHeaders, timeout);
|
|
1602
|
+
if (res.statusCode === 404) {
|
|
1603
|
+
return { passed: true, details: "HTTP 404 for unknown session ID (correct per spec)" };
|
|
1604
|
+
}
|
|
1605
|
+
if (res.statusCode === 400) {
|
|
1606
|
+
return {
|
|
1607
|
+
passed: false,
|
|
1608
|
+
details: "HTTP 400 \u2014 spec requires 404 (Not Found) for unrecognized session IDs, not 400"
|
|
1609
|
+
};
|
|
1610
|
+
}
|
|
1611
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 spec requires 404 for unrecognized MCP-Session-Id` };
|
|
1612
|
+
}
|
|
1613
|
+
);
|
|
1116
1614
|
await test(
|
|
1117
1615
|
"transport-get-stream",
|
|
1118
1616
|
"GET with session returns SSE or 405",
|
|
@@ -1129,7 +1627,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1129
1627
|
signal: AbortSignal.timeout(Math.min(timeout, 3e3))
|
|
1130
1628
|
});
|
|
1131
1629
|
const body = await res.body.text();
|
|
1132
|
-
const
|
|
1630
|
+
const rawCt2 = res.headers["content-type"];
|
|
1631
|
+
const ct = (Array.isArray(rawCt2) ? rawCt2[0] : rawCt2 || "").toLowerCase();
|
|
1133
1632
|
if (res.statusCode === 405) {
|
|
1134
1633
|
return { passed: true, details: "HTTP 405 (server does not support server-initiated messages)" };
|
|
1135
1634
|
}
|
|
@@ -1168,7 +1667,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1168
1667
|
signal: AbortSignal.timeout(timeout)
|
|
1169
1668
|
}).then(async (res) => {
|
|
1170
1669
|
const text = await res.body.text();
|
|
1171
|
-
const
|
|
1670
|
+
const rawCtConcurrent = res.headers["content-type"];
|
|
1671
|
+
const ct = (Array.isArray(rawCtConcurrent) ? rawCtConcurrent[0] : rawCtConcurrent || "").toLowerCase();
|
|
1172
1672
|
let body;
|
|
1173
1673
|
if (ct.includes("text/event-stream")) {
|
|
1174
1674
|
body = parseSSEResponse(text);
|
|
@@ -1195,6 +1695,42 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1195
1695
|
return { passed: true, details: `${results.length} concurrent requests handled correctly` };
|
|
1196
1696
|
}
|
|
1197
1697
|
);
|
|
1698
|
+
await test(
|
|
1699
|
+
"transport-sse-event-field",
|
|
1700
|
+
"SSE responses include event: message",
|
|
1701
|
+
"transport",
|
|
1702
|
+
false,
|
|
1703
|
+
"basic/transports#streamable-http",
|
|
1704
|
+
async () => {
|
|
1705
|
+
const res = await request(backendUrl, {
|
|
1706
|
+
method: "POST",
|
|
1707
|
+
headers: {
|
|
1708
|
+
"Content-Type": "application/json",
|
|
1709
|
+
Accept: "text/event-stream",
|
|
1710
|
+
...buildHeaders()
|
|
1711
|
+
},
|
|
1712
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: createIdCounter(99940)(), method: "ping" }),
|
|
1713
|
+
signal: AbortSignal.timeout(timeout)
|
|
1714
|
+
});
|
|
1715
|
+
const text = await res.body.text();
|
|
1716
|
+
const rawCtSse = res.headers["content-type"];
|
|
1717
|
+
const ct = (Array.isArray(rawCtSse) ? rawCtSse[0] : rawCtSse || "").toLowerCase();
|
|
1718
|
+
if (!ct.includes("text/event-stream")) {
|
|
1719
|
+
return { passed: true, details: "Server responded with JSON (not SSE) \u2014 event field check not applicable" };
|
|
1720
|
+
}
|
|
1721
|
+
const hasEventMessage = /^event:\s*message\s*$/m.test(text);
|
|
1722
|
+
if (hasEventMessage) {
|
|
1723
|
+
return { passed: true, details: "SSE response includes required event: message field" };
|
|
1724
|
+
}
|
|
1725
|
+
if (text.includes("data:")) {
|
|
1726
|
+
return {
|
|
1727
|
+
passed: false,
|
|
1728
|
+
details: "SSE response has data: fields but missing required event: message field (spec: MUST include event: message)"
|
|
1729
|
+
};
|
|
1730
|
+
}
|
|
1731
|
+
return { passed: true, details: "SSE response empty or no data fields \u2014 check not applicable" };
|
|
1732
|
+
}
|
|
1733
|
+
);
|
|
1198
1734
|
const hasTools = !!serverInfo.capabilities.tools;
|
|
1199
1735
|
let cachedToolsList = null;
|
|
1200
1736
|
await test(
|
|
@@ -1276,9 +1812,6 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1276
1812
|
issues.push(`${tool.name}: annotations.${field} should be boolean, got ${typeof ann[field]}`);
|
|
1277
1813
|
}
|
|
1278
1814
|
}
|
|
1279
|
-
if (ann.title !== void 0 && typeof ann.title !== "string") {
|
|
1280
|
-
issues.push(`${tool.name}: annotations.title should be string`);
|
|
1281
|
-
}
|
|
1282
1815
|
}
|
|
1283
1816
|
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1284
1817
|
return {
|
|
@@ -1826,6 +2359,822 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1826
2359
|
}
|
|
1827
2360
|
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error code -32600` };
|
|
1828
2361
|
});
|
|
2362
|
+
const undeclaredMethods = [
|
|
2363
|
+
{ method: "tools/list", capability: "tools", declared: hasTools },
|
|
2364
|
+
{ method: "resources/list", capability: "resources", declared: hasResources },
|
|
2365
|
+
{ method: "prompts/list", capability: "prompts", declared: hasPrompts }
|
|
2366
|
+
];
|
|
2367
|
+
const undeclared = undeclaredMethods.filter((m) => !m.declared);
|
|
2368
|
+
await test(
|
|
2369
|
+
"error-capability-gated",
|
|
2370
|
+
"Rejects methods for undeclared capabilities",
|
|
2371
|
+
"errors",
|
|
2372
|
+
false,
|
|
2373
|
+
"basic/lifecycle#capability-negotiation",
|
|
2374
|
+
async () => {
|
|
2375
|
+
if (undeclared.length === 0) {
|
|
2376
|
+
return {
|
|
2377
|
+
passed: true,
|
|
2378
|
+
details: "Server declares all capabilities (tools, resources, prompts) \u2014 no undeclared methods to test"
|
|
2379
|
+
};
|
|
2380
|
+
}
|
|
2381
|
+
const issues = [];
|
|
2382
|
+
for (const { method, capability } of undeclared) {
|
|
2383
|
+
const res = await rpc(method);
|
|
2384
|
+
const error = res.body?.error;
|
|
2385
|
+
if (!error && res.body?.result) {
|
|
2386
|
+
issues.push(`${method} returned success despite missing ${capability} capability`);
|
|
2387
|
+
}
|
|
2388
|
+
}
|
|
2389
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
2390
|
+
return {
|
|
2391
|
+
passed: true,
|
|
2392
|
+
details: `Tested ${undeclared.length} undeclared method(s): ${undeclared.map((m) => m.method).join(", ")} \u2014 all returned errors`
|
|
2393
|
+
};
|
|
2394
|
+
}
|
|
2395
|
+
);
|
|
2396
|
+
const listMethodForCursor = hasTools ? "tools/list" : hasResources ? "resources/list" : hasPrompts ? "prompts/list" : null;
|
|
2397
|
+
await test(
|
|
2398
|
+
"error-invalid-cursor",
|
|
2399
|
+
"Handles invalid pagination cursor gracefully",
|
|
2400
|
+
"errors",
|
|
2401
|
+
false,
|
|
2402
|
+
"basic",
|
|
2403
|
+
async () => {
|
|
2404
|
+
if (!listMethodForCursor) {
|
|
2405
|
+
return { passed: true, details: "No list methods available to test (skipped)" };
|
|
2406
|
+
}
|
|
2407
|
+
const res = await rpc(listMethodForCursor, { cursor: "!!!invalid-garbage-cursor-$$$" });
|
|
2408
|
+
const error = res.body?.error;
|
|
2409
|
+
if (error) {
|
|
2410
|
+
return { passed: true, details: `Invalid cursor rejected with error: ${error.code} \u2014 ${error.message}` };
|
|
2411
|
+
}
|
|
2412
|
+
const result = res.body?.result;
|
|
2413
|
+
if (result) {
|
|
2414
|
+
return { passed: true, details: "Server returned results (likely ignored invalid cursor)" };
|
|
2415
|
+
}
|
|
2416
|
+
return { passed: false, details: "No error or result for invalid cursor" };
|
|
2417
|
+
}
|
|
2418
|
+
);
|
|
2419
|
+
const hasAuth = !!userHeaders.Authorization || !!userHeaders.authorization;
|
|
2420
|
+
await test(
|
|
2421
|
+
"security-auth-required",
|
|
2422
|
+
"Rejects unauthenticated requests",
|
|
2423
|
+
"security",
|
|
2424
|
+
false,
|
|
2425
|
+
"basic/authorization",
|
|
2426
|
+
async () => {
|
|
2427
|
+
if (!hasAuth) {
|
|
2428
|
+
return {
|
|
2429
|
+
passed: false,
|
|
2430
|
+
details: "Server does not require auth (no --auth provided and server accepted unauthenticated requests)"
|
|
2431
|
+
};
|
|
2432
|
+
}
|
|
2433
|
+
const noAuthHeaders = {};
|
|
2434
|
+
if (sessionId) noAuthHeaders["mcp-session-id"] = sessionId;
|
|
2435
|
+
try {
|
|
2436
|
+
const res = await mcpRequest(backendUrl, "ping", void 0, nextId, noAuthHeaders, timeout);
|
|
2437
|
+
if (res.statusCode === 401 || res.statusCode === 403) {
|
|
2438
|
+
return { passed: true, details: `HTTP ${res.statusCode} (unauthenticated request rejected)` };
|
|
2439
|
+
}
|
|
2440
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server accepted unauthenticated request` };
|
|
2441
|
+
} catch (err) {
|
|
2442
|
+
return { passed: true, details: "Connection rejected (acceptable)" };
|
|
2443
|
+
}
|
|
2444
|
+
}
|
|
2445
|
+
);
|
|
2446
|
+
await test(
|
|
2447
|
+
"security-auth-malformed",
|
|
2448
|
+
"Rejects malformed auth credentials",
|
|
2449
|
+
"security",
|
|
2450
|
+
false,
|
|
2451
|
+
"basic/authorization",
|
|
2452
|
+
async () => {
|
|
2453
|
+
if (!hasAuth) {
|
|
2454
|
+
return { passed: false, details: "Skipped: server does not require auth" };
|
|
2455
|
+
}
|
|
2456
|
+
const malformedHeaders = {
|
|
2457
|
+
Authorization: "Bearer INVALID_GARBAGE_TOKEN_!@#$%^&*()"
|
|
2458
|
+
};
|
|
2459
|
+
if (sessionId) malformedHeaders["mcp-session-id"] = sessionId;
|
|
2460
|
+
try {
|
|
2461
|
+
const res = await mcpRequest(backendUrl, "ping", void 0, nextId, malformedHeaders, timeout);
|
|
2462
|
+
if (res.statusCode === 401 || res.statusCode === 403) {
|
|
2463
|
+
return { passed: true, details: `HTTP ${res.statusCode} (malformed auth rejected)` };
|
|
2464
|
+
}
|
|
2465
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server accepted malformed auth token` };
|
|
2466
|
+
} catch (err) {
|
|
2467
|
+
return { passed: true, details: "Connection rejected (acceptable)" };
|
|
2468
|
+
}
|
|
2469
|
+
}
|
|
2470
|
+
);
|
|
2471
|
+
await test("security-tls-required", "Enforces HTTPS/TLS", "security", false, "basic/authorization", async () => {
|
|
2472
|
+
const parsedUrl = new URL(url);
|
|
2473
|
+
if (parsedUrl.protocol !== "https:") {
|
|
2474
|
+
return { passed: false, details: `Server URL uses ${parsedUrl.protocol} \u2014 production servers should use HTTPS` };
|
|
2475
|
+
}
|
|
2476
|
+
const httpUrl = url.replace(/^https:/, "http:");
|
|
2477
|
+
try {
|
|
2478
|
+
const res = await request(httpUrl, {
|
|
2479
|
+
method: "POST",
|
|
2480
|
+
headers: { "Content-Type": "application/json", Accept: "application/json" },
|
|
2481
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 99950, method: "ping" }),
|
|
2482
|
+
signal: AbortSignal.timeout(Math.min(timeout, 5e3))
|
|
2483
|
+
});
|
|
2484
|
+
await res.body.text();
|
|
2485
|
+
if (res.statusCode === 301 || res.statusCode === 302 || res.statusCode === 308) {
|
|
2486
|
+
return { passed: true, details: `HTTP ${res.statusCode} redirect to HTTPS (good)` };
|
|
2487
|
+
}
|
|
2488
|
+
if (res.statusCode >= 400) {
|
|
2489
|
+
return { passed: true, details: `HTTP ${res.statusCode} (plaintext rejected)` };
|
|
2490
|
+
}
|
|
2491
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server accepts plaintext HTTP connections` };
|
|
2492
|
+
} catch {
|
|
2493
|
+
return { passed: true, details: "HTTP connection refused (HTTPS enforced)" };
|
|
2494
|
+
}
|
|
2495
|
+
});
|
|
2496
|
+
await test(
|
|
2497
|
+
"security-session-entropy",
|
|
2498
|
+
"Session IDs are high-entropy",
|
|
2499
|
+
"security",
|
|
2500
|
+
false,
|
|
2501
|
+
"basic/transports#streamable-http",
|
|
2502
|
+
async () => {
|
|
2503
|
+
if (!sessionId) {
|
|
2504
|
+
return { passed: true, details: "Server does not issue session IDs (skipped)" };
|
|
2505
|
+
}
|
|
2506
|
+
if (sessionId.length < 16) {
|
|
2507
|
+
return {
|
|
2508
|
+
passed: false,
|
|
2509
|
+
details: `Session ID too short (${sessionId.length} chars): "${sessionId}" \u2014 should be \u226516 chars`
|
|
2510
|
+
};
|
|
2511
|
+
}
|
|
2512
|
+
if (/^\d+$/.test(sessionId)) {
|
|
2513
|
+
return {
|
|
2514
|
+
passed: false,
|
|
2515
|
+
details: `Session ID is purely numeric: "${sessionId}" \u2014 likely sequential, not random`
|
|
2516
|
+
};
|
|
2517
|
+
}
|
|
2518
|
+
const uniqueChars = new Set(sessionId.toLowerCase()).size;
|
|
2519
|
+
if (uniqueChars < 8) {
|
|
2520
|
+
return {
|
|
2521
|
+
passed: false,
|
|
2522
|
+
details: `Session ID has low character diversity (${uniqueChars} unique chars): "${sessionId}"`
|
|
2523
|
+
};
|
|
2524
|
+
}
|
|
2525
|
+
return {
|
|
2526
|
+
passed: true,
|
|
2527
|
+
details: `Session ID has good entropy (${sessionId.length} chars, ${uniqueChars} unique): "${sessionId.substring(0, 16)}..."`
|
|
2528
|
+
};
|
|
2529
|
+
}
|
|
2530
|
+
);
|
|
2531
|
+
await test(
|
|
2532
|
+
"security-session-not-auth",
|
|
2533
|
+
"Session ID does not bypass auth",
|
|
2534
|
+
"security",
|
|
2535
|
+
false,
|
|
2536
|
+
"basic/transports#streamable-http",
|
|
2537
|
+
async () => {
|
|
2538
|
+
if (!hasAuth) {
|
|
2539
|
+
return { passed: true, details: "Skipped: server does not require auth" };
|
|
2540
|
+
}
|
|
2541
|
+
if (!sessionId) {
|
|
2542
|
+
return { passed: true, details: "Skipped: server does not issue session IDs" };
|
|
2543
|
+
}
|
|
2544
|
+
const sessionOnlyHeaders = {
|
|
2545
|
+
"mcp-session-id": sessionId
|
|
2546
|
+
};
|
|
2547
|
+
try {
|
|
2548
|
+
const res = await mcpRequest(backendUrl, "ping", void 0, nextId, sessionOnlyHeaders, timeout);
|
|
2549
|
+
if (res.statusCode === 401 || res.statusCode === 403) {
|
|
2550
|
+
return { passed: true, details: `HTTP ${res.statusCode} (session ID alone not sufficient for auth)` };
|
|
2551
|
+
}
|
|
2552
|
+
return {
|
|
2553
|
+
passed: false,
|
|
2554
|
+
details: `HTTP ${res.statusCode} \u2014 server accepted session ID without auth (spec: MUST NOT use sessions for authentication)`
|
|
2555
|
+
};
|
|
2556
|
+
} catch {
|
|
2557
|
+
return { passed: true, details: "Connection rejected (acceptable)" };
|
|
2558
|
+
}
|
|
2559
|
+
}
|
|
2560
|
+
);
|
|
2561
|
+
await test(
|
|
2562
|
+
"security-oauth-metadata",
|
|
2563
|
+
"Protected Resource Metadata endpoint exists",
|
|
2564
|
+
"security",
|
|
2565
|
+
false,
|
|
2566
|
+
"basic/authorization",
|
|
2567
|
+
async () => {
|
|
2568
|
+
if (!hasAuth) {
|
|
2569
|
+
return { passed: true, details: "Skipped: server does not require auth" };
|
|
2570
|
+
}
|
|
2571
|
+
const parsedUrl = new URL(url);
|
|
2572
|
+
const prmUrl = `${parsedUrl.protocol}//${parsedUrl.host}/.well-known/oauth-protected-resource`;
|
|
2573
|
+
try {
|
|
2574
|
+
const res = await request(prmUrl, {
|
|
2575
|
+
method: "GET",
|
|
2576
|
+
headers: { Accept: "application/json" },
|
|
2577
|
+
signal: AbortSignal.timeout(Math.min(timeout, 5e3))
|
|
2578
|
+
});
|
|
2579
|
+
const text = await res.body.text();
|
|
2580
|
+
if (res.statusCode === 200) {
|
|
2581
|
+
try {
|
|
2582
|
+
const meta = JSON.parse(text);
|
|
2583
|
+
if (!meta.resource) {
|
|
2584
|
+
return { passed: false, details: "PRM response missing required 'resource' field" };
|
|
2585
|
+
}
|
|
2586
|
+
if (!Array.isArray(meta.authorization_servers) || meta.authorization_servers.length === 0) {
|
|
2587
|
+
return { passed: false, details: "PRM response missing 'authorization_servers' array" };
|
|
2588
|
+
}
|
|
2589
|
+
return {
|
|
2590
|
+
passed: true,
|
|
2591
|
+
details: `Protected Resource Metadata found: resource=${meta.resource}, ${meta.authorization_servers.length} auth server(s)`
|
|
2592
|
+
};
|
|
2593
|
+
} catch {
|
|
2594
|
+
return { passed: false, details: "PRM endpoint returned non-JSON response" };
|
|
2595
|
+
}
|
|
2596
|
+
}
|
|
2597
|
+
const legacyUrl = `${parsedUrl.protocol}//${parsedUrl.host}/.well-known/oauth-authorization-server`;
|
|
2598
|
+
try {
|
|
2599
|
+
const legacyRes = await request(legacyUrl, {
|
|
2600
|
+
method: "GET",
|
|
2601
|
+
headers: { Accept: "application/json" },
|
|
2602
|
+
signal: AbortSignal.timeout(Math.min(timeout, 5e3))
|
|
2603
|
+
});
|
|
2604
|
+
const legacyText = await legacyRes.body.text();
|
|
2605
|
+
if (legacyRes.statusCode === 200) {
|
|
2606
|
+
try {
|
|
2607
|
+
const legacyMeta = JSON.parse(legacyText);
|
|
2608
|
+
if (legacyMeta.issuer && legacyMeta.token_endpoint) {
|
|
2609
|
+
warnings.push(
|
|
2610
|
+
"Server uses legacy /.well-known/oauth-authorization-server instead of /.well-known/oauth-protected-resource (RFC 9728). Update to PRM for 2025-11-25 compliance."
|
|
2611
|
+
);
|
|
2612
|
+
return {
|
|
2613
|
+
passed: true,
|
|
2614
|
+
details: `Legacy OAuth AS metadata found: issuer=${legacyMeta.issuer} (should migrate to PRM)`
|
|
2615
|
+
};
|
|
2616
|
+
}
|
|
2617
|
+
} catch {
|
|
2618
|
+
}
|
|
2619
|
+
}
|
|
2620
|
+
} catch {
|
|
2621
|
+
}
|
|
2622
|
+
return {
|
|
2623
|
+
passed: false,
|
|
2624
|
+
details: `PRM endpoint returned HTTP ${res.statusCode} and no legacy OAuth metadata found`
|
|
2625
|
+
};
|
|
2626
|
+
} catch {
|
|
2627
|
+
return { passed: false, details: "PRM endpoint unreachable" };
|
|
2628
|
+
}
|
|
2629
|
+
}
|
|
2630
|
+
);
|
|
2631
|
+
await test(
|
|
2632
|
+
"security-token-in-uri",
|
|
2633
|
+
"Rejects auth tokens in query string",
|
|
2634
|
+
"security",
|
|
2635
|
+
false,
|
|
2636
|
+
"basic/authorization",
|
|
2637
|
+
async () => {
|
|
2638
|
+
if (!hasAuth) {
|
|
2639
|
+
return { passed: true, details: "Skipped: server does not require auth" };
|
|
2640
|
+
}
|
|
2641
|
+
const authValue = userHeaders.Authorization || userHeaders.authorization || "";
|
|
2642
|
+
const token = authValue.replace(/^Bearer\s+/i, "");
|
|
2643
|
+
if (!token) {
|
|
2644
|
+
return { passed: true, details: "Skipped: could not extract token from auth header" };
|
|
2645
|
+
}
|
|
2646
|
+
const uriWithToken = `${url}${url.includes("?") ? "&" : "?"}access_token=${encodeURIComponent(token)}`;
|
|
2647
|
+
try {
|
|
2648
|
+
const noAuthHeaders = {};
|
|
2649
|
+
if (sessionId) noAuthHeaders["mcp-session-id"] = sessionId;
|
|
2650
|
+
const res = await mcpRequest(uriWithToken, "ping", void 0, nextId, noAuthHeaders, timeout);
|
|
2651
|
+
if (res.statusCode === 401 || res.statusCode === 403) {
|
|
2652
|
+
return { passed: true, details: `HTTP ${res.statusCode} (token in query string rejected)` };
|
|
2653
|
+
}
|
|
2654
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
2655
|
+
return {
|
|
2656
|
+
passed: false,
|
|
2657
|
+
details: "Server accepted auth token in query string (spec: MUST NOT transmit credentials in URIs)"
|
|
2658
|
+
};
|
|
2659
|
+
}
|
|
2660
|
+
return { passed: true, details: `HTTP ${res.statusCode} (token in query string not accepted)` };
|
|
2661
|
+
} catch {
|
|
2662
|
+
return { passed: true, details: "Connection rejected (acceptable)" };
|
|
2663
|
+
}
|
|
2664
|
+
}
|
|
2665
|
+
);
|
|
2666
|
+
await test(
|
|
2667
|
+
"security-cors-headers",
|
|
2668
|
+
"CORS headers are restrictive",
|
|
2669
|
+
"security",
|
|
2670
|
+
false,
|
|
2671
|
+
"basic/transports#streamable-http",
|
|
2672
|
+
async () => {
|
|
2673
|
+
try {
|
|
2674
|
+
const res = await request(backendUrl, {
|
|
2675
|
+
method: "OPTIONS",
|
|
2676
|
+
headers: {
|
|
2677
|
+
Origin: "https://evil.example.com",
|
|
2678
|
+
"Access-Control-Request-Method": "POST",
|
|
2679
|
+
...buildHeaders()
|
|
2680
|
+
},
|
|
2681
|
+
signal: AbortSignal.timeout(Math.min(timeout, 5e3))
|
|
2682
|
+
});
|
|
2683
|
+
await res.body.text();
|
|
2684
|
+
const acao = res.headers["access-control-allow-origin"];
|
|
2685
|
+
if (!acao) {
|
|
2686
|
+
return { passed: true, details: "No CORS headers returned (server-to-server only, acceptable)" };
|
|
2687
|
+
}
|
|
2688
|
+
if (acao === "*") {
|
|
2689
|
+
return {
|
|
2690
|
+
passed: false,
|
|
2691
|
+
details: 'Access-Control-Allow-Origin is "*" (wildcard) \u2014 allows cross-origin credential theft'
|
|
2692
|
+
};
|
|
2693
|
+
}
|
|
2694
|
+
if (acao === "https://evil.example.com") {
|
|
2695
|
+
return { passed: false, details: "Server reflects arbitrary Origin in CORS \u2014 effectively wildcard" };
|
|
2696
|
+
}
|
|
2697
|
+
return { passed: true, details: `CORS restricted to: ${acao}` };
|
|
2698
|
+
} catch {
|
|
2699
|
+
return { passed: true, details: "OPTIONS request failed (no CORS, acceptable)" };
|
|
2700
|
+
}
|
|
2701
|
+
}
|
|
2702
|
+
);
|
|
2703
|
+
await test(
|
|
2704
|
+
"security-origin-validation",
|
|
2705
|
+
"Validates Origin header on requests",
|
|
2706
|
+
"security",
|
|
2707
|
+
false,
|
|
2708
|
+
"basic/transports#streamable-http",
|
|
2709
|
+
async () => {
|
|
2710
|
+
try {
|
|
2711
|
+
const res = await request(backendUrl, {
|
|
2712
|
+
method: "POST",
|
|
2713
|
+
headers: {
|
|
2714
|
+
"Content-Type": "application/json",
|
|
2715
|
+
Accept: "application/json, text/event-stream",
|
|
2716
|
+
Origin: "https://evil-rebinding-attack.example.com",
|
|
2717
|
+
...buildHeaders()
|
|
2718
|
+
},
|
|
2719
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: createIdCounter(99970)(), method: "ping" }),
|
|
2720
|
+
signal: AbortSignal.timeout(timeout)
|
|
2721
|
+
});
|
|
2722
|
+
await res.body.text();
|
|
2723
|
+
if (res.statusCode === 403 || res.statusCode === 401) {
|
|
2724
|
+
return { passed: true, details: `HTTP ${res.statusCode} (suspicious Origin rejected)` };
|
|
2725
|
+
}
|
|
2726
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
2727
|
+
return {
|
|
2728
|
+
passed: false,
|
|
2729
|
+
details: `HTTP ${res.statusCode} \u2014 server accepted request with untrusted Origin header (spec: MUST validate Origin for DNS rebinding protection)`
|
|
2730
|
+
};
|
|
2731
|
+
}
|
|
2732
|
+
if (res.statusCode >= 400) {
|
|
2733
|
+
return { passed: true, details: `HTTP ${res.statusCode} (suspicious Origin rejected)` };
|
|
2734
|
+
}
|
|
2735
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
2736
|
+
} catch {
|
|
2737
|
+
return { passed: true, details: "Connection rejected (acceptable)" };
|
|
2738
|
+
}
|
|
2739
|
+
}
|
|
2740
|
+
);
|
|
2741
|
+
async function runInjectionTest(toolName, paramName, payloads, detectPattern, label) {
|
|
2742
|
+
const issues = [];
|
|
2743
|
+
for (const payload of payloads) {
|
|
2744
|
+
try {
|
|
2745
|
+
const res = await rpc("tools/call", { name: toolName, arguments: { [paramName]: payload } });
|
|
2746
|
+
const content = res.body?.result?.content;
|
|
2747
|
+
if (Array.isArray(content)) {
|
|
2748
|
+
const text = content.map((c) => c.text || "").join(" ");
|
|
2749
|
+
if (detectPattern.test(text)) {
|
|
2750
|
+
issues.push(`Payload "${payload}" ${label} (output: ${text.substring(0, 100)})`);
|
|
2751
|
+
}
|
|
2752
|
+
}
|
|
2753
|
+
} catch {
|
|
2754
|
+
}
|
|
2755
|
+
}
|
|
2756
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
2757
|
+
return {
|
|
2758
|
+
passed: true,
|
|
2759
|
+
details: `Tested ${payloads.length} payloads against ${toolName}.${paramName} \u2014 no ${label.split(" ")[0]} detected`
|
|
2760
|
+
};
|
|
2761
|
+
}
|
|
2762
|
+
if (toolNames.length > 0) {
|
|
2763
|
+
const allTools = cachedToolsList ?? [];
|
|
2764
|
+
const toolsWithStringParams = allTools.filter((t) => {
|
|
2765
|
+
const props = t.inputSchema?.properties;
|
|
2766
|
+
if (!props) return false;
|
|
2767
|
+
return Object.values(props).some((p) => p.type === "string");
|
|
2768
|
+
});
|
|
2769
|
+
const injectionTarget = toolsWithStringParams[0] || allTools[0];
|
|
2770
|
+
const targetStringParam = injectionTarget?.inputSchema?.properties ? Object.entries(injectionTarget.inputSchema.properties).find(
|
|
2771
|
+
([_, v]) => v.type === "string"
|
|
2772
|
+
)?.[0] ?? null : null;
|
|
2773
|
+
await test(
|
|
2774
|
+
"security-command-injection",
|
|
2775
|
+
"Resists command injection in tool params",
|
|
2776
|
+
"security",
|
|
2777
|
+
false,
|
|
2778
|
+
"server/tools#calling-tools",
|
|
2779
|
+
async () => {
|
|
2780
|
+
if (!injectionTarget || !targetStringParam)
|
|
2781
|
+
return { passed: true, details: "No tools with string parameters to test" };
|
|
2782
|
+
return runInjectionTest(
|
|
2783
|
+
injectionTarget.name,
|
|
2784
|
+
targetStringParam,
|
|
2785
|
+
INJECTION_PAYLOADS.command,
|
|
2786
|
+
/root:.*:\d+:\d+:.*:\/|uid=\d+\(\w+\)|drwxr|pwned/i,
|
|
2787
|
+
"appears to have executed"
|
|
2788
|
+
);
|
|
2789
|
+
}
|
|
2790
|
+
);
|
|
2791
|
+
await test(
|
|
2792
|
+
"security-sql-injection",
|
|
2793
|
+
"Resists SQL injection in tool params",
|
|
2794
|
+
"security",
|
|
2795
|
+
false,
|
|
2796
|
+
"server/tools#calling-tools",
|
|
2797
|
+
async () => {
|
|
2798
|
+
if (!injectionTarget || !targetStringParam)
|
|
2799
|
+
return { passed: true, details: "No tools with string parameters to test" };
|
|
2800
|
+
return runInjectionTest(
|
|
2801
|
+
injectionTarget.name,
|
|
2802
|
+
targetStringParam,
|
|
2803
|
+
INJECTION_PAYLOADS.sql,
|
|
2804
|
+
/syntax error|sql|mysql|postgres|sqlite|information_schema|table_name/i,
|
|
2805
|
+
"triggered database error"
|
|
2806
|
+
);
|
|
2807
|
+
}
|
|
2808
|
+
);
|
|
2809
|
+
await test(
|
|
2810
|
+
"security-path-traversal",
|
|
2811
|
+
"Resists path traversal in tool params",
|
|
2812
|
+
"security",
|
|
2813
|
+
false,
|
|
2814
|
+
"server/tools#calling-tools",
|
|
2815
|
+
async () => {
|
|
2816
|
+
if (!injectionTarget || !targetStringParam)
|
|
2817
|
+
return { passed: true, details: "No tools with string parameters to test" };
|
|
2818
|
+
return runInjectionTest(
|
|
2819
|
+
injectionTarget.name,
|
|
2820
|
+
targetStringParam,
|
|
2821
|
+
INJECTION_PAYLOADS.pathTraversal,
|
|
2822
|
+
/root:.*:0:0|\[boot loader\]|\[extensions\]/i,
|
|
2823
|
+
"returned sensitive file content"
|
|
2824
|
+
);
|
|
2825
|
+
}
|
|
2826
|
+
);
|
|
2827
|
+
await test(
|
|
2828
|
+
"security-ssrf-internal",
|
|
2829
|
+
"Resists SSRF to internal networks",
|
|
2830
|
+
"security",
|
|
2831
|
+
false,
|
|
2832
|
+
"server/tools#calling-tools",
|
|
2833
|
+
async () => {
|
|
2834
|
+
const urlParamTool = allTools.find((t) => {
|
|
2835
|
+
const props = t.inputSchema?.properties;
|
|
2836
|
+
if (!props) return false;
|
|
2837
|
+
return Object.entries(props).some(
|
|
2838
|
+
([k, v]) => v.type === "string" && /url|uri|endpoint|link|href/i.test(k)
|
|
2839
|
+
);
|
|
2840
|
+
});
|
|
2841
|
+
if (!urlParamTool) return { passed: true, details: "No tools with URL parameters found (skipped)" };
|
|
2842
|
+
const urlParam = Object.entries(urlParamTool.inputSchema.properties).find(
|
|
2843
|
+
([k, v]) => v.type === "string" && /url|uri|endpoint|link|href/i.test(k)
|
|
2844
|
+
)?.[0];
|
|
2845
|
+
if (!urlParam) return { passed: true, details: "No URL parameter found" };
|
|
2846
|
+
return runInjectionTest(
|
|
2847
|
+
urlParamTool.name,
|
|
2848
|
+
urlParam,
|
|
2849
|
+
INJECTION_PAYLOADS.ssrf,
|
|
2850
|
+
/ami-|instance-id|hostname|iam|security-credentials/i,
|
|
2851
|
+
"returned internal data"
|
|
2852
|
+
);
|
|
2853
|
+
}
|
|
2854
|
+
);
|
|
2855
|
+
} else {
|
|
2856
|
+
for (const testId of [
|
|
2857
|
+
"security-command-injection",
|
|
2858
|
+
"security-sql-injection",
|
|
2859
|
+
"security-path-traversal",
|
|
2860
|
+
"security-ssrf-internal"
|
|
2861
|
+
]) {
|
|
2862
|
+
await test(
|
|
2863
|
+
testId,
|
|
2864
|
+
TEST_DEFINITIONS_MAP.get(testId)?.name || testId,
|
|
2865
|
+
"security",
|
|
2866
|
+
false,
|
|
2867
|
+
"server/tools#calling-tools",
|
|
2868
|
+
async () => ({ passed: true, details: "No tools available to test (skipped)" })
|
|
2869
|
+
);
|
|
2870
|
+
}
|
|
2871
|
+
}
|
|
2872
|
+
await test(
|
|
2873
|
+
"security-oversized-input",
|
|
2874
|
+
"Handles oversized inputs gracefully",
|
|
2875
|
+
"security",
|
|
2876
|
+
false,
|
|
2877
|
+
"server/tools#calling-tools",
|
|
2878
|
+
async () => {
|
|
2879
|
+
const largeValue = "A".repeat(1048576);
|
|
2880
|
+
try {
|
|
2881
|
+
const res = await request(backendUrl, {
|
|
2882
|
+
method: "POST",
|
|
2883
|
+
headers: {
|
|
2884
|
+
"Content-Type": "application/json",
|
|
2885
|
+
Accept: "application/json, text/event-stream",
|
|
2886
|
+
...buildHeaders()
|
|
2887
|
+
},
|
|
2888
|
+
body: JSON.stringify({
|
|
2889
|
+
jsonrpc: "2.0",
|
|
2890
|
+
id: nextId(),
|
|
2891
|
+
method: "tools/call",
|
|
2892
|
+
params: { name: toolNames[0] || "test", arguments: { data: largeValue } }
|
|
2893
|
+
}),
|
|
2894
|
+
signal: AbortSignal.timeout(timeout)
|
|
2895
|
+
});
|
|
2896
|
+
await res.body.text();
|
|
2897
|
+
if (res.statusCode === 413) {
|
|
2898
|
+
return { passed: true, details: "HTTP 413 Payload Too Large (good)" };
|
|
2899
|
+
}
|
|
2900
|
+
if (res.statusCode >= 400) {
|
|
2901
|
+
return { passed: true, details: `HTTP ${res.statusCode} (oversized input rejected)` };
|
|
2902
|
+
}
|
|
2903
|
+
return { passed: true, details: `HTTP ${res.statusCode} \u2014 server handled 1MB payload without crashing` };
|
|
2904
|
+
} catch (err) {
|
|
2905
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2906
|
+
if (msg.includes("timeout") || msg.includes("abort")) {
|
|
2907
|
+
return { passed: false, details: "Request timed out \u2014 server may be struggling with oversized input" };
|
|
2908
|
+
}
|
|
2909
|
+
return { passed: true, details: "Connection rejected (acceptable for oversized input)" };
|
|
2910
|
+
}
|
|
2911
|
+
}
|
|
2912
|
+
);
|
|
2913
|
+
await test(
|
|
2914
|
+
"security-extra-params",
|
|
2915
|
+
"Rejects or ignores extra tool params",
|
|
2916
|
+
"security",
|
|
2917
|
+
false,
|
|
2918
|
+
"server/tools#calling-tools",
|
|
2919
|
+
async () => {
|
|
2920
|
+
if (toolNames.length === 0) {
|
|
2921
|
+
return { passed: true, details: "No tools available to test (skipped)" };
|
|
2922
|
+
}
|
|
2923
|
+
try {
|
|
2924
|
+
const res = await rpc("tools/call", {
|
|
2925
|
+
name: toolNames[0],
|
|
2926
|
+
arguments: { __injected_param__: "malicious_value", __proto__: { admin: true } }
|
|
2927
|
+
});
|
|
2928
|
+
const error = res.body?.error;
|
|
2929
|
+
if (error) {
|
|
2930
|
+
return { passed: true, details: `Extra params rejected with error: ${error.code} \u2014 ${error.message}` };
|
|
2931
|
+
}
|
|
2932
|
+
return { passed: true, details: "Server processed request (extra params likely ignored)" };
|
|
2933
|
+
} catch {
|
|
2934
|
+
return { passed: true, details: "Request rejected (acceptable)" };
|
|
2935
|
+
}
|
|
2936
|
+
}
|
|
2937
|
+
);
|
|
2938
|
+
await test(
|
|
2939
|
+
"security-tool-schema-defined",
|
|
2940
|
+
"All tools define inputSchema",
|
|
2941
|
+
"security",
|
|
2942
|
+
false,
|
|
2943
|
+
"server/tools#data-types",
|
|
2944
|
+
async () => {
|
|
2945
|
+
if (!toolsListOk) return { passed: true, details: "Skipped: tools/list not available" };
|
|
2946
|
+
const tools = cachedToolsList ?? [];
|
|
2947
|
+
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
2948
|
+
const missing = tools.filter((t) => !t.inputSchema || t.inputSchema.type !== "object");
|
|
2949
|
+
if (missing.length > 0) {
|
|
2950
|
+
return {
|
|
2951
|
+
passed: false,
|
|
2952
|
+
details: `${missing.length} tool(s) missing inputSchema: ${missing.map((t) => t.name).join(", ")}`
|
|
2953
|
+
};
|
|
2954
|
+
}
|
|
2955
|
+
return { passed: true, details: `All ${tools.length} tool(s) have inputSchema defined` };
|
|
2956
|
+
}
|
|
2957
|
+
);
|
|
2958
|
+
await test(
|
|
2959
|
+
"security-tool-rug-pull",
|
|
2960
|
+
"Tool definitions are stable across calls",
|
|
2961
|
+
"security",
|
|
2962
|
+
false,
|
|
2963
|
+
"server/tools#listing-tools",
|
|
2964
|
+
async () => {
|
|
2965
|
+
if (!toolsListOk) return { passed: true, details: "Skipped: tools/list not available" };
|
|
2966
|
+
try {
|
|
2967
|
+
const res = await rpc("tools/list");
|
|
2968
|
+
const tools2 = res.body?.result?.tools;
|
|
2969
|
+
if (!Array.isArray(tools2)) return { passed: false, details: "Second tools/list call failed" };
|
|
2970
|
+
const tools1 = cachedToolsList ?? [];
|
|
2971
|
+
if (tools1.length !== tools2.length) {
|
|
2972
|
+
return {
|
|
2973
|
+
passed: false,
|
|
2974
|
+
details: `Tool count changed: ${tools1.length} \u2192 ${tools2.length} (possible rug-pull)`
|
|
2975
|
+
};
|
|
2976
|
+
}
|
|
2977
|
+
const names1 = tools1.map((t) => t.name).sort().join(",");
|
|
2978
|
+
const names2 = tools2.map((t) => t.name).sort().join(",");
|
|
2979
|
+
if (names1 !== names2) {
|
|
2980
|
+
return { passed: false, details: "Tool names changed between calls (possible rug-pull)" };
|
|
2981
|
+
}
|
|
2982
|
+
for (const t1 of tools1) {
|
|
2983
|
+
const t2 = tools2.find((t) => t.name === t1.name);
|
|
2984
|
+
if (t2 && t1.description !== t2.description) {
|
|
2985
|
+
return {
|
|
2986
|
+
passed: false,
|
|
2987
|
+
details: `Tool "${t1.name}" description changed between calls (possible rug-pull)`
|
|
2988
|
+
};
|
|
2989
|
+
}
|
|
2990
|
+
}
|
|
2991
|
+
return { passed: true, details: `${tools1.length} tool(s) consistent across 2 calls` };
|
|
2992
|
+
} catch {
|
|
2993
|
+
return { passed: false, details: "Second tools/list call threw an error" };
|
|
2994
|
+
}
|
|
2995
|
+
}
|
|
2996
|
+
);
|
|
2997
|
+
await test(
|
|
2998
|
+
"security-tool-description-poisoning",
|
|
2999
|
+
"Tool descriptions free of injection patterns",
|
|
3000
|
+
"security",
|
|
3001
|
+
false,
|
|
3002
|
+
"server/tools#data-types",
|
|
3003
|
+
async () => {
|
|
3004
|
+
if (!toolsListOk) return { passed: true, details: "Skipped: tools/list not available" };
|
|
3005
|
+
const tools = cachedToolsList ?? [];
|
|
3006
|
+
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
3007
|
+
const suspiciousPatterns = [
|
|
3008
|
+
{ pattern: /ignore\s+(all\s+)?previous/i, label: "ignore previous instructions" },
|
|
3009
|
+
{ pattern: /override\s+(system|instructions|rules)/i, label: "override instructions" },
|
|
3010
|
+
{ pattern: /system\s+prompt/i, label: "system prompt reference" },
|
|
3011
|
+
{ pattern: /you\s+must\s+(always|never)/i, label: "behavioral override" },
|
|
3012
|
+
{ pattern: /do\s+not\s+(tell|inform|mention)/i, label: "concealment instruction" },
|
|
3013
|
+
{ pattern: /\b[A-Za-z0-9+/]{40,}={0,2}\b/, label: "possible Base64-encoded payload" },
|
|
3014
|
+
// Zero-width characters (U+200B, U+200C, U+200D, U+FEFF)
|
|
3015
|
+
{ pattern: /\u200B|\u200C|\u200D|\uFEFF/, label: "hidden Unicode characters" }
|
|
3016
|
+
];
|
|
3017
|
+
const issues = [];
|
|
3018
|
+
for (const tool of tools) {
|
|
3019
|
+
const textsToCheck = [
|
|
3020
|
+
tool.description || "",
|
|
3021
|
+
...tool.inputSchema?.properties ? Object.values(tool.inputSchema.properties).map((p) => p.description || "") : []
|
|
3022
|
+
];
|
|
3023
|
+
const combined = textsToCheck.join(" ");
|
|
3024
|
+
for (const { pattern, label } of suspiciousPatterns) {
|
|
3025
|
+
if (pattern.test(combined)) {
|
|
3026
|
+
issues.push(`Tool "${tool.name}": ${label}`);
|
|
3027
|
+
}
|
|
3028
|
+
}
|
|
3029
|
+
}
|
|
3030
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
3031
|
+
return { passed: true, details: `${tools.length} tool(s) scanned \u2014 no injection patterns found` };
|
|
3032
|
+
}
|
|
3033
|
+
);
|
|
3034
|
+
await test(
|
|
3035
|
+
"security-tool-cross-reference",
|
|
3036
|
+
"Tools do not reference other tools by name",
|
|
3037
|
+
"security",
|
|
3038
|
+
false,
|
|
3039
|
+
"server/tools#data-types",
|
|
3040
|
+
async () => {
|
|
3041
|
+
if (!toolsListOk) return { passed: true, details: "Skipped: tools/list not available" };
|
|
3042
|
+
const tools = cachedToolsList ?? [];
|
|
3043
|
+
if (tools.length < 2)
|
|
3044
|
+
return { passed: true, details: "Fewer than 2 tools \u2014 cross-reference check not applicable" };
|
|
3045
|
+
const names = tools.map((t) => t.name).filter(Boolean);
|
|
3046
|
+
const issues = [];
|
|
3047
|
+
for (const tool of tools) {
|
|
3048
|
+
const desc = (tool.description || "").toLowerCase();
|
|
3049
|
+
for (const otherName of names) {
|
|
3050
|
+
if (otherName === tool.name) continue;
|
|
3051
|
+
if (desc.includes(otherName.toLowerCase())) {
|
|
3052
|
+
issues.push(`Tool "${tool.name}" description references "${otherName}"`);
|
|
3053
|
+
}
|
|
3054
|
+
}
|
|
3055
|
+
}
|
|
3056
|
+
if (issues.length > 0) {
|
|
3057
|
+
warnings.push(`Cross-tool references found: ${issues.join("; ")}`);
|
|
3058
|
+
return { passed: false, details: issues.join("; ") };
|
|
3059
|
+
}
|
|
3060
|
+
return { passed: true, details: `${tools.length} tool(s) checked \u2014 no cross-references found` };
|
|
3061
|
+
}
|
|
3062
|
+
);
|
|
3063
|
+
await test(
|
|
3064
|
+
"security-error-no-stacktrace",
|
|
3065
|
+
"Error responses do not leak stack traces",
|
|
3066
|
+
"security",
|
|
3067
|
+
false,
|
|
3068
|
+
"basic",
|
|
3069
|
+
async () => {
|
|
3070
|
+
const errorResponses = [];
|
|
3071
|
+
const errorPayloads = [
|
|
3072
|
+
"{this is not valid json!!!",
|
|
3073
|
+
JSON.stringify({ jsonrpc: "2.0", id: nextId(), method: "nonexistent/___crash___test___" }),
|
|
3074
|
+
JSON.stringify({
|
|
3075
|
+
jsonrpc: "2.0",
|
|
3076
|
+
id: nextId(),
|
|
3077
|
+
method: "tools/call",
|
|
3078
|
+
params: { name: "___nonexistent___tool___" }
|
|
3079
|
+
})
|
|
3080
|
+
];
|
|
3081
|
+
for (const payload of errorPayloads) {
|
|
3082
|
+
try {
|
|
3083
|
+
const res = await request(backendUrl, {
|
|
3084
|
+
method: "POST",
|
|
3085
|
+
headers: {
|
|
3086
|
+
"Content-Type": "application/json",
|
|
3087
|
+
Accept: "application/json, text/event-stream",
|
|
3088
|
+
...buildHeaders()
|
|
3089
|
+
},
|
|
3090
|
+
body: payload,
|
|
3091
|
+
signal: AbortSignal.timeout(timeout)
|
|
3092
|
+
});
|
|
3093
|
+
const text = await res.body.text();
|
|
3094
|
+
errorResponses.push(text);
|
|
3095
|
+
} catch {
|
|
3096
|
+
}
|
|
3097
|
+
}
|
|
3098
|
+
const issues = [];
|
|
3099
|
+
for (const text of errorResponses) {
|
|
3100
|
+
for (const pattern of STACK_TRACE_PATTERNS) {
|
|
3101
|
+
if (pattern.test(text)) {
|
|
3102
|
+
issues.push(`Response contains: ${pattern.source} (matched in: ${text.substring(0, 80)}...)`);
|
|
3103
|
+
break;
|
|
3104
|
+
}
|
|
3105
|
+
}
|
|
3106
|
+
}
|
|
3107
|
+
if (issues.length > 0) return { passed: false, details: issues.slice(0, 3).join("; ") };
|
|
3108
|
+
return {
|
|
3109
|
+
passed: true,
|
|
3110
|
+
details: `${errorResponses.length} error responses checked \u2014 no stack traces or sensitive data found`
|
|
3111
|
+
};
|
|
3112
|
+
}
|
|
3113
|
+
);
|
|
3114
|
+
await test(
|
|
3115
|
+
"security-error-no-internal-ip",
|
|
3116
|
+
"Error responses do not leak internal IPs",
|
|
3117
|
+
"security",
|
|
3118
|
+
false,
|
|
3119
|
+
"basic",
|
|
3120
|
+
async () => {
|
|
3121
|
+
try {
|
|
3122
|
+
const res = await request(backendUrl, {
|
|
3123
|
+
method: "POST",
|
|
3124
|
+
headers: {
|
|
3125
|
+
"Content-Type": "application/json",
|
|
3126
|
+
Accept: "application/json, text/event-stream",
|
|
3127
|
+
...buildHeaders()
|
|
3128
|
+
},
|
|
3129
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: nextId(), method: "___trigger_error___" }),
|
|
3130
|
+
signal: AbortSignal.timeout(timeout)
|
|
3131
|
+
});
|
|
3132
|
+
const text = await res.body.text();
|
|
3133
|
+
for (const pattern of INTERNAL_IP_PATTERNS) {
|
|
3134
|
+
const match = text.match(pattern);
|
|
3135
|
+
if (match) {
|
|
3136
|
+
return { passed: false, details: `Error response contains internal IP: ${match[0]}` };
|
|
3137
|
+
}
|
|
3138
|
+
}
|
|
3139
|
+
return { passed: true, details: "No internal IP addresses found in error responses" };
|
|
3140
|
+
} catch {
|
|
3141
|
+
return { passed: true, details: "No response to check (connection error)" };
|
|
3142
|
+
}
|
|
3143
|
+
}
|
|
3144
|
+
);
|
|
3145
|
+
await test(
|
|
3146
|
+
"security-rate-limiting",
|
|
3147
|
+
"Rate limiting is enforced",
|
|
3148
|
+
"security",
|
|
3149
|
+
false,
|
|
3150
|
+
"basic/transports#streamable-http",
|
|
3151
|
+
async () => {
|
|
3152
|
+
const burstSize = 50;
|
|
3153
|
+
let got429 = false;
|
|
3154
|
+
const promises = Array.from(
|
|
3155
|
+
{ length: burstSize },
|
|
3156
|
+
() => mcpRequest(backendUrl, "ping", void 0, nextId, buildHeaders(), timeout).then((res) => {
|
|
3157
|
+
if (res.statusCode === 429) got429 = true;
|
|
3158
|
+
return res.statusCode;
|
|
3159
|
+
}).catch(() => 0)
|
|
3160
|
+
);
|
|
3161
|
+
const statusCodes = await Promise.all(promises);
|
|
3162
|
+
if (got429) {
|
|
3163
|
+
return { passed: true, details: `Rate limiting detected (429 returned after ${burstSize} rapid requests)` };
|
|
3164
|
+
}
|
|
3165
|
+
const errorCount = statusCodes.filter((c) => c >= 500).length;
|
|
3166
|
+
if (errorCount > burstSize / 2) {
|
|
3167
|
+
return {
|
|
3168
|
+
passed: false,
|
|
3169
|
+
details: `Server returned ${errorCount}/${burstSize} 5xx errors under load \u2014 should return 429 instead of crashing`
|
|
3170
|
+
};
|
|
3171
|
+
}
|
|
3172
|
+
return {
|
|
3173
|
+
passed: false,
|
|
3174
|
+
details: `No rate limiting detected (${burstSize} rapid requests all returned ${[...new Set(statusCodes)].join(",")})`
|
|
3175
|
+
};
|
|
3176
|
+
}
|
|
3177
|
+
);
|
|
1829
3178
|
await test(
|
|
1830
3179
|
"transport-delete",
|
|
1831
3180
|
"DELETE accepted or returns 405",
|
|
@@ -1909,13 +3258,13 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1909
3258
|
function registerTools(server) {
|
|
1910
3259
|
server.tool(
|
|
1911
3260
|
"mcp_compliance_test",
|
|
1912
|
-
"Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all
|
|
3261
|
+
"Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all 78 tests covering transport, lifecycle, tools, resources, prompts, errors, schema validation, and security.",
|
|
1913
3262
|
{
|
|
1914
3263
|
url: z.string().url().describe("The MCP server URL to test (must be HTTP or HTTPS)"),
|
|
1915
3264
|
auth: z.string().optional().describe('Authorization header value (e.g., "Bearer tok123")'),
|
|
1916
3265
|
headers: z.record(z.string()).optional().describe('Additional headers to include on all requests (e.g., {"X-Api-Key": "abc"})'),
|
|
1917
|
-
timeout: z.number().optional().describe("Request timeout in milliseconds (default: 15000)"),
|
|
1918
|
-
retries: z.number().optional().describe("Number of retries for failed tests (default: 0)"),
|
|
3266
|
+
timeout: z.number().int().min(1).max(3e5).optional().describe("Request timeout in milliseconds (default: 15000, max: 300000)"),
|
|
3267
|
+
retries: z.number().int().min(0).max(10).optional().describe("Number of retries for failed tests (default: 0, max: 10)"),
|
|
1919
3268
|
only: z.array(z.string()).optional().describe("Only run tests matching these categories or test IDs"),
|
|
1920
3269
|
skip: z.array(z.string()).optional().describe("Skip tests matching these categories or test IDs")
|
|
1921
3270
|
},
|
|
@@ -1980,7 +3329,7 @@ ${JSON.stringify(report, null, 2)}` }
|
|
|
1980
3329
|
url: z.string().url().describe("The MCP server URL to test"),
|
|
1981
3330
|
auth: z.string().optional().describe('Authorization header value (e.g., "Bearer tok123")'),
|
|
1982
3331
|
headers: z.record(z.string()).optional().describe("Additional headers to include on all requests"),
|
|
1983
|
-
timeout: z.number().optional().describe("Request timeout in milliseconds (default: 15000)")
|
|
3332
|
+
timeout: z.number().int().min(1).max(3e5).optional().describe("Request timeout in milliseconds (default: 15000, max: 300000)")
|
|
1984
3333
|
},
|
|
1985
3334
|
{
|
|
1986
3335
|
title: "Get Compliance Badge",
|
|
@@ -2104,9 +3453,10 @@ var CATEGORY_LABELS = {
|
|
|
2104
3453
|
resources: "Resources",
|
|
2105
3454
|
prompts: "Prompts",
|
|
2106
3455
|
errors: "Error Handling",
|
|
2107
|
-
schema: "Schema Validation"
|
|
3456
|
+
schema: "Schema Validation",
|
|
3457
|
+
security: "Security"
|
|
2108
3458
|
};
|
|
2109
|
-
var CATEGORY_ORDER = ["transport", "lifecycle", "tools", "resources", "prompts", "errors", "schema"];
|
|
3459
|
+
var CATEGORY_ORDER = ["transport", "lifecycle", "tools", "resources", "prompts", "errors", "schema", "security"];
|
|
2110
3460
|
function gradeColor(grade) {
|
|
2111
3461
|
switch (grade) {
|
|
2112
3462
|
case "A":
|
|
@@ -2330,7 +3680,9 @@ function parseList(value) {
|
|
|
2330
3680
|
}
|
|
2331
3681
|
var program = new Command();
|
|
2332
3682
|
program.name("mcp-compliance").description("Test MCP servers for spec compliance").version(version2);
|
|
2333
|
-
program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").
|
|
3683
|
+
program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").addOption(
|
|
3684
|
+
new Option("--format <format>", "Output format").choices(["terminal", "json", "sarif"]).default("terminal")
|
|
3685
|
+
).option("--strict", "Exit with code 1 on any required test failure (for CI)").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").option("--retries <n>", "Number of retries for failed tests", "0").option(
|
|
2334
3686
|
"--only <items>",
|
|
2335
3687
|
'Only run matching categories or test IDs, comma-separated (e.g., "transport,lifecycle" or "transport-post,lifecycle-init")',
|
|
2336
3688
|
parseList
|