@yawlabs/mcp-compliance 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -7
- package/dist/{chunk-KNOSZ3TD.js → chunk-QLFYT4I7.js} +1277 -62
- package/dist/index.js +1342 -105
- package/dist/mcp/server.js +8 -5
- package/dist/runner.d.ts +13 -3
- package/dist/runner.js +7 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -18,16 +18,14 @@ import { createRequire } from "module";
|
|
|
18
18
|
import { request } from "undici";
|
|
19
19
|
|
|
20
20
|
// src/badge.ts
|
|
21
|
+
import { createHash } from "crypto";
|
|
22
|
+
function urlHash(url) {
|
|
23
|
+
return createHash("sha256").update(url).digest("hex").slice(0, 12);
|
|
24
|
+
}
|
|
21
25
|
function generateBadge(url) {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
} catch {
|
|
26
|
-
parsed = new URL("https://unknown");
|
|
27
|
-
}
|
|
28
|
-
const encoded = encodeURIComponent(parsed.href);
|
|
29
|
-
const imageUrl = `https://mcp.hosting/api/compliance/${encoded}/badge`;
|
|
30
|
-
const reportUrl = `https://mcp.hosting/compliance/${encoded}`;
|
|
26
|
+
const hash = urlHash(url);
|
|
27
|
+
const imageUrl = `https://mcp.hosting/api/compliance/${hash}/badge`;
|
|
28
|
+
const reportUrl = `https://mcp.hosting/compliance/${hash}`;
|
|
31
29
|
return {
|
|
32
30
|
imageUrl,
|
|
33
31
|
reportUrl,
|
|
@@ -73,7 +71,7 @@ function computeScore(tests) {
|
|
|
73
71
|
|
|
74
72
|
// src/types.ts
|
|
75
73
|
var TEST_DEFINITIONS = [
|
|
76
|
-
// ── Transport (
|
|
74
|
+
// ── Transport (10 tests) ─────────────────────────────────────────
|
|
77
75
|
{
|
|
78
76
|
id: "transport-post",
|
|
79
77
|
name: "HTTP POST accepted",
|
|
@@ -137,7 +135,34 @@ var TEST_DEFINITIONS = [
|
|
|
137
135
|
description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests.",
|
|
138
136
|
recommendation: "Check if the parsed JSON body is an array. If so, return a JSON-RPC error or HTTP 400. Do not process batch requests \u2014 MCP explicitly forbids them."
|
|
139
137
|
},
|
|
140
|
-
|
|
138
|
+
{
|
|
139
|
+
id: "transport-content-type-init",
|
|
140
|
+
name: "Initialize response has valid content type",
|
|
141
|
+
category: "transport",
|
|
142
|
+
required: false,
|
|
143
|
+
specRef: "basic/transports#streamable-http",
|
|
144
|
+
description: "Validates that the initialize response uses application/json or text/event-stream content type. Some servers return other types for the handshake.",
|
|
145
|
+
recommendation: 'Ensure the initialize response uses Content-Type "application/json" or "text/event-stream". Do not return text/html or other types for JSON-RPC responses.'
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
id: "transport-get-stream",
|
|
149
|
+
name: "GET with session returns SSE or 405",
|
|
150
|
+
category: "transport",
|
|
151
|
+
required: false,
|
|
152
|
+
specRef: "basic/transports#streamable-http",
|
|
153
|
+
description: "Tests the GET endpoint with an active session ID for server-initiated messages. After initialization, the server should either return an SSE stream or 405.",
|
|
154
|
+
recommendation: "If your server supports server-initiated messages, return text/event-stream on GET with a valid session ID. Otherwise, return 405 Method Not Allowed."
|
|
155
|
+
},
|
|
156
|
+
{
|
|
157
|
+
id: "transport-concurrent",
|
|
158
|
+
name: "Handles concurrent requests",
|
|
159
|
+
category: "transport",
|
|
160
|
+
required: false,
|
|
161
|
+
specRef: "basic/transports#streamable-http",
|
|
162
|
+
description: "Sends multiple JSON-RPC requests in parallel and verifies the server responds to all with correct matching IDs. Tests that the server can handle concurrent connections.",
|
|
163
|
+
recommendation: "Ensure your server can handle multiple simultaneous requests. Each response must include the correct id matching the request. Use async handlers or connection pooling."
|
|
164
|
+
},
|
|
165
|
+
// ── Lifecycle (12 tests) ─────────────────────────────────────────
|
|
141
166
|
{
|
|
142
167
|
id: "lifecycle-init",
|
|
143
168
|
name: "Initialize handshake",
|
|
@@ -228,6 +253,24 @@ var TEST_DEFINITIONS = [
|
|
|
228
253
|
description: "If the server declares completions capability, tests that the completion/complete method is accepted.",
|
|
229
254
|
recommendation: 'If you declare completions in capabilities, implement the "completion/complete" handler. Return a completion object with a values array, even if empty.'
|
|
230
255
|
},
|
|
256
|
+
{
|
|
257
|
+
id: "lifecycle-cancellation",
|
|
258
|
+
name: "Handles cancellation notifications",
|
|
259
|
+
category: "lifecycle",
|
|
260
|
+
required: false,
|
|
261
|
+
specRef: "basic/utilities#cancellation",
|
|
262
|
+
description: "Tests that the server accepts notifications/cancelled without error. Servers should gracefully handle cancellation of unknown or completed requests.",
|
|
263
|
+
recommendation: "Accept notifications/cancelled and stop any in-progress work for the referenced requestId. If the request is unknown or already complete, silently ignore the cancellation."
|
|
264
|
+
},
|
|
265
|
+
{
|
|
266
|
+
id: "lifecycle-progress",
|
|
267
|
+
name: "Accepts progress notifications",
|
|
268
|
+
category: "lifecycle",
|
|
269
|
+
required: false,
|
|
270
|
+
specRef: "basic/utilities#progress",
|
|
271
|
+
description: "Tests that the server accepts notifications/progress without error. Servers should handle progress notifications for request tracking.",
|
|
272
|
+
recommendation: "Accept notifications/progress with progressToken, progress, and optional total fields. Ignore notifications for unknown progress tokens."
|
|
273
|
+
},
|
|
231
274
|
// ── Tools (4 tests) ──────────────────────────────────────────────
|
|
232
275
|
{
|
|
233
276
|
id: "tools-list",
|
|
@@ -466,15 +509,259 @@ var TEST_DEFINITIONS = [
|
|
|
466
509
|
specRef: "server/resources#data-types",
|
|
467
510
|
description: "Validates every resource has a valid URI (parseable as a URL) and a name field.",
|
|
468
511
|
recommendation: "Ensure every resource has a valid, parseable URI and a name field. Add description and mimeType for better client integration."
|
|
512
|
+
},
|
|
513
|
+
// ── Security: Auth & Transport (8 tests) ─────────────────────────
|
|
514
|
+
{
|
|
515
|
+
id: "security-auth-required",
|
|
516
|
+
name: "Rejects unauthenticated requests",
|
|
517
|
+
category: "security",
|
|
518
|
+
required: false,
|
|
519
|
+
specRef: "basic/authorization",
|
|
520
|
+
description: "Sends a request without an Authorization header and verifies the server returns HTTP 401. Servers exposed over the network should require authentication.",
|
|
521
|
+
recommendation: "Implement authentication on your MCP endpoint. Return HTTP 401 Unauthorized for requests without valid credentials. Use OAuth 2.1 or Bearer tokens as recommended by the MCP spec."
|
|
522
|
+
},
|
|
523
|
+
{
|
|
524
|
+
id: "security-auth-malformed",
|
|
525
|
+
name: "Rejects malformed auth credentials",
|
|
526
|
+
category: "security",
|
|
527
|
+
required: false,
|
|
528
|
+
specRef: "basic/authorization",
|
|
529
|
+
description: "Sends a request with a malformed Authorization header (garbage value) and verifies the server returns HTTP 401 or 403. Servers must validate auth tokens, not just check for presence.",
|
|
530
|
+
recommendation: "Validate the format and signature of Authorization header values. Reject malformed or invalid tokens with HTTP 401. Do not treat any non-empty Authorization header as valid."
|
|
531
|
+
},
|
|
532
|
+
{
|
|
533
|
+
id: "security-tls-required",
|
|
534
|
+
name: "Enforces HTTPS/TLS",
|
|
535
|
+
category: "security",
|
|
536
|
+
required: false,
|
|
537
|
+
specRef: "basic/authorization",
|
|
538
|
+
description: "If the server URL uses HTTPS, attempts an HTTP (plaintext) connection and verifies it is rejected or redirected. Production MCP servers should not accept plaintext connections.",
|
|
539
|
+
recommendation: "Configure your server to reject HTTP connections or redirect to HTTPS. Use TLS 1.2 or higher. The MCP spec requires HTTPS for production deployments."
|
|
540
|
+
},
|
|
541
|
+
{
|
|
542
|
+
id: "security-session-entropy",
|
|
543
|
+
name: "Session IDs are high-entropy",
|
|
544
|
+
category: "security",
|
|
545
|
+
required: false,
|
|
546
|
+
specRef: "basic/transports#streamable-http",
|
|
547
|
+
description: "Analyzes the MCP-Session-Id returned by the server. Session IDs should be cryptographically random and not sequential or predictable.",
|
|
548
|
+
recommendation: "Generate session IDs using a cryptographically secure random source (e.g., crypto.randomUUID()). Session IDs should be at least 128 bits of entropy. Do not use sequential counters or timestamps."
|
|
549
|
+
},
|
|
550
|
+
{
|
|
551
|
+
id: "security-session-not-auth",
|
|
552
|
+
name: "Session ID does not bypass auth",
|
|
553
|
+
category: "security",
|
|
554
|
+
required: false,
|
|
555
|
+
specRef: "basic/transports#streamable-http",
|
|
556
|
+
description: "Verifies that presenting a valid MCP-Session-Id without an Authorization header is still rejected. Per spec, servers MUST NOT use sessions for authentication.",
|
|
557
|
+
recommendation: "Always validate the Authorization header independently of the MCP-Session-Id. Sessions are for request routing, not authentication. Reject requests that lack valid auth even if they have a valid session ID."
|
|
558
|
+
},
|
|
559
|
+
{
|
|
560
|
+
id: "security-oauth-metadata",
|
|
561
|
+
name: "OAuth metadata endpoint exists",
|
|
562
|
+
category: "security",
|
|
563
|
+
required: false,
|
|
564
|
+
specRef: "basic/authorization",
|
|
565
|
+
description: "Checks for a well-known OAuth authorization server metadata endpoint at /.well-known/oauth-authorization-server. If the server requires auth, it should advertise how to obtain tokens.",
|
|
566
|
+
recommendation: "Publish an OAuth 2.0 Authorization Server Metadata document at /.well-known/oauth-authorization-server on your server's origin. Include issuer, token_endpoint, and supported grant types."
|
|
567
|
+
},
|
|
568
|
+
{
|
|
569
|
+
id: "security-token-in-uri",
|
|
570
|
+
name: "Rejects auth tokens in query string",
|
|
571
|
+
category: "security",
|
|
572
|
+
required: false,
|
|
573
|
+
specRef: "basic/authorization",
|
|
574
|
+
description: "Sends a request with the auth token in the URL query string instead of the Authorization header. The MCP spec forbids transmitting credentials in URIs.",
|
|
575
|
+
recommendation: "Never accept authentication tokens from URL query parameters. Tokens in URIs are logged by proxies, appear in browser history, and leak via the Referer header. Only accept tokens in the Authorization header."
|
|
576
|
+
},
|
|
577
|
+
{
|
|
578
|
+
id: "security-cors-headers",
|
|
579
|
+
name: "CORS headers are restrictive",
|
|
580
|
+
category: "security",
|
|
581
|
+
required: false,
|
|
582
|
+
specRef: "basic/transports#streamable-http",
|
|
583
|
+
description: "If the server returns CORS headers, verifies that Access-Control-Allow-Origin is not set to wildcard (*). Wildcard CORS on an authenticated API allows cross-origin credential theft.",
|
|
584
|
+
recommendation: 'Set Access-Control-Allow-Origin to specific trusted origins, not "*". If CORS is not needed (server-to-server only), do not send CORS headers at all.'
|
|
585
|
+
},
|
|
586
|
+
// ── Security: Input Validation (6 tests) ─────────────────────────
|
|
587
|
+
{
|
|
588
|
+
id: "security-command-injection",
|
|
589
|
+
name: "Resists command injection in tool params",
|
|
590
|
+
category: "security",
|
|
591
|
+
required: false,
|
|
592
|
+
specRef: "server/tools#calling-tools",
|
|
593
|
+
description: "Calls each tool with OS command injection payloads in string parameters (e.g., '; cat /etc/passwd', '$(whoami)'). Verifies the server does not execute injected commands.",
|
|
594
|
+
recommendation: "Never pass tool argument values directly to shell commands. Use parameterized APIs, execFile() instead of exec(), or allowlists. Sanitize all user-provided input before use in system calls."
|
|
595
|
+
},
|
|
596
|
+
{
|
|
597
|
+
id: "security-sql-injection",
|
|
598
|
+
name: "Resists SQL injection in tool params",
|
|
599
|
+
category: "security",
|
|
600
|
+
required: false,
|
|
601
|
+
specRef: "server/tools#calling-tools",
|
|
602
|
+
description: `Calls each tool with SQL injection payloads in string parameters (e.g., "' OR 1=1 --"). Verifies the server does not return database errors or unexpected data.`,
|
|
603
|
+
recommendation: "Use parameterized queries or prepared statements for all database operations. Never concatenate user input into SQL strings. Return generic error messages that do not reveal database structure."
|
|
604
|
+
},
|
|
605
|
+
{
|
|
606
|
+
id: "security-path-traversal",
|
|
607
|
+
name: "Resists path traversal in tool params",
|
|
608
|
+
category: "security",
|
|
609
|
+
required: false,
|
|
610
|
+
specRef: "server/tools#calling-tools",
|
|
611
|
+
description: "Calls each tool with path traversal payloads in string parameters (e.g., '../../etc/passwd', '..\\\\..\\\\windows\\\\system.ini'). Verifies the server does not expose files outside its intended scope.",
|
|
612
|
+
recommendation: "Validate and sanitize file paths. Use path.resolve() and verify the result is within the allowed directory. Reject paths containing '..' segments. Use a chroot or sandboxed filesystem for file operations."
|
|
613
|
+
},
|
|
614
|
+
{
|
|
615
|
+
id: "security-ssrf-internal",
|
|
616
|
+
name: "Resists SSRF to internal networks",
|
|
617
|
+
category: "security",
|
|
618
|
+
required: false,
|
|
619
|
+
specRef: "server/tools#calling-tools",
|
|
620
|
+
description: "For tools that accept URL parameters, submits internal IP addresses (169.254.169.254, 127.0.0.1, 10.0.0.0/8) and cloud metadata endpoints. Verifies the server blocks requests to internal networks.",
|
|
621
|
+
recommendation: "Validate and restrict URLs in tool parameters. Block requests to private IP ranges (10.x, 172.16-31.x, 192.168.x, 127.x, 169.254.x), link-local addresses, and cloud metadata endpoints. Use an allowlist of permitted domains."
|
|
622
|
+
},
|
|
623
|
+
{
|
|
624
|
+
id: "security-oversized-input",
|
|
625
|
+
name: "Handles oversized inputs gracefully",
|
|
626
|
+
category: "security",
|
|
627
|
+
required: false,
|
|
628
|
+
specRef: "server/tools#calling-tools",
|
|
629
|
+
description: "Sends a tools/call request with an extremely large argument value (1MB+ string). Verifies the server rejects it with an error instead of crashing or consuming excessive resources.",
|
|
630
|
+
recommendation: "Implement request body size limits. Return HTTP 413 or JSON-RPC error for oversized payloads. Set explicit maxBodyLength in your HTTP server configuration."
|
|
631
|
+
},
|
|
632
|
+
{
|
|
633
|
+
id: "security-extra-params",
|
|
634
|
+
name: "Rejects or ignores extra tool params",
|
|
635
|
+
category: "security",
|
|
636
|
+
required: false,
|
|
637
|
+
specRef: "server/tools#calling-tools",
|
|
638
|
+
description: "Calls a tool with unexpected additional parameters beyond what the schema defines. Verifies the server either rejects them (strict) or silently ignores them (permissive) without errors.",
|
|
639
|
+
recommendation: "Use JSON Schema validation with additionalProperties: false to reject unexpected parameters, or strip unknown properties before processing. Do not pass unvalidated properties to internal functions."
|
|
640
|
+
},
|
|
641
|
+
// ── Security: Tool Integrity (4 tests) ───────────────────────────
|
|
642
|
+
{
|
|
643
|
+
id: "security-tool-schema-defined",
|
|
644
|
+
name: "All tools define inputSchema",
|
|
645
|
+
category: "security",
|
|
646
|
+
required: false,
|
|
647
|
+
specRef: "server/tools#data-types",
|
|
648
|
+
description: "Verifies all tools have an inputSchema with type 'object'. Tools without schemas cannot have their inputs validated, creating an injection risk.",
|
|
649
|
+
recommendation: "Define a complete JSON Schema (inputSchema with type: 'object') for every tool. Specify all expected properties, their types, and constraints. This enables input validation and prevents parameter injection."
|
|
650
|
+
},
|
|
651
|
+
{
|
|
652
|
+
id: "security-tool-rug-pull",
|
|
653
|
+
name: "Tool definitions are stable across calls",
|
|
654
|
+
category: "security",
|
|
655
|
+
required: false,
|
|
656
|
+
specRef: "server/tools#listing-tools",
|
|
657
|
+
description: "Calls tools/list twice and compares the results. Tool definitions should not change between calls within the same session, which could indicate a rug-pull attack.",
|
|
658
|
+
recommendation: "Ensure tools/list returns consistent results within a session. If tools change dynamically, send a tools/list_changed notification. Never silently alter tool definitions \u2014 this is a known MCP attack vector (tool poisoning)."
|
|
659
|
+
},
|
|
660
|
+
{
|
|
661
|
+
id: "security-tool-description-poisoning",
|
|
662
|
+
name: "Tool descriptions free of injection patterns",
|
|
663
|
+
category: "security",
|
|
664
|
+
required: false,
|
|
665
|
+
specRef: "server/tools#data-types",
|
|
666
|
+
description: "Scans all tool names, descriptions, and parameter descriptions for prompt injection patterns: 'ignore previous', 'override', 'system prompt', hidden Unicode characters, and Base64-encoded strings.",
|
|
667
|
+
recommendation: "Review all tool descriptions for prompt injection patterns. Remove any text that attempts to override LLM instructions, references system prompts, or contains hidden characters. Tool descriptions are rendered to LLMs and can be used for prompt injection."
|
|
668
|
+
},
|
|
669
|
+
{
|
|
670
|
+
id: "security-tool-cross-reference",
|
|
671
|
+
name: "Tools do not reference other tools by name",
|
|
672
|
+
category: "security",
|
|
673
|
+
required: false,
|
|
674
|
+
specRef: "server/tools#data-types",
|
|
675
|
+
description: "Checks that tool descriptions do not reference other tool names. Cross-references between tools can be used to manipulate LLM tool selection and create implicit execution chains.",
|
|
676
|
+
recommendation: "Avoid referencing other tool names in tool descriptions. Each tool should be self-contained. If tools have dependencies, document them in server instructions, not in individual tool descriptions."
|
|
677
|
+
},
|
|
678
|
+
// ── Security: Information Disclosure (3 tests) ───────────────────
|
|
679
|
+
{
|
|
680
|
+
id: "security-error-no-stacktrace",
|
|
681
|
+
name: "Error responses do not leak stack traces",
|
|
682
|
+
category: "security",
|
|
683
|
+
required: false,
|
|
684
|
+
specRef: "basic",
|
|
685
|
+
description: "Triggers various error conditions and inspects responses for stack traces, file paths, and internal implementation details. Error responses should not reveal server internals.",
|
|
686
|
+
recommendation: "Sanitize error responses before returning them to clients. Remove stack traces, file paths, database connection strings, and internal IP addresses. Use generic error messages for unexpected failures."
|
|
687
|
+
},
|
|
688
|
+
{
|
|
689
|
+
id: "security-error-no-internal-ip",
|
|
690
|
+
name: "Error responses do not leak internal IPs",
|
|
691
|
+
category: "security",
|
|
692
|
+
required: false,
|
|
693
|
+
specRef: "basic",
|
|
694
|
+
description: "Inspects error response bodies for private IP addresses (10.x, 172.16-31.x, 192.168.x, 127.x) that would reveal internal network topology.",
|
|
695
|
+
recommendation: "Strip internal IP addresses from error responses. Configure your reverse proxy to not forward X-Real-IP or internal addressing. Use a centralized error handler that sanitizes responses."
|
|
696
|
+
},
|
|
697
|
+
{
|
|
698
|
+
id: "security-rate-limiting",
|
|
699
|
+
name: "Rate limiting is enforced",
|
|
700
|
+
category: "security",
|
|
701
|
+
required: false,
|
|
702
|
+
specRef: "basic/transports#streamable-http",
|
|
703
|
+
description: "Sends a burst of rapid requests and checks if the server eventually returns HTTP 429 Too Many Requests. Production servers should implement rate limiting to prevent abuse.",
|
|
704
|
+
recommendation: "Implement rate limiting on your MCP endpoint. Return HTTP 429 with a Retry-After header when limits are exceeded. Consider per-IP, per-token, and per-session rate limits."
|
|
469
705
|
}
|
|
470
706
|
];
|
|
471
707
|
|
|
472
708
|
// src/runner.ts
|
|
709
|
+
var TEST_DEFINITIONS_MAP = new Map(TEST_DEFINITIONS.map((t) => [t.id, t]));
|
|
473
710
|
var _require = createRequire(import.meta.url);
|
|
474
711
|
var { version: TOOL_VERSION } = _require("../package.json");
|
|
475
712
|
var SPEC_VERSION = "2025-11-25";
|
|
476
713
|
var SPEC_BASE = `https://modelcontextprotocol.io/specification/${SPEC_VERSION}`;
|
|
477
714
|
var VALID_CONTENT_TYPES = ["text", "image", "audio", "resource", "resource_link"];
|
|
715
|
+
var INJECTION_PAYLOADS = {
|
|
716
|
+
command: ["; cat /etc/passwd", "$(whoami)", "| ls -la", "`id`", "&& echo pwned"],
|
|
717
|
+
sql: ["' OR 1=1 --", "'; DROP TABLE users; --", "1 UNION SELECT * FROM information_schema.tables--"],
|
|
718
|
+
pathTraversal: ["../../etc/passwd", "..\\..\\windows\\system.ini", "%2e%2e%2f%2e%2e%2fetc%2fpasswd"],
|
|
719
|
+
ssrf: ["http://169.254.169.254/latest/meta-data/", "http://127.0.0.1:22", "http://[::1]/", "http://10.0.0.1/"]
|
|
720
|
+
};
|
|
721
|
+
var STACK_TRACE_PATTERNS = [
|
|
722
|
+
/at\s+\S+\s+\(.*:\d+:\d+\)/i,
|
|
723
|
+
// Node.js: "at Function (file.js:10:5)"
|
|
724
|
+
/Traceback\s+\(most recent/i,
|
|
725
|
+
// Python
|
|
726
|
+
/\.py",\s+line\s+\d+/i,
|
|
727
|
+
// Python file reference
|
|
728
|
+
/\.java:\d+\)/i,
|
|
729
|
+
// Java
|
|
730
|
+
/\.go:\d+/i,
|
|
731
|
+
// Go
|
|
732
|
+
/from\s+\S+\.rb:\d+/i,
|
|
733
|
+
// Ruby
|
|
734
|
+
/\.cs:line\s+\d+/i,
|
|
735
|
+
// C#/.NET
|
|
736
|
+
/#\d+\s+\/.*\.php\(\d+\)/i,
|
|
737
|
+
// PHP
|
|
738
|
+
/panicked\s+at\s+'/i,
|
|
739
|
+
// Rust
|
|
740
|
+
/ENOENT|EACCES|EPERM/,
|
|
741
|
+
// Node.js system errors
|
|
742
|
+
/node_modules\//,
|
|
743
|
+
// Node.js module paths
|
|
744
|
+
/\/usr\/local\/|\/home\//,
|
|
745
|
+
// Unix paths
|
|
746
|
+
/[A-Z]:\\.*\\/,
|
|
747
|
+
// Windows paths
|
|
748
|
+
/password|passwd|secret|credential/i,
|
|
749
|
+
// Sensitive terms
|
|
750
|
+
/jdbc:|mysql:|postgres:|mongodb:/i
|
|
751
|
+
// DB connection strings
|
|
752
|
+
];
|
|
753
|
+
var INTERNAL_IP_PATTERNS = [
|
|
754
|
+
/\b10\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/,
|
|
755
|
+
/\b172\.(1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3}\b/,
|
|
756
|
+
/\b192\.168\.\d{1,3}\.\d{1,3}\b/,
|
|
757
|
+
/\b127\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/,
|
|
758
|
+
/\b::1\b/,
|
|
759
|
+
// IPv6 loopback
|
|
760
|
+
/\bfe80:/i,
|
|
761
|
+
// IPv6 link-local
|
|
762
|
+
/\bf[cd][0-9a-f]{2}:/i
|
|
763
|
+
// IPv6 unique local (fc00::/fd00::)
|
|
764
|
+
];
|
|
478
765
|
function createIdCounter(start = 0) {
|
|
479
766
|
let id = start;
|
|
480
767
|
return () => ++id;
|
|
@@ -569,19 +856,39 @@ async function mcpNotification(backendUrl, method, params, extraHeaders, timeout
|
|
|
569
856
|
return { statusCode: res.statusCode, headers: responseHeaders };
|
|
570
857
|
}
|
|
571
858
|
async function runComplianceSuite(url, options = {}) {
|
|
572
|
-
let parsed;
|
|
573
859
|
try {
|
|
574
|
-
parsed = new URL(url);
|
|
860
|
+
const parsed = new URL(url);
|
|
575
861
|
if (!["http:", "https:"].includes(parsed.protocol)) {
|
|
576
862
|
throw new Error("Only HTTP and HTTPS URLs are supported");
|
|
577
863
|
}
|
|
578
864
|
} catch (e) {
|
|
579
|
-
if (e.message.includes("Only HTTP")) throw e;
|
|
865
|
+
if (e instanceof Error && e.message.includes("Only HTTP")) throw e;
|
|
580
866
|
throw new Error(`Invalid URL: ${url}`);
|
|
581
867
|
}
|
|
582
868
|
const backendUrl = url;
|
|
869
|
+
let serverReachable = true;
|
|
870
|
+
try {
|
|
871
|
+
const preflight = await request(backendUrl, {
|
|
872
|
+
method: "POST",
|
|
873
|
+
headers: {
|
|
874
|
+
"Content-Type": "application/json",
|
|
875
|
+
Accept: "application/json, text/event-stream",
|
|
876
|
+
...options.headers
|
|
877
|
+
},
|
|
878
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 0, method: "ping" }),
|
|
879
|
+
signal: AbortSignal.timeout(Math.min(options.timeout || 15e3, 1e4))
|
|
880
|
+
});
|
|
881
|
+
await preflight.body.text();
|
|
882
|
+
} catch {
|
|
883
|
+
serverReachable = false;
|
|
884
|
+
}
|
|
583
885
|
const tests = [];
|
|
584
886
|
const warnings = [];
|
|
887
|
+
if (!serverReachable) {
|
|
888
|
+
warnings.push(
|
|
889
|
+
`Server at ${url} is unreachable \u2014 all tests will fail. Check the URL and ensure the server is running.`
|
|
890
|
+
);
|
|
891
|
+
}
|
|
585
892
|
const nextId = createIdCounter(1e3);
|
|
586
893
|
const timeout = options.timeout || 15e3;
|
|
587
894
|
const retries = options.retries || 0;
|
|
@@ -626,7 +933,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
626
933
|
if (lastResult.passed) break;
|
|
627
934
|
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
|
|
628
935
|
} catch (err) {
|
|
629
|
-
|
|
936
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
937
|
+
lastResult = { passed: false, details: `Error: ${message}` };
|
|
630
938
|
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
|
|
631
939
|
}
|
|
632
940
|
}
|
|
@@ -655,10 +963,23 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
655
963
|
body: JSON.stringify({ jsonrpc: "2.0", id: 99901, method: "ping" }),
|
|
656
964
|
signal: AbortSignal.timeout(timeout)
|
|
657
965
|
});
|
|
658
|
-
await res.body.text();
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
966
|
+
const text = await res.body.text();
|
|
967
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
968
|
+
return { passed: true, details: `HTTP ${res.statusCode}` };
|
|
969
|
+
}
|
|
970
|
+
if (res.statusCode === 401 || res.statusCode === 403) {
|
|
971
|
+
return { passed: false, details: `HTTP ${res.statusCode} (auth required \u2014 pass --auth)` };
|
|
972
|
+
}
|
|
973
|
+
if (res.statusCode === 400) {
|
|
974
|
+
try {
|
|
975
|
+
const body = JSON.parse(text);
|
|
976
|
+
if (body?.error || body?.jsonrpc) {
|
|
977
|
+
return { passed: true, details: "HTTP 400 with JSON-RPC response (server requires initialization first)" };
|
|
978
|
+
}
|
|
979
|
+
} catch {
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
662
983
|
}
|
|
663
984
|
);
|
|
664
985
|
await test(
|
|
@@ -688,18 +1009,29 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
688
1009
|
false,
|
|
689
1010
|
"basic/transports#streamable-http",
|
|
690
1011
|
async () => {
|
|
1012
|
+
const getHeaders = { Accept: "text/event-stream", ...buildHeaders() };
|
|
691
1013
|
const res = await request(backendUrl, {
|
|
692
1014
|
method: "GET",
|
|
693
|
-
headers:
|
|
1015
|
+
headers: getHeaders,
|
|
694
1016
|
signal: AbortSignal.timeout(timeout)
|
|
695
1017
|
});
|
|
696
|
-
await res.body.text();
|
|
1018
|
+
const body = await res.body.text();
|
|
697
1019
|
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
698
1020
|
if (res.statusCode === 405) {
|
|
699
1021
|
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
700
1022
|
}
|
|
701
1023
|
if (ct.includes("text/event-stream")) {
|
|
702
|
-
|
|
1024
|
+
if (body.trim().length > 0) {
|
|
1025
|
+
const hasDataFields = body.includes("data:");
|
|
1026
|
+
const hasEventFields = body.includes("event:");
|
|
1027
|
+
if (!hasDataFields && !hasEventFields) {
|
|
1028
|
+
return {
|
|
1029
|
+
passed: false,
|
|
1030
|
+
details: "Content-Type is text/event-stream but body has no SSE data: or event: fields"
|
|
1031
|
+
};
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
return { passed: true, details: "Returns text/event-stream with valid SSE format" };
|
|
703
1035
|
}
|
|
704
1036
|
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
705
1037
|
return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
|
|
@@ -707,31 +1039,6 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
707
1039
|
return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
|
|
708
1040
|
}
|
|
709
1041
|
);
|
|
710
|
-
await test(
|
|
711
|
-
"transport-delete",
|
|
712
|
-
"DELETE accepted or returns 405",
|
|
713
|
-
"transport",
|
|
714
|
-
false,
|
|
715
|
-
"basic/transports#streamable-http",
|
|
716
|
-
async () => {
|
|
717
|
-
const res = await request(backendUrl, {
|
|
718
|
-
method: "DELETE",
|
|
719
|
-
headers: { ...userHeaders },
|
|
720
|
-
signal: AbortSignal.timeout(timeout)
|
|
721
|
-
});
|
|
722
|
-
await res.body.text();
|
|
723
|
-
if (res.statusCode === 405) {
|
|
724
|
-
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
725
|
-
}
|
|
726
|
-
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
727
|
-
return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
|
|
728
|
-
}
|
|
729
|
-
if (res.statusCode === 400 || res.statusCode === 404) {
|
|
730
|
-
return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
|
|
731
|
-
}
|
|
732
|
-
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
733
|
-
}
|
|
734
|
-
);
|
|
735
1042
|
await test(
|
|
736
1043
|
"transport-batch-reject",
|
|
737
1044
|
"Rejects JSON-RPC batch requests",
|
|
@@ -769,7 +1076,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
769
1076
|
try {
|
|
770
1077
|
initRes = await rpc("initialize", {
|
|
771
1078
|
protocolVersion: SPEC_VERSION,
|
|
772
|
-
capabilities: {
|
|
1079
|
+
capabilities: {},
|
|
773
1080
|
clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
|
|
774
1081
|
});
|
|
775
1082
|
const result = initRes?.body?.result;
|
|
@@ -782,7 +1089,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
782
1089
|
if (sid) sessionId = sid;
|
|
783
1090
|
if (result.protocolVersion) negotiatedProtocolVersion = result.protocolVersion;
|
|
784
1091
|
}
|
|
785
|
-
} catch
|
|
1092
|
+
} catch {
|
|
786
1093
|
}
|
|
787
1094
|
try {
|
|
788
1095
|
await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders(), timeout);
|
|
@@ -901,7 +1208,17 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
901
1208
|
if (res.body?.error) {
|
|
902
1209
|
return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
|
|
903
1210
|
}
|
|
904
|
-
|
|
1211
|
+
const invalidRes = await rpc("logging/setLevel", { level: "__invalid_level__" });
|
|
1212
|
+
const validatesInput = !!invalidRes.body?.error;
|
|
1213
|
+
const validLevels = ["debug", "warning", "error"];
|
|
1214
|
+
const accepted = [];
|
|
1215
|
+
for (const level of validLevels) {
|
|
1216
|
+
const r = await rpc("logging/setLevel", { level });
|
|
1217
|
+
if (!r.body?.error) accepted.push(level);
|
|
1218
|
+
}
|
|
1219
|
+
const details = validatesInput ? `logging/setLevel accepted (validates levels, ${accepted.length + 1} levels accepted)` : "logging/setLevel accepted (warning: server does not reject invalid log levels)";
|
|
1220
|
+
if (!validatesInput) warnings.push("Server accepts invalid log levels without error");
|
|
1221
|
+
return { passed: true, details };
|
|
905
1222
|
}
|
|
906
1223
|
);
|
|
907
1224
|
const hasCompletions = !!serverInfo.capabilities.completions;
|
|
@@ -930,6 +1247,59 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
930
1247
|
return { passed: true, details: "completion/complete accepted" };
|
|
931
1248
|
}
|
|
932
1249
|
);
|
|
1250
|
+
await test(
|
|
1251
|
+
"lifecycle-cancellation",
|
|
1252
|
+
"Handles cancellation notifications",
|
|
1253
|
+
"lifecycle",
|
|
1254
|
+
false,
|
|
1255
|
+
"basic/utilities#cancellation",
|
|
1256
|
+
async () => {
|
|
1257
|
+
const res = await mcpNotification(
|
|
1258
|
+
backendUrl,
|
|
1259
|
+
"notifications/cancelled",
|
|
1260
|
+
{ requestId: 99999, reason: "compliance test" },
|
|
1261
|
+
buildHeaders(),
|
|
1262
|
+
timeout
|
|
1263
|
+
);
|
|
1264
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1265
|
+
return { passed: true, details: `HTTP ${res.statusCode} (cancellation accepted)` };
|
|
1266
|
+
}
|
|
1267
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should accept cancellation notifications` };
|
|
1268
|
+
}
|
|
1269
|
+
);
|
|
1270
|
+
await test(
|
|
1271
|
+
"lifecycle-progress",
|
|
1272
|
+
"Accepts progress notifications",
|
|
1273
|
+
"lifecycle",
|
|
1274
|
+
false,
|
|
1275
|
+
"basic/utilities#progress",
|
|
1276
|
+
async () => {
|
|
1277
|
+
const res = await mcpNotification(
|
|
1278
|
+
backendUrl,
|
|
1279
|
+
"notifications/progress",
|
|
1280
|
+
{ progressToken: "compliance-test-token", progress: 50, total: 100 },
|
|
1281
|
+
buildHeaders(),
|
|
1282
|
+
timeout
|
|
1283
|
+
);
|
|
1284
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1285
|
+
return { passed: true, details: `HTTP ${res.statusCode} (progress notification accepted)` };
|
|
1286
|
+
}
|
|
1287
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should accept progress notifications` };
|
|
1288
|
+
}
|
|
1289
|
+
);
|
|
1290
|
+
await test(
|
|
1291
|
+
"transport-content-type-init",
|
|
1292
|
+
"Initialize response has valid content type",
|
|
1293
|
+
"transport",
|
|
1294
|
+
false,
|
|
1295
|
+
"basic/transports#streamable-http",
|
|
1296
|
+
async () => {
|
|
1297
|
+
if (!initRes) return { passed: false, details: "No init response to check" };
|
|
1298
|
+
const ct = (initRes.headers["content-type"] || "").toLowerCase();
|
|
1299
|
+
const valid = ct.includes("application/json") || ct.includes("text/event-stream");
|
|
1300
|
+
return { passed: valid, details: `Content-Type: ${ct || "missing"}` };
|
|
1301
|
+
}
|
|
1302
|
+
);
|
|
933
1303
|
await test(
|
|
934
1304
|
"transport-notification-202",
|
|
935
1305
|
"Notification returns 202 Accepted",
|
|
@@ -987,6 +1357,88 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
987
1357
|
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
988
1358
|
}
|
|
989
1359
|
);
|
|
1360
|
+
await test(
|
|
1361
|
+
"transport-get-stream",
|
|
1362
|
+
"GET with session returns SSE or 405",
|
|
1363
|
+
"transport",
|
|
1364
|
+
false,
|
|
1365
|
+
"basic/transports#streamable-http",
|
|
1366
|
+
async () => {
|
|
1367
|
+
if (!sessionId) {
|
|
1368
|
+
return { passed: true, details: "No session ID \u2014 server-initiated messages not applicable" };
|
|
1369
|
+
}
|
|
1370
|
+
const res = await request(backendUrl, {
|
|
1371
|
+
method: "GET",
|
|
1372
|
+
headers: { Accept: "text/event-stream", ...buildHeaders() },
|
|
1373
|
+
signal: AbortSignal.timeout(Math.min(timeout, 3e3))
|
|
1374
|
+
});
|
|
1375
|
+
const body = await res.body.text();
|
|
1376
|
+
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
1377
|
+
if (res.statusCode === 405) {
|
|
1378
|
+
return { passed: true, details: "HTTP 405 (server does not support server-initiated messages)" };
|
|
1379
|
+
}
|
|
1380
|
+
if (ct.includes("text/event-stream")) {
|
|
1381
|
+
if (body.trim().length > 0) {
|
|
1382
|
+
const hasSSEFields = body.includes("data:") || body.includes("event:");
|
|
1383
|
+
if (!hasSSEFields) {
|
|
1384
|
+
return { passed: false, details: "Content-Type is text/event-stream but body has no SSE fields" };
|
|
1385
|
+
}
|
|
1386
|
+
}
|
|
1387
|
+
return { passed: true, details: "GET with session returns SSE stream for server-initiated messages" };
|
|
1388
|
+
}
|
|
1389
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1390
|
+
return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
|
|
1391
|
+
}
|
|
1392
|
+
return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
|
|
1393
|
+
}
|
|
1394
|
+
);
|
|
1395
|
+
await test(
|
|
1396
|
+
"transport-concurrent",
|
|
1397
|
+
"Handles concurrent requests",
|
|
1398
|
+
"transport",
|
|
1399
|
+
false,
|
|
1400
|
+
"basic/transports#streamable-http",
|
|
1401
|
+
async () => {
|
|
1402
|
+
const ids = [createIdCounter(99930)(), createIdCounter(99931)(), createIdCounter(99932)()];
|
|
1403
|
+
const promises = ids.map(
|
|
1404
|
+
(id) => request(backendUrl, {
|
|
1405
|
+
method: "POST",
|
|
1406
|
+
headers: {
|
|
1407
|
+
"Content-Type": "application/json",
|
|
1408
|
+
Accept: "application/json, text/event-stream",
|
|
1409
|
+
...buildHeaders()
|
|
1410
|
+
},
|
|
1411
|
+
body: JSON.stringify({ jsonrpc: "2.0", id, method: "ping" }),
|
|
1412
|
+
signal: AbortSignal.timeout(timeout)
|
|
1413
|
+
}).then(async (res) => {
|
|
1414
|
+
const text = await res.body.text();
|
|
1415
|
+
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
1416
|
+
let body;
|
|
1417
|
+
if (ct.includes("text/event-stream")) {
|
|
1418
|
+
body = parseSSEResponse(text);
|
|
1419
|
+
}
|
|
1420
|
+
if (!body) {
|
|
1421
|
+
try {
|
|
1422
|
+
body = JSON.parse(text);
|
|
1423
|
+
} catch {
|
|
1424
|
+
}
|
|
1425
|
+
}
|
|
1426
|
+
return { statusCode: res.statusCode, body, requestId: id };
|
|
1427
|
+
})
|
|
1428
|
+
);
|
|
1429
|
+
const results = await Promise.all(promises);
|
|
1430
|
+
const issues = [];
|
|
1431
|
+
for (const r of results) {
|
|
1432
|
+
if (r.statusCode < 200 || r.statusCode >= 300) {
|
|
1433
|
+
issues.push(`Request id=${r.requestId}: HTTP ${r.statusCode}`);
|
|
1434
|
+
} else if (r.body?.id !== r.requestId) {
|
|
1435
|
+
issues.push(`Request id=${r.requestId}: response id=${r.body?.id} (mismatch)`);
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1439
|
+
return { passed: true, details: `${results.length} concurrent requests handled correctly` };
|
|
1440
|
+
}
|
|
1441
|
+
);
|
|
990
1442
|
const hasTools = !!serverInfo.capabilities.tools;
|
|
991
1443
|
let cachedToolsList = null;
|
|
992
1444
|
await test(
|
|
@@ -1008,6 +1460,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1008
1460
|
};
|
|
1009
1461
|
}
|
|
1010
1462
|
);
|
|
1463
|
+
const toolsListOk = cachedToolsList !== null;
|
|
1011
1464
|
await test(
|
|
1012
1465
|
"tools-schema",
|
|
1013
1466
|
"All tools have name and inputSchema",
|
|
@@ -1015,7 +1468,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1015
1468
|
hasTools,
|
|
1016
1469
|
"server/tools#data-types",
|
|
1017
1470
|
async () => {
|
|
1018
|
-
|
|
1471
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1472
|
+
const tools = cachedToolsList ?? [];
|
|
1019
1473
|
const issues = [];
|
|
1020
1474
|
for (const tool of tools) {
|
|
1021
1475
|
if (!tool.name) {
|
|
@@ -1047,7 +1501,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1047
1501
|
false,
|
|
1048
1502
|
"server/tools#annotations",
|
|
1049
1503
|
async () => {
|
|
1050
|
-
|
|
1504
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1505
|
+
const tools = cachedToolsList ?? [];
|
|
1051
1506
|
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1052
1507
|
const issues = [];
|
|
1053
1508
|
let annotatedCount = 0;
|
|
@@ -1077,7 +1532,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1077
1532
|
}
|
|
1078
1533
|
);
|
|
1079
1534
|
await test("tools-title-field", "Tools include title field", "schema", false, "server/tools#data-types", async () => {
|
|
1080
|
-
|
|
1535
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1536
|
+
const tools = cachedToolsList ?? [];
|
|
1081
1537
|
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1082
1538
|
const withTitle = tools.filter((t) => typeof t.title === "string");
|
|
1083
1539
|
const issues = [];
|
|
@@ -1099,7 +1555,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1099
1555
|
false,
|
|
1100
1556
|
"server/tools#structured-content",
|
|
1101
1557
|
async () => {
|
|
1102
|
-
|
|
1558
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1559
|
+
const tools = cachedToolsList ?? [];
|
|
1103
1560
|
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1104
1561
|
const issues = [];
|
|
1105
1562
|
let withSchema = 0;
|
|
@@ -1140,14 +1597,14 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1140
1597
|
return { passed: true, details: `Protocol error: code ${code} \u2014 ${error.message}` };
|
|
1141
1598
|
}
|
|
1142
1599
|
if (result?.content && Array.isArray(result.content)) {
|
|
1600
|
+
if (result.isError) {
|
|
1601
|
+
return { passed: true, details: "Tool returned execution error with content (valid)" };
|
|
1602
|
+
}
|
|
1143
1603
|
const badItems = result.content.filter((c) => !c.type);
|
|
1144
1604
|
if (badItems.length > 0)
|
|
1145
1605
|
return { passed: false, details: `${badItems.length} content item(s) missing 'type' field` };
|
|
1146
1606
|
return { passed: true, details: `Returned ${result.content.length} content item(s)` };
|
|
1147
1607
|
}
|
|
1148
|
-
if (result?.isError && result?.content && Array.isArray(result.content)) {
|
|
1149
|
-
return { passed: true, details: "Tool returned execution error with content (valid)" };
|
|
1150
|
-
}
|
|
1151
1608
|
return { passed: false, details: "Response missing content array" };
|
|
1152
1609
|
}
|
|
1153
1610
|
);
|
|
@@ -1249,6 +1706,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1249
1706
|
return { passed: true, details: `${resourceCount} resource(s)` };
|
|
1250
1707
|
}
|
|
1251
1708
|
);
|
|
1709
|
+
const resourcesListOk = cachedResourcesList !== null;
|
|
1252
1710
|
await test(
|
|
1253
1711
|
"resources-schema",
|
|
1254
1712
|
"Resources have uri and name",
|
|
@@ -1256,7 +1714,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1256
1714
|
true,
|
|
1257
1715
|
"server/resources#data-types",
|
|
1258
1716
|
async () => {
|
|
1259
|
-
|
|
1717
|
+
if (!resourcesListOk) return { passed: false, details: "Skipped: resources/list failed" };
|
|
1718
|
+
const resources = cachedResourcesList ?? [];
|
|
1260
1719
|
const issues = [];
|
|
1261
1720
|
for (const r of resources) {
|
|
1262
1721
|
if (!r.uri) issues.push("Resource missing uri");
|
|
@@ -1285,7 +1744,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1285
1744
|
false,
|
|
1286
1745
|
"server/resources#reading-resources",
|
|
1287
1746
|
async () => {
|
|
1288
|
-
const resources = cachedResourcesList ??
|
|
1747
|
+
const resources = cachedResourcesList ?? [];
|
|
1289
1748
|
const firstUri = resources[0]?.uri;
|
|
1290
1749
|
if (!firstUri) return { passed: false, details: "No resource URI to test" };
|
|
1291
1750
|
const readRes = await rpc("resources/read", { uri: firstUri });
|
|
@@ -1318,8 +1777,15 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1318
1777
|
if (!Array.isArray(templates)) return { passed: false, details: "No resourceTemplates array" };
|
|
1319
1778
|
const issues = [];
|
|
1320
1779
|
for (const t of templates) {
|
|
1321
|
-
if (!t.uriTemplate)
|
|
1780
|
+
if (!t.uriTemplate) {
|
|
1781
|
+
issues.push("Template missing uriTemplate");
|
|
1782
|
+
} else if (typeof t.uriTemplate !== "string") {
|
|
1783
|
+
issues.push(`uriTemplate should be a string, got ${typeof t.uriTemplate}`);
|
|
1784
|
+
} else if (!t.uriTemplate.includes("{") || !t.uriTemplate.includes("}")) {
|
|
1785
|
+
warnings.push(`Template "${t.name || t.uriTemplate}" has no URI template parameters (e.g., {id})`);
|
|
1786
|
+
}
|
|
1322
1787
|
if (!t.name) issues.push(`${t.uriTemplate || "?"}: missing name`);
|
|
1788
|
+
if (!t.description) warnings.push(`Template "${t.name || t.uriTemplate || "?"}" missing description`);
|
|
1323
1789
|
}
|
|
1324
1790
|
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1325
1791
|
return { passed: true, details: `${templates.length} resource template(s)` };
|
|
@@ -1361,7 +1827,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1361
1827
|
true,
|
|
1362
1828
|
"server/resources#subscriptions",
|
|
1363
1829
|
async () => {
|
|
1364
|
-
const resources = cachedResourcesList ??
|
|
1830
|
+
const resources = cachedResourcesList ?? [];
|
|
1365
1831
|
const firstUri = resources[0]?.uri;
|
|
1366
1832
|
if (!firstUri) return { passed: false, details: "No resource URI for subscribe test" };
|
|
1367
1833
|
const subRes = await rpc("resources/subscribe", { uri: firstUri });
|
|
@@ -1405,8 +1871,10 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1405
1871
|
};
|
|
1406
1872
|
}
|
|
1407
1873
|
);
|
|
1874
|
+
const promptsListOk = cachedPromptsList !== null;
|
|
1408
1875
|
await test("prompts-schema", "Prompts have name field", "schema", true, "server/prompts#data-types", async () => {
|
|
1409
|
-
|
|
1876
|
+
if (!promptsListOk) return { passed: false, details: "Skipped: prompts/list failed" };
|
|
1877
|
+
const prompts = cachedPromptsList ?? [];
|
|
1410
1878
|
const issues = [];
|
|
1411
1879
|
for (const p of prompts) {
|
|
1412
1880
|
if (!p.name) issues.push("Prompt missing name");
|
|
@@ -1602,36 +2070,780 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1602
2070
|
}
|
|
1603
2071
|
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error code -32600` };
|
|
1604
2072
|
});
|
|
1605
|
-
const
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
}
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
2073
|
+
const hasAuth = !!userHeaders.Authorization || !!userHeaders.authorization;
|
|
2074
|
+
await test(
|
|
2075
|
+
"security-auth-required",
|
|
2076
|
+
"Rejects unauthenticated requests",
|
|
2077
|
+
"security",
|
|
2078
|
+
false,
|
|
2079
|
+
"basic/authorization",
|
|
2080
|
+
async () => {
|
|
2081
|
+
if (!hasAuth) {
|
|
2082
|
+
return {
|
|
2083
|
+
passed: false,
|
|
2084
|
+
details: "Server does not require auth (no --auth provided and server accepted unauthenticated requests)"
|
|
2085
|
+
};
|
|
2086
|
+
}
|
|
2087
|
+
const noAuthHeaders = {};
|
|
2088
|
+
if (sessionId) noAuthHeaders["mcp-session-id"] = sessionId;
|
|
2089
|
+
try {
|
|
2090
|
+
const res = await mcpRequest(backendUrl, "ping", void 0, nextId, noAuthHeaders, timeout);
|
|
2091
|
+
if (res.statusCode === 401 || res.statusCode === 403) {
|
|
2092
|
+
return { passed: true, details: `HTTP ${res.statusCode} (unauthenticated request rejected)` };
|
|
2093
|
+
}
|
|
2094
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server accepted unauthenticated request` };
|
|
2095
|
+
} catch (err) {
|
|
2096
|
+
return { passed: true, details: "Connection rejected (acceptable)" };
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
2099
|
+
);
|
|
2100
|
+
await test(
|
|
2101
|
+
"security-auth-malformed",
|
|
2102
|
+
"Rejects malformed auth credentials",
|
|
2103
|
+
"security",
|
|
2104
|
+
false,
|
|
2105
|
+
"basic/authorization",
|
|
2106
|
+
async () => {
|
|
2107
|
+
if (!hasAuth) {
|
|
2108
|
+
return { passed: false, details: "Skipped: server does not require auth" };
|
|
2109
|
+
}
|
|
2110
|
+
const malformedHeaders = {
|
|
2111
|
+
Authorization: "Bearer INVALID_GARBAGE_TOKEN_!@#$%^&*()"
|
|
2112
|
+
};
|
|
2113
|
+
if (sessionId) malformedHeaders["mcp-session-id"] = sessionId;
|
|
2114
|
+
try {
|
|
2115
|
+
const res = await mcpRequest(backendUrl, "ping", void 0, nextId, malformedHeaders, timeout);
|
|
2116
|
+
if (res.statusCode === 401 || res.statusCode === 403) {
|
|
2117
|
+
return { passed: true, details: `HTTP ${res.statusCode} (malformed auth rejected)` };
|
|
2118
|
+
}
|
|
2119
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server accepted malformed auth token` };
|
|
2120
|
+
} catch (err) {
|
|
2121
|
+
return { passed: true, details: "Connection rejected (acceptable)" };
|
|
2122
|
+
}
|
|
2123
|
+
}
|
|
2124
|
+
);
|
|
2125
|
+
await test("security-tls-required", "Enforces HTTPS/TLS", "security", false, "basic/authorization", async () => {
|
|
2126
|
+
const parsedUrl = new URL(url);
|
|
2127
|
+
if (parsedUrl.protocol !== "https:") {
|
|
2128
|
+
return { passed: false, details: `Server URL uses ${parsedUrl.protocol} \u2014 production servers should use HTTPS` };
|
|
2129
|
+
}
|
|
2130
|
+
const httpUrl = url.replace(/^https:/, "http:");
|
|
2131
|
+
try {
|
|
2132
|
+
const res = await request(httpUrl, {
|
|
2133
|
+
method: "POST",
|
|
2134
|
+
headers: { "Content-Type": "application/json", Accept: "application/json" },
|
|
2135
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 99950, method: "ping" }),
|
|
2136
|
+
signal: AbortSignal.timeout(Math.min(timeout, 5e3))
|
|
2137
|
+
});
|
|
2138
|
+
await res.body.text();
|
|
2139
|
+
if (res.statusCode === 301 || res.statusCode === 302 || res.statusCode === 308) {
|
|
2140
|
+
return { passed: true, details: `HTTP ${res.statusCode} redirect to HTTPS (good)` };
|
|
2141
|
+
}
|
|
2142
|
+
if (res.statusCode >= 400) {
|
|
2143
|
+
return { passed: true, details: `HTTP ${res.statusCode} (plaintext rejected)` };
|
|
2144
|
+
}
|
|
2145
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server accepts plaintext HTTP connections` };
|
|
2146
|
+
} catch {
|
|
2147
|
+
return { passed: true, details: "HTTP connection refused (HTTPS enforced)" };
|
|
2148
|
+
}
|
|
2149
|
+
});
|
|
2150
|
+
await test(
|
|
2151
|
+
"security-session-entropy",
|
|
2152
|
+
"Session IDs are high-entropy",
|
|
2153
|
+
"security",
|
|
2154
|
+
false,
|
|
2155
|
+
"basic/transports#streamable-http",
|
|
2156
|
+
async () => {
|
|
2157
|
+
if (!sessionId) {
|
|
2158
|
+
return { passed: true, details: "Server does not issue session IDs (skipped)" };
|
|
2159
|
+
}
|
|
2160
|
+
if (sessionId.length < 16) {
|
|
2161
|
+
return {
|
|
2162
|
+
passed: false,
|
|
2163
|
+
details: `Session ID too short (${sessionId.length} chars): "${sessionId}" \u2014 should be \u226516 chars`
|
|
2164
|
+
};
|
|
2165
|
+
}
|
|
2166
|
+
if (/^\d+$/.test(sessionId)) {
|
|
2167
|
+
return {
|
|
2168
|
+
passed: false,
|
|
2169
|
+
details: `Session ID is purely numeric: "${sessionId}" \u2014 likely sequential, not random`
|
|
2170
|
+
};
|
|
2171
|
+
}
|
|
2172
|
+
const uniqueChars = new Set(sessionId.toLowerCase()).size;
|
|
2173
|
+
if (uniqueChars < 8) {
|
|
2174
|
+
return {
|
|
2175
|
+
passed: false,
|
|
2176
|
+
details: `Session ID has low character diversity (${uniqueChars} unique chars): "${sessionId}"`
|
|
2177
|
+
};
|
|
2178
|
+
}
|
|
2179
|
+
return {
|
|
2180
|
+
passed: true,
|
|
2181
|
+
details: `Session ID has good entropy (${sessionId.length} chars, ${uniqueChars} unique): "${sessionId.substring(0, 16)}..."`
|
|
2182
|
+
};
|
|
2183
|
+
}
|
|
2184
|
+
);
|
|
2185
|
+
await test(
|
|
2186
|
+
"security-session-not-auth",
|
|
2187
|
+
"Session ID does not bypass auth",
|
|
2188
|
+
"security",
|
|
2189
|
+
false,
|
|
2190
|
+
"basic/transports#streamable-http",
|
|
2191
|
+
async () => {
|
|
2192
|
+
if (!hasAuth) {
|
|
2193
|
+
return { passed: true, details: "Skipped: server does not require auth" };
|
|
2194
|
+
}
|
|
2195
|
+
if (!sessionId) {
|
|
2196
|
+
return { passed: true, details: "Skipped: server does not issue session IDs" };
|
|
2197
|
+
}
|
|
2198
|
+
const sessionOnlyHeaders = {
|
|
2199
|
+
"mcp-session-id": sessionId
|
|
2200
|
+
};
|
|
2201
|
+
try {
|
|
2202
|
+
const res = await mcpRequest(backendUrl, "ping", void 0, nextId, sessionOnlyHeaders, timeout);
|
|
2203
|
+
if (res.statusCode === 401 || res.statusCode === 403) {
|
|
2204
|
+
return { passed: true, details: `HTTP ${res.statusCode} (session ID alone not sufficient for auth)` };
|
|
2205
|
+
}
|
|
2206
|
+
return {
|
|
2207
|
+
passed: false,
|
|
2208
|
+
details: `HTTP ${res.statusCode} \u2014 server accepted session ID without auth (spec: MUST NOT use sessions for authentication)`
|
|
2209
|
+
};
|
|
2210
|
+
} catch {
|
|
2211
|
+
return { passed: true, details: "Connection rejected (acceptable)" };
|
|
2212
|
+
}
|
|
2213
|
+
}
|
|
2214
|
+
);
|
|
2215
|
+
await test(
|
|
2216
|
+
"security-oauth-metadata",
|
|
2217
|
+
"OAuth metadata endpoint exists",
|
|
2218
|
+
"security",
|
|
2219
|
+
false,
|
|
2220
|
+
"basic/authorization",
|
|
2221
|
+
async () => {
|
|
2222
|
+
if (!hasAuth) {
|
|
2223
|
+
return { passed: true, details: "Skipped: server does not require auth" };
|
|
2224
|
+
}
|
|
2225
|
+
const parsedUrl = new URL(url);
|
|
2226
|
+
const metadataUrl = `${parsedUrl.protocol}//${parsedUrl.host}/.well-known/oauth-authorization-server`;
|
|
2227
|
+
try {
|
|
2228
|
+
const res = await request(metadataUrl, {
|
|
2229
|
+
method: "GET",
|
|
2230
|
+
headers: { Accept: "application/json" },
|
|
2231
|
+
signal: AbortSignal.timeout(Math.min(timeout, 5e3))
|
|
2232
|
+
});
|
|
2233
|
+
const text = await res.body.text();
|
|
2234
|
+
if (res.statusCode === 200) {
|
|
2235
|
+
try {
|
|
2236
|
+
const meta = JSON.parse(text);
|
|
2237
|
+
if (meta.issuer && meta.token_endpoint) {
|
|
2238
|
+
return { passed: true, details: `OAuth metadata found: issuer=${meta.issuer}` };
|
|
2239
|
+
}
|
|
2240
|
+
return {
|
|
2241
|
+
passed: false,
|
|
2242
|
+
details: "OAuth metadata response missing required fields (issuer, token_endpoint)"
|
|
2243
|
+
};
|
|
2244
|
+
} catch {
|
|
2245
|
+
return { passed: false, details: "OAuth metadata endpoint returned non-JSON response" };
|
|
2246
|
+
}
|
|
2247
|
+
}
|
|
2248
|
+
return { passed: false, details: `OAuth metadata endpoint returned HTTP ${res.statusCode}` };
|
|
2249
|
+
} catch {
|
|
2250
|
+
return { passed: false, details: "OAuth metadata endpoint unreachable" };
|
|
2251
|
+
}
|
|
2252
|
+
}
|
|
2253
|
+
);
|
|
2254
|
+
await test(
|
|
2255
|
+
"security-token-in-uri",
|
|
2256
|
+
"Rejects auth tokens in query string",
|
|
2257
|
+
"security",
|
|
2258
|
+
false,
|
|
2259
|
+
"basic/authorization",
|
|
2260
|
+
async () => {
|
|
2261
|
+
if (!hasAuth) {
|
|
2262
|
+
return { passed: true, details: "Skipped: server does not require auth" };
|
|
2263
|
+
}
|
|
2264
|
+
const authValue = userHeaders.Authorization || userHeaders.authorization || "";
|
|
2265
|
+
const token = authValue.replace(/^Bearer\s+/i, "");
|
|
2266
|
+
if (!token) {
|
|
2267
|
+
return { passed: true, details: "Skipped: could not extract token from auth header" };
|
|
2268
|
+
}
|
|
2269
|
+
const uriWithToken = `${url}${url.includes("?") ? "&" : "?"}access_token=${encodeURIComponent(token)}`;
|
|
2270
|
+
try {
|
|
2271
|
+
const noAuthHeaders = {};
|
|
2272
|
+
if (sessionId) noAuthHeaders["mcp-session-id"] = sessionId;
|
|
2273
|
+
const res = await mcpRequest(uriWithToken, "ping", void 0, nextId, noAuthHeaders, timeout);
|
|
2274
|
+
if (res.statusCode === 401 || res.statusCode === 403) {
|
|
2275
|
+
return { passed: true, details: `HTTP ${res.statusCode} (token in query string rejected)` };
|
|
2276
|
+
}
|
|
2277
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
2278
|
+
return {
|
|
2279
|
+
passed: false,
|
|
2280
|
+
details: "Server accepted auth token in query string (spec: MUST NOT transmit credentials in URIs)"
|
|
2281
|
+
};
|
|
2282
|
+
}
|
|
2283
|
+
return { passed: true, details: `HTTP ${res.statusCode} (token in query string not accepted)` };
|
|
2284
|
+
} catch {
|
|
2285
|
+
return { passed: true, details: "Connection rejected (acceptable)" };
|
|
2286
|
+
}
|
|
2287
|
+
}
|
|
2288
|
+
);
|
|
2289
|
+
await test(
|
|
2290
|
+
"security-cors-headers",
|
|
2291
|
+
"CORS headers are restrictive",
|
|
2292
|
+
"security",
|
|
2293
|
+
false,
|
|
2294
|
+
"basic/transports#streamable-http",
|
|
2295
|
+
async () => {
|
|
2296
|
+
try {
|
|
2297
|
+
const res = await request(backendUrl, {
|
|
2298
|
+
method: "OPTIONS",
|
|
2299
|
+
headers: {
|
|
2300
|
+
Origin: "https://evil.example.com",
|
|
2301
|
+
"Access-Control-Request-Method": "POST",
|
|
2302
|
+
...buildHeaders()
|
|
2303
|
+
},
|
|
2304
|
+
signal: AbortSignal.timeout(Math.min(timeout, 5e3))
|
|
2305
|
+
});
|
|
2306
|
+
await res.body.text();
|
|
2307
|
+
const acao = res.headers["access-control-allow-origin"];
|
|
2308
|
+
if (!acao) {
|
|
2309
|
+
return { passed: true, details: "No CORS headers returned (server-to-server only, acceptable)" };
|
|
2310
|
+
}
|
|
2311
|
+
if (acao === "*") {
|
|
2312
|
+
return {
|
|
2313
|
+
passed: false,
|
|
2314
|
+
details: 'Access-Control-Allow-Origin is "*" (wildcard) \u2014 allows cross-origin credential theft'
|
|
2315
|
+
};
|
|
2316
|
+
}
|
|
2317
|
+
if (acao === "https://evil.example.com") {
|
|
2318
|
+
return { passed: false, details: "Server reflects arbitrary Origin in CORS \u2014 effectively wildcard" };
|
|
2319
|
+
}
|
|
2320
|
+
return { passed: true, details: `CORS restricted to: ${acao}` };
|
|
2321
|
+
} catch {
|
|
2322
|
+
return { passed: true, details: "OPTIONS request failed (no CORS, acceptable)" };
|
|
2323
|
+
}
|
|
2324
|
+
}
|
|
2325
|
+
);
|
|
2326
|
+
async function runInjectionTest(toolName, paramName, payloads, detectPattern, label) {
|
|
2327
|
+
const issues = [];
|
|
2328
|
+
for (const payload of payloads) {
|
|
2329
|
+
try {
|
|
2330
|
+
const res = await rpc("tools/call", { name: toolName, arguments: { [paramName]: payload } });
|
|
2331
|
+
const content = res.body?.result?.content;
|
|
2332
|
+
if (Array.isArray(content)) {
|
|
2333
|
+
const text = content.map((c) => c.text || "").join(" ");
|
|
2334
|
+
if (detectPattern.test(text)) {
|
|
2335
|
+
issues.push(`Payload "${payload}" ${label} (output: ${text.substring(0, 100)})`);
|
|
2336
|
+
}
|
|
2337
|
+
}
|
|
2338
|
+
} catch {
|
|
2339
|
+
}
|
|
2340
|
+
}
|
|
2341
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
2342
|
+
return {
|
|
2343
|
+
passed: true,
|
|
2344
|
+
details: `Tested ${payloads.length} payloads against ${toolName}.${paramName} \u2014 no ${label.split(" ")[0]} detected`
|
|
2345
|
+
};
|
|
2346
|
+
}
|
|
2347
|
+
if (toolNames.length > 0) {
|
|
2348
|
+
const allTools = cachedToolsList ?? [];
|
|
2349
|
+
const toolsWithStringParams = allTools.filter((t) => {
|
|
2350
|
+
const props = t.inputSchema?.properties;
|
|
2351
|
+
if (!props) return false;
|
|
2352
|
+
return Object.values(props).some((p) => p.type === "string");
|
|
2353
|
+
});
|
|
2354
|
+
const injectionTarget = toolsWithStringParams[0] || allTools[0];
|
|
2355
|
+
const targetStringParam = injectionTarget?.inputSchema?.properties ? Object.entries(injectionTarget.inputSchema.properties).find(
|
|
2356
|
+
([_, v]) => v.type === "string"
|
|
2357
|
+
)?.[0] ?? null : null;
|
|
2358
|
+
await test(
|
|
2359
|
+
"security-command-injection",
|
|
2360
|
+
"Resists command injection in tool params",
|
|
2361
|
+
"security",
|
|
2362
|
+
false,
|
|
2363
|
+
"server/tools#calling-tools",
|
|
2364
|
+
async () => {
|
|
2365
|
+
if (!injectionTarget || !targetStringParam)
|
|
2366
|
+
return { passed: true, details: "No tools with string parameters to test" };
|
|
2367
|
+
return runInjectionTest(
|
|
2368
|
+
injectionTarget.name,
|
|
2369
|
+
targetStringParam,
|
|
2370
|
+
INJECTION_PAYLOADS.command,
|
|
2371
|
+
/root:.*:\d+:\d+:.*:\/|uid=\d+\(\w+\)|drwxr|pwned/i,
|
|
2372
|
+
"appears to have executed"
|
|
2373
|
+
);
|
|
2374
|
+
}
|
|
2375
|
+
);
|
|
2376
|
+
await test(
|
|
2377
|
+
"security-sql-injection",
|
|
2378
|
+
"Resists SQL injection in tool params",
|
|
2379
|
+
"security",
|
|
2380
|
+
false,
|
|
2381
|
+
"server/tools#calling-tools",
|
|
2382
|
+
async () => {
|
|
2383
|
+
if (!injectionTarget || !targetStringParam)
|
|
2384
|
+
return { passed: true, details: "No tools with string parameters to test" };
|
|
2385
|
+
return runInjectionTest(
|
|
2386
|
+
injectionTarget.name,
|
|
2387
|
+
targetStringParam,
|
|
2388
|
+
INJECTION_PAYLOADS.sql,
|
|
2389
|
+
/syntax error|sql|mysql|postgres|sqlite|information_schema|table_name/i,
|
|
2390
|
+
"triggered database error"
|
|
2391
|
+
);
|
|
2392
|
+
}
|
|
2393
|
+
);
|
|
2394
|
+
await test(
|
|
2395
|
+
"security-path-traversal",
|
|
2396
|
+
"Resists path traversal in tool params",
|
|
2397
|
+
"security",
|
|
2398
|
+
false,
|
|
2399
|
+
"server/tools#calling-tools",
|
|
2400
|
+
async () => {
|
|
2401
|
+
if (!injectionTarget || !targetStringParam)
|
|
2402
|
+
return { passed: true, details: "No tools with string parameters to test" };
|
|
2403
|
+
return runInjectionTest(
|
|
2404
|
+
injectionTarget.name,
|
|
2405
|
+
targetStringParam,
|
|
2406
|
+
INJECTION_PAYLOADS.pathTraversal,
|
|
2407
|
+
/root:.*:0:0|\[boot loader\]|\[extensions\]/i,
|
|
2408
|
+
"returned sensitive file content"
|
|
2409
|
+
);
|
|
2410
|
+
}
|
|
2411
|
+
);
|
|
2412
|
+
await test(
|
|
2413
|
+
"security-ssrf-internal",
|
|
2414
|
+
"Resists SSRF to internal networks",
|
|
2415
|
+
"security",
|
|
2416
|
+
false,
|
|
2417
|
+
"server/tools#calling-tools",
|
|
2418
|
+
async () => {
|
|
2419
|
+
const urlParamTool = allTools.find((t) => {
|
|
2420
|
+
const props = t.inputSchema?.properties;
|
|
2421
|
+
if (!props) return false;
|
|
2422
|
+
return Object.entries(props).some(
|
|
2423
|
+
([k, v]) => v.type === "string" && /url|uri|endpoint|link|href/i.test(k)
|
|
2424
|
+
);
|
|
2425
|
+
});
|
|
2426
|
+
if (!urlParamTool) return { passed: true, details: "No tools with URL parameters found (skipped)" };
|
|
2427
|
+
const urlParam = Object.entries(urlParamTool.inputSchema.properties).find(
|
|
2428
|
+
([k, v]) => v.type === "string" && /url|uri|endpoint|link|href/i.test(k)
|
|
2429
|
+
)?.[0];
|
|
2430
|
+
if (!urlParam) return { passed: true, details: "No URL parameter found" };
|
|
2431
|
+
return runInjectionTest(
|
|
2432
|
+
urlParamTool.name,
|
|
2433
|
+
urlParam,
|
|
2434
|
+
INJECTION_PAYLOADS.ssrf,
|
|
2435
|
+
/ami-|instance-id|hostname|iam|security-credentials/i,
|
|
2436
|
+
"returned internal data"
|
|
2437
|
+
);
|
|
2438
|
+
}
|
|
2439
|
+
);
|
|
2440
|
+
} else {
|
|
2441
|
+
for (const testId of [
|
|
2442
|
+
"security-command-injection",
|
|
2443
|
+
"security-sql-injection",
|
|
2444
|
+
"security-path-traversal",
|
|
2445
|
+
"security-ssrf-internal"
|
|
2446
|
+
]) {
|
|
2447
|
+
await test(
|
|
2448
|
+
testId,
|
|
2449
|
+
TEST_DEFINITIONS_MAP.get(testId)?.name || testId,
|
|
2450
|
+
"security",
|
|
2451
|
+
false,
|
|
2452
|
+
"server/tools#calling-tools",
|
|
2453
|
+
async () => ({ passed: true, details: "No tools available to test (skipped)" })
|
|
2454
|
+
);
|
|
2455
|
+
}
|
|
2456
|
+
}
|
|
2457
|
+
await test(
|
|
2458
|
+
"security-oversized-input",
|
|
2459
|
+
"Handles oversized inputs gracefully",
|
|
2460
|
+
"security",
|
|
2461
|
+
false,
|
|
2462
|
+
"server/tools#calling-tools",
|
|
2463
|
+
async () => {
|
|
2464
|
+
const largeValue = "A".repeat(1048576);
|
|
2465
|
+
try {
|
|
2466
|
+
const res = await request(backendUrl, {
|
|
2467
|
+
method: "POST",
|
|
2468
|
+
headers: {
|
|
2469
|
+
"Content-Type": "application/json",
|
|
2470
|
+
Accept: "application/json, text/event-stream",
|
|
2471
|
+
...buildHeaders()
|
|
2472
|
+
},
|
|
2473
|
+
body: JSON.stringify({
|
|
2474
|
+
jsonrpc: "2.0",
|
|
2475
|
+
id: nextId(),
|
|
2476
|
+
method: "tools/call",
|
|
2477
|
+
params: { name: toolNames[0] || "test", arguments: { data: largeValue } }
|
|
2478
|
+
}),
|
|
2479
|
+
signal: AbortSignal.timeout(timeout)
|
|
2480
|
+
});
|
|
2481
|
+
await res.body.text();
|
|
2482
|
+
if (res.statusCode === 413) {
|
|
2483
|
+
return { passed: true, details: "HTTP 413 Payload Too Large (good)" };
|
|
2484
|
+
}
|
|
2485
|
+
if (res.statusCode >= 400) {
|
|
2486
|
+
return { passed: true, details: `HTTP ${res.statusCode} (oversized input rejected)` };
|
|
2487
|
+
}
|
|
2488
|
+
return { passed: true, details: `HTTP ${res.statusCode} \u2014 server handled 1MB payload without crashing` };
|
|
2489
|
+
} catch (err) {
|
|
2490
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2491
|
+
if (msg.includes("timeout") || msg.includes("abort")) {
|
|
2492
|
+
return { passed: false, details: "Request timed out \u2014 server may be struggling with oversized input" };
|
|
2493
|
+
}
|
|
2494
|
+
return { passed: true, details: "Connection rejected (acceptable for oversized input)" };
|
|
2495
|
+
}
|
|
2496
|
+
}
|
|
2497
|
+
);
|
|
2498
|
+
await test(
|
|
2499
|
+
"security-extra-params",
|
|
2500
|
+
"Rejects or ignores extra tool params",
|
|
2501
|
+
"security",
|
|
2502
|
+
false,
|
|
2503
|
+
"server/tools#calling-tools",
|
|
2504
|
+
async () => {
|
|
2505
|
+
if (toolNames.length === 0) {
|
|
2506
|
+
return { passed: true, details: "No tools available to test (skipped)" };
|
|
2507
|
+
}
|
|
2508
|
+
try {
|
|
2509
|
+
const res = await rpc("tools/call", {
|
|
2510
|
+
name: toolNames[0],
|
|
2511
|
+
arguments: { __injected_param__: "malicious_value", __proto__: { admin: true } }
|
|
2512
|
+
});
|
|
2513
|
+
const error = res.body?.error;
|
|
2514
|
+
if (error) {
|
|
2515
|
+
return { passed: true, details: `Extra params rejected with error: ${error.code} \u2014 ${error.message}` };
|
|
2516
|
+
}
|
|
2517
|
+
return { passed: true, details: "Server processed request (extra params likely ignored)" };
|
|
2518
|
+
} catch {
|
|
2519
|
+
return { passed: true, details: "Request rejected (acceptable)" };
|
|
2520
|
+
}
|
|
2521
|
+
}
|
|
2522
|
+
);
|
|
2523
|
+
await test(
|
|
2524
|
+
"security-tool-schema-defined",
|
|
2525
|
+
"All tools define inputSchema",
|
|
2526
|
+
"security",
|
|
2527
|
+
false,
|
|
2528
|
+
"server/tools#data-types",
|
|
2529
|
+
async () => {
|
|
2530
|
+
if (!toolsListOk) return { passed: true, details: "Skipped: tools/list not available" };
|
|
2531
|
+
const tools = cachedToolsList ?? [];
|
|
2532
|
+
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
2533
|
+
const missing = tools.filter((t) => !t.inputSchema || t.inputSchema.type !== "object");
|
|
2534
|
+
if (missing.length > 0) {
|
|
2535
|
+
return {
|
|
2536
|
+
passed: false,
|
|
2537
|
+
details: `${missing.length} tool(s) missing inputSchema: ${missing.map((t) => t.name).join(", ")}`
|
|
2538
|
+
};
|
|
2539
|
+
}
|
|
2540
|
+
return { passed: true, details: `All ${tools.length} tool(s) have inputSchema defined` };
|
|
2541
|
+
}
|
|
2542
|
+
);
|
|
2543
|
+
await test(
|
|
2544
|
+
"security-tool-rug-pull",
|
|
2545
|
+
"Tool definitions are stable across calls",
|
|
2546
|
+
"security",
|
|
2547
|
+
false,
|
|
2548
|
+
"server/tools#listing-tools",
|
|
2549
|
+
async () => {
|
|
2550
|
+
if (!toolsListOk) return { passed: true, details: "Skipped: tools/list not available" };
|
|
2551
|
+
try {
|
|
2552
|
+
const res = await rpc("tools/list");
|
|
2553
|
+
const tools2 = res.body?.result?.tools;
|
|
2554
|
+
if (!Array.isArray(tools2)) return { passed: false, details: "Second tools/list call failed" };
|
|
2555
|
+
const tools1 = cachedToolsList ?? [];
|
|
2556
|
+
if (tools1.length !== tools2.length) {
|
|
2557
|
+
return {
|
|
2558
|
+
passed: false,
|
|
2559
|
+
details: `Tool count changed: ${tools1.length} \u2192 ${tools2.length} (possible rug-pull)`
|
|
2560
|
+
};
|
|
2561
|
+
}
|
|
2562
|
+
const names1 = tools1.map((t) => t.name).sort().join(",");
|
|
2563
|
+
const names2 = tools2.map((t) => t.name).sort().join(",");
|
|
2564
|
+
if (names1 !== names2) {
|
|
2565
|
+
return { passed: false, details: "Tool names changed between calls (possible rug-pull)" };
|
|
2566
|
+
}
|
|
2567
|
+
for (const t1 of tools1) {
|
|
2568
|
+
const t2 = tools2.find((t) => t.name === t1.name);
|
|
2569
|
+
if (t2 && t1.description !== t2.description) {
|
|
2570
|
+
return {
|
|
2571
|
+
passed: false,
|
|
2572
|
+
details: `Tool "${t1.name}" description changed between calls (possible rug-pull)`
|
|
2573
|
+
};
|
|
2574
|
+
}
|
|
2575
|
+
}
|
|
2576
|
+
return { passed: true, details: `${tools1.length} tool(s) consistent across 2 calls` };
|
|
2577
|
+
} catch {
|
|
2578
|
+
return { passed: false, details: "Second tools/list call threw an error" };
|
|
2579
|
+
}
|
|
2580
|
+
}
|
|
2581
|
+
);
|
|
2582
|
+
await test(
|
|
2583
|
+
"security-tool-description-poisoning",
|
|
2584
|
+
"Tool descriptions free of injection patterns",
|
|
2585
|
+
"security",
|
|
2586
|
+
false,
|
|
2587
|
+
"server/tools#data-types",
|
|
2588
|
+
async () => {
|
|
2589
|
+
if (!toolsListOk) return { passed: true, details: "Skipped: tools/list not available" };
|
|
2590
|
+
const tools = cachedToolsList ?? [];
|
|
2591
|
+
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
2592
|
+
const suspiciousPatterns = [
|
|
2593
|
+
{ pattern: /ignore\s+(all\s+)?previous/i, label: "ignore previous instructions" },
|
|
2594
|
+
{ pattern: /override\s+(system|instructions|rules)/i, label: "override instructions" },
|
|
2595
|
+
{ pattern: /system\s+prompt/i, label: "system prompt reference" },
|
|
2596
|
+
{ pattern: /you\s+must\s+(always|never)/i, label: "behavioral override" },
|
|
2597
|
+
{ pattern: /do\s+not\s+(tell|inform|mention)/i, label: "concealment instruction" },
|
|
2598
|
+
{ pattern: /\b[A-Za-z0-9+/]{40,}={0,2}\b/, label: "possible Base64-encoded payload" },
|
|
2599
|
+
// Zero-width characters (U+200B, U+200C, U+200D, U+FEFF)
|
|
2600
|
+
{ pattern: /\u200B|\u200C|\u200D|\uFEFF/, label: "hidden Unicode characters" }
|
|
2601
|
+
];
|
|
2602
|
+
const issues = [];
|
|
2603
|
+
for (const tool of tools) {
|
|
2604
|
+
const textsToCheck = [
|
|
2605
|
+
tool.description || "",
|
|
2606
|
+
...tool.inputSchema?.properties ? Object.values(tool.inputSchema.properties).map((p) => p.description || "") : []
|
|
2607
|
+
];
|
|
2608
|
+
const combined = textsToCheck.join(" ");
|
|
2609
|
+
for (const { pattern, label } of suspiciousPatterns) {
|
|
2610
|
+
if (pattern.test(combined)) {
|
|
2611
|
+
issues.push(`Tool "${tool.name}": ${label}`);
|
|
2612
|
+
}
|
|
2613
|
+
}
|
|
2614
|
+
}
|
|
2615
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
2616
|
+
return { passed: true, details: `${tools.length} tool(s) scanned \u2014 no injection patterns found` };
|
|
2617
|
+
}
|
|
2618
|
+
);
|
|
2619
|
+
await test(
|
|
2620
|
+
"security-tool-cross-reference",
|
|
2621
|
+
"Tools do not reference other tools by name",
|
|
2622
|
+
"security",
|
|
2623
|
+
false,
|
|
2624
|
+
"server/tools#data-types",
|
|
2625
|
+
async () => {
|
|
2626
|
+
if (!toolsListOk) return { passed: true, details: "Skipped: tools/list not available" };
|
|
2627
|
+
const tools = cachedToolsList ?? [];
|
|
2628
|
+
if (tools.length < 2)
|
|
2629
|
+
return { passed: true, details: "Fewer than 2 tools \u2014 cross-reference check not applicable" };
|
|
2630
|
+
const names = tools.map((t) => t.name).filter(Boolean);
|
|
2631
|
+
const issues = [];
|
|
2632
|
+
for (const tool of tools) {
|
|
2633
|
+
const desc = (tool.description || "").toLowerCase();
|
|
2634
|
+
for (const otherName of names) {
|
|
2635
|
+
if (otherName === tool.name) continue;
|
|
2636
|
+
if (desc.includes(otherName.toLowerCase())) {
|
|
2637
|
+
issues.push(`Tool "${tool.name}" description references "${otherName}"`);
|
|
2638
|
+
}
|
|
2639
|
+
}
|
|
2640
|
+
}
|
|
2641
|
+
if (issues.length > 0) {
|
|
2642
|
+
warnings.push(`Cross-tool references found: ${issues.join("; ")}`);
|
|
2643
|
+
return { passed: false, details: issues.join("; ") };
|
|
2644
|
+
}
|
|
2645
|
+
return { passed: true, details: `${tools.length} tool(s) checked \u2014 no cross-references found` };
|
|
2646
|
+
}
|
|
2647
|
+
);
|
|
2648
|
+
await test(
|
|
2649
|
+
"security-error-no-stacktrace",
|
|
2650
|
+
"Error responses do not leak stack traces",
|
|
2651
|
+
"security",
|
|
2652
|
+
false,
|
|
2653
|
+
"basic",
|
|
2654
|
+
async () => {
|
|
2655
|
+
const errorResponses = [];
|
|
2656
|
+
const errorPayloads = [
|
|
2657
|
+
"{this is not valid json!!!",
|
|
2658
|
+
JSON.stringify({ jsonrpc: "2.0", id: nextId(), method: "nonexistent/___crash___test___" }),
|
|
2659
|
+
JSON.stringify({
|
|
2660
|
+
jsonrpc: "2.0",
|
|
2661
|
+
id: nextId(),
|
|
2662
|
+
method: "tools/call",
|
|
2663
|
+
params: { name: "___nonexistent___tool___" }
|
|
2664
|
+
})
|
|
2665
|
+
];
|
|
2666
|
+
for (const payload of errorPayloads) {
|
|
2667
|
+
try {
|
|
2668
|
+
const res = await request(backendUrl, {
|
|
2669
|
+
method: "POST",
|
|
2670
|
+
headers: {
|
|
2671
|
+
"Content-Type": "application/json",
|
|
2672
|
+
Accept: "application/json, text/event-stream",
|
|
2673
|
+
...buildHeaders()
|
|
2674
|
+
},
|
|
2675
|
+
body: payload,
|
|
2676
|
+
signal: AbortSignal.timeout(timeout)
|
|
2677
|
+
});
|
|
2678
|
+
const text = await res.body.text();
|
|
2679
|
+
errorResponses.push(text);
|
|
2680
|
+
} catch {
|
|
2681
|
+
}
|
|
2682
|
+
}
|
|
2683
|
+
const issues = [];
|
|
2684
|
+
for (const text of errorResponses) {
|
|
2685
|
+
for (const pattern of STACK_TRACE_PATTERNS) {
|
|
2686
|
+
if (pattern.test(text)) {
|
|
2687
|
+
issues.push(`Response contains: ${pattern.source} (matched in: ${text.substring(0, 80)}...)`);
|
|
2688
|
+
break;
|
|
2689
|
+
}
|
|
2690
|
+
}
|
|
2691
|
+
}
|
|
2692
|
+
if (issues.length > 0) return { passed: false, details: issues.slice(0, 3).join("; ") };
|
|
2693
|
+
return {
|
|
2694
|
+
passed: true,
|
|
2695
|
+
details: `${errorResponses.length} error responses checked \u2014 no stack traces or sensitive data found`
|
|
2696
|
+
};
|
|
2697
|
+
}
|
|
2698
|
+
);
|
|
2699
|
+
await test(
|
|
2700
|
+
"security-error-no-internal-ip",
|
|
2701
|
+
"Error responses do not leak internal IPs",
|
|
2702
|
+
"security",
|
|
2703
|
+
false,
|
|
2704
|
+
"basic",
|
|
2705
|
+
async () => {
|
|
2706
|
+
try {
|
|
2707
|
+
const res = await request(backendUrl, {
|
|
2708
|
+
method: "POST",
|
|
2709
|
+
headers: {
|
|
2710
|
+
"Content-Type": "application/json",
|
|
2711
|
+
Accept: "application/json, text/event-stream",
|
|
2712
|
+
...buildHeaders()
|
|
2713
|
+
},
|
|
2714
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: nextId(), method: "___trigger_error___" }),
|
|
2715
|
+
signal: AbortSignal.timeout(timeout)
|
|
2716
|
+
});
|
|
2717
|
+
const text = await res.body.text();
|
|
2718
|
+
for (const pattern of INTERNAL_IP_PATTERNS) {
|
|
2719
|
+
const match = text.match(pattern);
|
|
2720
|
+
if (match) {
|
|
2721
|
+
return { passed: false, details: `Error response contains internal IP: ${match[0]}` };
|
|
2722
|
+
}
|
|
2723
|
+
}
|
|
2724
|
+
return { passed: true, details: "No internal IP addresses found in error responses" };
|
|
2725
|
+
} catch {
|
|
2726
|
+
return { passed: true, details: "No response to check (connection error)" };
|
|
2727
|
+
}
|
|
2728
|
+
}
|
|
2729
|
+
);
|
|
2730
|
+
await test(
|
|
2731
|
+
"security-rate-limiting",
|
|
2732
|
+
"Rate limiting is enforced",
|
|
2733
|
+
"security",
|
|
2734
|
+
false,
|
|
2735
|
+
"basic/transports#streamable-http",
|
|
2736
|
+
async () => {
|
|
2737
|
+
const burstSize = 50;
|
|
2738
|
+
let got429 = false;
|
|
2739
|
+
const promises = Array.from(
|
|
2740
|
+
{ length: burstSize },
|
|
2741
|
+
() => mcpRequest(backendUrl, "ping", void 0, nextId, buildHeaders(), timeout).then((res) => {
|
|
2742
|
+
if (res.statusCode === 429) got429 = true;
|
|
2743
|
+
return res.statusCode;
|
|
2744
|
+
}).catch(() => 0)
|
|
2745
|
+
);
|
|
2746
|
+
const statusCodes = await Promise.all(promises);
|
|
2747
|
+
if (got429) {
|
|
2748
|
+
return { passed: true, details: `Rate limiting detected (429 returned after ${burstSize} rapid requests)` };
|
|
2749
|
+
}
|
|
2750
|
+
const errorCount = statusCodes.filter((c) => c >= 500).length;
|
|
2751
|
+
if (errorCount > burstSize / 2) {
|
|
2752
|
+
return {
|
|
2753
|
+
passed: false,
|
|
2754
|
+
details: `Server returned ${errorCount}/${burstSize} 5xx errors under load \u2014 should return 429 instead of crashing`
|
|
2755
|
+
};
|
|
2756
|
+
}
|
|
2757
|
+
return {
|
|
2758
|
+
passed: false,
|
|
2759
|
+
details: `No rate limiting detected (${burstSize} rapid requests all returned ${[...new Set(statusCodes)].join(",")})`
|
|
2760
|
+
};
|
|
2761
|
+
}
|
|
2762
|
+
);
|
|
2763
|
+
await test(
|
|
2764
|
+
"transport-delete",
|
|
2765
|
+
"DELETE accepted or returns 405",
|
|
2766
|
+
"transport",
|
|
2767
|
+
false,
|
|
2768
|
+
"basic/transports#streamable-http",
|
|
2769
|
+
async () => {
|
|
2770
|
+
const deleteHeaders = { ...buildHeaders() };
|
|
2771
|
+
const res = await request(backendUrl, {
|
|
2772
|
+
method: "DELETE",
|
|
2773
|
+
headers: deleteHeaders,
|
|
2774
|
+
signal: AbortSignal.timeout(timeout)
|
|
2775
|
+
});
|
|
2776
|
+
await res.body.text();
|
|
2777
|
+
if (res.statusCode === 405) {
|
|
2778
|
+
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
2779
|
+
}
|
|
2780
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
2781
|
+
if (sessionId) {
|
|
2782
|
+
try {
|
|
2783
|
+
const verifyRes = await mcpRequest(
|
|
2784
|
+
backendUrl,
|
|
2785
|
+
"ping",
|
|
2786
|
+
void 0,
|
|
2787
|
+
createIdCounter(99920),
|
|
2788
|
+
deleteHeaders,
|
|
2789
|
+
timeout
|
|
2790
|
+
);
|
|
2791
|
+
if (verifyRes.statusCode === 400 || verifyRes.statusCode === 404 || verifyRes.statusCode === 409) {
|
|
2792
|
+
return {
|
|
2793
|
+
passed: true,
|
|
2794
|
+
details: `HTTP ${res.statusCode} (session terminated, post-delete request correctly rejected with ${verifyRes.statusCode})`
|
|
2795
|
+
};
|
|
2796
|
+
}
|
|
2797
|
+
} catch {
|
|
2798
|
+
return {
|
|
2799
|
+
passed: true,
|
|
2800
|
+
details: `HTTP ${res.statusCode} (session terminated, post-delete request rejected)`
|
|
2801
|
+
};
|
|
2802
|
+
}
|
|
2803
|
+
}
|
|
2804
|
+
return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
|
|
2805
|
+
}
|
|
2806
|
+
if (res.statusCode === 400 || res.statusCode === 404) {
|
|
2807
|
+
return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
|
|
2808
|
+
}
|
|
2809
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
2810
|
+
}
|
|
2811
|
+
);
|
|
2812
|
+
const MAX_WARNINGS = 50;
|
|
2813
|
+
if (warnings.length > MAX_WARNINGS) {
|
|
2814
|
+
const truncated = warnings.length - MAX_WARNINGS;
|
|
2815
|
+
warnings.splice(MAX_WARNINGS, truncated, `... and ${truncated} more warning(s) suppressed`);
|
|
2816
|
+
}
|
|
2817
|
+
const { score, grade, overall, summary, categories } = computeScore(tests);
|
|
2818
|
+
const badge = generateBadge(url);
|
|
2819
|
+
return {
|
|
2820
|
+
specVersion: SPEC_VERSION,
|
|
2821
|
+
toolVersion: TOOL_VERSION,
|
|
2822
|
+
url,
|
|
2823
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2824
|
+
score,
|
|
2825
|
+
grade,
|
|
2826
|
+
overall,
|
|
2827
|
+
summary,
|
|
2828
|
+
categories,
|
|
2829
|
+
tests,
|
|
2830
|
+
warnings,
|
|
2831
|
+
serverInfo,
|
|
2832
|
+
toolCount,
|
|
2833
|
+
toolNames,
|
|
2834
|
+
resourceCount,
|
|
2835
|
+
resourceNames,
|
|
2836
|
+
promptCount,
|
|
2837
|
+
promptNames,
|
|
2838
|
+
badge
|
|
2839
|
+
};
|
|
2840
|
+
}
|
|
2841
|
+
|
|
2842
|
+
// src/mcp/tools.ts
|
|
2843
|
+
function registerTools(server) {
|
|
1632
2844
|
server.tool(
|
|
1633
2845
|
"mcp_compliance_test",
|
|
1634
|
-
"Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all
|
|
2846
|
+
"Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all 69 tests covering transport, lifecycle, tools, resources, prompts, errors, schema validation, and security.",
|
|
1635
2847
|
{
|
|
1636
2848
|
url: z.string().url().describe("The MCP server URL to test (must be HTTP or HTTPS)"),
|
|
1637
2849
|
auth: z.string().optional().describe('Authorization header value (e.g., "Bearer tok123")'),
|
|
@@ -1687,8 +2899,9 @@ ${JSON.stringify(report, null, 2)}` }
|
|
|
1687
2899
|
]
|
|
1688
2900
|
};
|
|
1689
2901
|
} catch (err) {
|
|
2902
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1690
2903
|
return {
|
|
1691
|
-
content: [{ type: "text", text: `Error running compliance test: ${
|
|
2904
|
+
content: [{ type: "text", text: `Error running compliance test: ${message}` }],
|
|
1692
2905
|
isError: true
|
|
1693
2906
|
};
|
|
1694
2907
|
}
|
|
@@ -1736,8 +2949,9 @@ ${JSON.stringify(report, null, 2)}` }
|
|
|
1736
2949
|
]
|
|
1737
2950
|
};
|
|
1738
2951
|
} catch (err) {
|
|
2952
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1739
2953
|
return {
|
|
1740
|
-
content: [{ type: "text", text: `Error: ${
|
|
2954
|
+
content: [{ type: "text", text: `Error: ${message}` }],
|
|
1741
2955
|
isError: true
|
|
1742
2956
|
};
|
|
1743
2957
|
}
|
|
@@ -1781,7 +2995,7 @@ ${TEST_DEFINITIONS.map((t) => t.id).join(", ")}`
|
|
|
1781
2995
|
`Name: ${def.name}`,
|
|
1782
2996
|
`Category: ${def.category}`,
|
|
1783
2997
|
`Required: ${def.required ? "Yes" : "No"}`,
|
|
1784
|
-
`Spec reference:
|
|
2998
|
+
`Spec reference: ${SPEC_BASE}/${def.specRef}`,
|
|
1785
2999
|
"",
|
|
1786
3000
|
def.description,
|
|
1787
3001
|
"",
|
|
@@ -1824,9 +3038,10 @@ var CATEGORY_LABELS = {
|
|
|
1824
3038
|
resources: "Resources",
|
|
1825
3039
|
prompts: "Prompts",
|
|
1826
3040
|
errors: "Error Handling",
|
|
1827
|
-
schema: "Schema Validation"
|
|
3041
|
+
schema: "Schema Validation",
|
|
3042
|
+
security: "Security"
|
|
1828
3043
|
};
|
|
1829
|
-
var CATEGORY_ORDER = ["transport", "lifecycle", "tools", "resources", "prompts", "errors", "schema"];
|
|
3044
|
+
var CATEGORY_ORDER = ["transport", "lifecycle", "tools", "resources", "prompts", "errors", "schema", "security"];
|
|
1830
3045
|
function gradeColor(grade) {
|
|
1831
3046
|
switch (grade) {
|
|
1832
3047
|
case "A":
|
|
@@ -1950,7 +3165,6 @@ function formatJson(report) {
|
|
|
1950
3165
|
return JSON.stringify(report, null, 2);
|
|
1951
3166
|
}
|
|
1952
3167
|
function formatSarif(report) {
|
|
1953
|
-
const SPEC_BASE2 = `https://modelcontextprotocol.io/specification/${report.specVersion}`;
|
|
1954
3168
|
const rules = report.tests.map((t) => {
|
|
1955
3169
|
const def = TEST_DEFINITIONS.find((d) => d.id === t.id);
|
|
1956
3170
|
return {
|
|
@@ -1958,7 +3172,7 @@ function formatSarif(report) {
|
|
|
1958
3172
|
name: t.name,
|
|
1959
3173
|
shortDescription: { text: t.name },
|
|
1960
3174
|
fullDescription: { text: def?.description || t.details },
|
|
1961
|
-
helpUri: t.specRef || `${
|
|
3175
|
+
helpUri: t.specRef || `${SPEC_BASE}/basic`,
|
|
1962
3176
|
properties: {
|
|
1963
3177
|
category: t.category,
|
|
1964
3178
|
required: t.required
|
|
@@ -2010,7 +3224,13 @@ function formatSarif(report) {
|
|
|
2010
3224
|
grade: report.grade,
|
|
2011
3225
|
score: report.score,
|
|
2012
3226
|
overall: report.overall,
|
|
2013
|
-
specVersion: report.specVersion
|
|
3227
|
+
specVersion: report.specVersion,
|
|
3228
|
+
serverUrl: report.url,
|
|
3229
|
+
serverName: report.serverInfo.name,
|
|
3230
|
+
serverVersion: report.serverInfo.version,
|
|
3231
|
+
protocolVersion: report.serverInfo.protocolVersion,
|
|
3232
|
+
testsPassed: report.summary.passed,
|
|
3233
|
+
testsTotal: report.summary.total
|
|
2014
3234
|
}
|
|
2015
3235
|
}
|
|
2016
3236
|
]
|
|
@@ -2033,12 +3253,27 @@ function parseHeaderArg(value, prev) {
|
|
|
2033
3253
|
prev[key] = val;
|
|
2034
3254
|
return prev;
|
|
2035
3255
|
}
|
|
3256
|
+
function parsePositiveInt(value, name, min = 0) {
|
|
3257
|
+
const n = Number.parseInt(value, 10);
|
|
3258
|
+
if (Number.isNaN(n) || n < min) {
|
|
3259
|
+
throw new Error(`${name} must be an integer >= ${min}, got "${value}"`);
|
|
3260
|
+
}
|
|
3261
|
+
return n;
|
|
3262
|
+
}
|
|
2036
3263
|
function parseList(value) {
|
|
2037
3264
|
return value.split(",").map((s) => s.trim()).filter(Boolean);
|
|
2038
3265
|
}
|
|
2039
3266
|
var program = new Command();
|
|
2040
3267
|
program.name("mcp-compliance").description("Test MCP servers for spec compliance").version(version2);
|
|
2041
|
-
program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal, json, or sarif", "terminal").option("--strict", "Exit with code 1 on any required test failure (for CI)").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").option("--retries <n>", "Number of retries for failed tests", "0").option(
|
|
3268
|
+
program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal, json, or sarif", "terminal").option("--strict", "Exit with code 1 on any required test failure (for CI)").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").option("--retries <n>", "Number of retries for failed tests", "0").option(
|
|
3269
|
+
"--only <items>",
|
|
3270
|
+
'Only run matching categories or test IDs, comma-separated (e.g., "transport,lifecycle" or "transport-post,lifecycle-init")',
|
|
3271
|
+
parseList
|
|
3272
|
+
).option(
|
|
3273
|
+
"--skip <items>",
|
|
3274
|
+
'Skip matching categories or test IDs, comma-separated (e.g., "schema" or "tools-pagination")',
|
|
3275
|
+
parseList
|
|
3276
|
+
).option("--verbose", "Print each test result as it runs").action(
|
|
2042
3277
|
async (url, opts) => {
|
|
2043
3278
|
try {
|
|
2044
3279
|
const headers = { ...opts.header };
|
|
@@ -2050,8 +3285,8 @@ Testing ${url}...
|
|
|
2050
3285
|
}
|
|
2051
3286
|
const report = await runComplianceSuite(url, {
|
|
2052
3287
|
headers,
|
|
2053
|
-
timeout:
|
|
2054
|
-
retries:
|
|
3288
|
+
timeout: parsePositiveInt(opts.timeout, "--timeout", 1),
|
|
3289
|
+
retries: parsePositiveInt(opts.retries, "--retries"),
|
|
2055
3290
|
only: opts.only,
|
|
2056
3291
|
skip: opts.skip,
|
|
2057
3292
|
onProgress: opts.verbose ? (testId, passed, details) => {
|
|
@@ -2073,11 +3308,12 @@ Testing ${url}...
|
|
|
2073
3308
|
process.exit(1);
|
|
2074
3309
|
}
|
|
2075
3310
|
} catch (err) {
|
|
3311
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2076
3312
|
if (opts.format === "json" || opts.format === "sarif") {
|
|
2077
|
-
console.error(JSON.stringify({ error:
|
|
3313
|
+
console.error(JSON.stringify({ error: message }));
|
|
2078
3314
|
} else {
|
|
2079
3315
|
console.error(chalk2.red(`
|
|
2080
|
-
Error: ${
|
|
3316
|
+
Error: ${message}
|
|
2081
3317
|
`));
|
|
2082
3318
|
}
|
|
2083
3319
|
process.exit(1);
|
|
@@ -2093,15 +3329,16 @@ Testing ${url}...
|
|
|
2093
3329
|
`));
|
|
2094
3330
|
const report = await runComplianceSuite(url, {
|
|
2095
3331
|
headers,
|
|
2096
|
-
timeout:
|
|
3332
|
+
timeout: parsePositiveInt(opts.timeout, "--timeout", 1)
|
|
2097
3333
|
});
|
|
2098
3334
|
console.log(`Grade: ${report.grade} (${report.score}%)
|
|
2099
3335
|
`);
|
|
2100
3336
|
console.log(report.badge.markdown);
|
|
2101
3337
|
console.log("");
|
|
2102
3338
|
} catch (err) {
|
|
3339
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2103
3340
|
console.error(chalk2.red(`
|
|
2104
|
-
Error: ${
|
|
3341
|
+
Error: ${message}
|
|
2105
3342
|
`));
|
|
2106
3343
|
process.exit(1);
|
|
2107
3344
|
}
|