@yawlabs/mcp-compliance 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{chunk-KNOSZ3TD.js → chunk-Z7VLPYIO.js} +343 -62
- package/dist/index.js +378 -76
- package/dist/mcp/server.js +8 -5
- package/dist/runner.d.ts +12 -2
- package/dist/runner.js +7 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -15,7 +15,7 @@ MCP servers are multiplying fast — but most ship without compliance testing. B
|
|
|
15
15
|
|
|
16
16
|
This tool solves that:
|
|
17
17
|
|
|
18
|
-
- **
|
|
18
|
+
- **48 tests across 7 categories** — transport, lifecycle, tools, resources, prompts, error handling, and schema validation. No gaps.
|
|
19
19
|
- **Capability-driven** — tests adapt to what the server declares. If it says it supports tools, tool tests become required. No false failures for features the server doesn't claim.
|
|
20
20
|
- **Graded scoring** — A-F letter grade with a weighted score (required tests 70%, optional 30%). One number to communicate compliance.
|
|
21
21
|
- **CI-ready** — `--strict` mode exits with code 1 on required test failures. Drop it into any pipeline.
|
|
@@ -100,7 +100,7 @@ mcp-compliance badge https://my-server.com/mcp
|
|
|
100
100
|
|
|
101
101
|
Outputs the markdown embed for a compliance badge hosted at [mcp.hosting](https://mcp.hosting).
|
|
102
102
|
|
|
103
|
-
## What the
|
|
103
|
+
## What the 48 tests check
|
|
104
104
|
|
|
105
105
|
<details>
|
|
106
106
|
<summary><strong>Transport (7 tests)</strong></summary>
|
|
@@ -3,16 +3,14 @@ import { createRequire } from "module";
|
|
|
3
3
|
import { request } from "undici";
|
|
4
4
|
|
|
5
5
|
// src/badge.ts
|
|
6
|
+
import { createHash } from "crypto";
|
|
7
|
+
function urlHash(url) {
|
|
8
|
+
return createHash("sha256").update(url).digest("hex").slice(0, 12);
|
|
9
|
+
}
|
|
6
10
|
function generateBadge(url) {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
} catch {
|
|
11
|
-
parsed = new URL("https://unknown");
|
|
12
|
-
}
|
|
13
|
-
const encoded = encodeURIComponent(parsed.href);
|
|
14
|
-
const imageUrl = `https://mcp.hosting/api/compliance/${encoded}/badge`;
|
|
15
|
-
const reportUrl = `https://mcp.hosting/compliance/${encoded}`;
|
|
11
|
+
const hash = urlHash(url);
|
|
12
|
+
const imageUrl = `https://mcp.hosting/api/compliance/${hash}/badge`;
|
|
13
|
+
const reportUrl = `https://mcp.hosting/compliance/${hash}`;
|
|
16
14
|
return {
|
|
17
15
|
imageUrl,
|
|
18
16
|
reportUrl,
|
|
@@ -58,7 +56,7 @@ function computeScore(tests) {
|
|
|
58
56
|
|
|
59
57
|
// src/types.ts
|
|
60
58
|
var TEST_DEFINITIONS = [
|
|
61
|
-
// ── Transport (
|
|
59
|
+
// ── Transport (10 tests) ─────────────────────────────────────────
|
|
62
60
|
{
|
|
63
61
|
id: "transport-post",
|
|
64
62
|
name: "HTTP POST accepted",
|
|
@@ -122,7 +120,34 @@ var TEST_DEFINITIONS = [
|
|
|
122
120
|
description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests.",
|
|
123
121
|
recommendation: "Check if the parsed JSON body is an array. If so, return a JSON-RPC error or HTTP 400. Do not process batch requests \u2014 MCP explicitly forbids them."
|
|
124
122
|
},
|
|
125
|
-
|
|
123
|
+
{
|
|
124
|
+
id: "transport-content-type-init",
|
|
125
|
+
name: "Initialize response has valid content type",
|
|
126
|
+
category: "transport",
|
|
127
|
+
required: false,
|
|
128
|
+
specRef: "basic/transports#streamable-http",
|
|
129
|
+
description: "Validates that the initialize response uses application/json or text/event-stream content type. Some servers return other types for the handshake.",
|
|
130
|
+
recommendation: 'Ensure the initialize response uses Content-Type "application/json" or "text/event-stream". Do not return text/html or other types for JSON-RPC responses.'
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
id: "transport-get-stream",
|
|
134
|
+
name: "GET with session returns SSE or 405",
|
|
135
|
+
category: "transport",
|
|
136
|
+
required: false,
|
|
137
|
+
specRef: "basic/transports#streamable-http",
|
|
138
|
+
description: "Tests the GET endpoint with an active session ID for server-initiated messages. After initialization, the server should either return an SSE stream or 405.",
|
|
139
|
+
recommendation: "If your server supports server-initiated messages, return text/event-stream on GET with a valid session ID. Otherwise, return 405 Method Not Allowed."
|
|
140
|
+
},
|
|
141
|
+
{
|
|
142
|
+
id: "transport-concurrent",
|
|
143
|
+
name: "Handles concurrent requests",
|
|
144
|
+
category: "transport",
|
|
145
|
+
required: false,
|
|
146
|
+
specRef: "basic/transports#streamable-http",
|
|
147
|
+
description: "Sends multiple JSON-RPC requests in parallel and verifies the server responds to all with correct matching IDs. Tests that the server can handle concurrent connections.",
|
|
148
|
+
recommendation: "Ensure your server can handle multiple simultaneous requests. Each response must include the correct id matching the request. Use async handlers or connection pooling."
|
|
149
|
+
},
|
|
150
|
+
// ── Lifecycle (12 tests) ─────────────────────────────────────────
|
|
126
151
|
{
|
|
127
152
|
id: "lifecycle-init",
|
|
128
153
|
name: "Initialize handshake",
|
|
@@ -213,6 +238,24 @@ var TEST_DEFINITIONS = [
|
|
|
213
238
|
description: "If the server declares completions capability, tests that the completion/complete method is accepted.",
|
|
214
239
|
recommendation: 'If you declare completions in capabilities, implement the "completion/complete" handler. Return a completion object with a values array, even if empty.'
|
|
215
240
|
},
|
|
241
|
+
{
|
|
242
|
+
id: "lifecycle-cancellation",
|
|
243
|
+
name: "Handles cancellation notifications",
|
|
244
|
+
category: "lifecycle",
|
|
245
|
+
required: false,
|
|
246
|
+
specRef: "basic/utilities#cancellation",
|
|
247
|
+
description: "Tests that the server accepts notifications/cancelled without error. Servers should gracefully handle cancellation of unknown or completed requests.",
|
|
248
|
+
recommendation: "Accept notifications/cancelled and stop any in-progress work for the referenced requestId. If the request is unknown or already complete, silently ignore the cancellation."
|
|
249
|
+
},
|
|
250
|
+
{
|
|
251
|
+
id: "lifecycle-progress",
|
|
252
|
+
name: "Accepts progress notifications",
|
|
253
|
+
category: "lifecycle",
|
|
254
|
+
required: false,
|
|
255
|
+
specRef: "basic/utilities#progress",
|
|
256
|
+
description: "Tests that the server accepts notifications/progress without error. Servers should handle progress notifications for request tracking.",
|
|
257
|
+
recommendation: "Accept notifications/progress with progressToken, progress, and optional total fields. Ignore notifications for unknown progress tokens."
|
|
258
|
+
},
|
|
216
259
|
// ── Tools (4 tests) ──────────────────────────────────────────────
|
|
217
260
|
{
|
|
218
261
|
id: "tools-list",
|
|
@@ -554,19 +597,39 @@ async function mcpNotification(backendUrl, method, params, extraHeaders, timeout
|
|
|
554
597
|
return { statusCode: res.statusCode, headers: responseHeaders };
|
|
555
598
|
}
|
|
556
599
|
async function runComplianceSuite(url, options = {}) {
|
|
557
|
-
let parsed;
|
|
558
600
|
try {
|
|
559
|
-
parsed = new URL(url);
|
|
601
|
+
const parsed = new URL(url);
|
|
560
602
|
if (!["http:", "https:"].includes(parsed.protocol)) {
|
|
561
603
|
throw new Error("Only HTTP and HTTPS URLs are supported");
|
|
562
604
|
}
|
|
563
605
|
} catch (e) {
|
|
564
|
-
if (e.message.includes("Only HTTP")) throw e;
|
|
606
|
+
if (e instanceof Error && e.message.includes("Only HTTP")) throw e;
|
|
565
607
|
throw new Error(`Invalid URL: ${url}`);
|
|
566
608
|
}
|
|
567
609
|
const backendUrl = url;
|
|
610
|
+
let serverReachable = true;
|
|
611
|
+
try {
|
|
612
|
+
const preflight = await request(backendUrl, {
|
|
613
|
+
method: "POST",
|
|
614
|
+
headers: {
|
|
615
|
+
"Content-Type": "application/json",
|
|
616
|
+
Accept: "application/json, text/event-stream",
|
|
617
|
+
...options.headers
|
|
618
|
+
},
|
|
619
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 0, method: "ping" }),
|
|
620
|
+
signal: AbortSignal.timeout(Math.min(options.timeout || 15e3, 1e4))
|
|
621
|
+
});
|
|
622
|
+
await preflight.body.text();
|
|
623
|
+
} catch {
|
|
624
|
+
serverReachable = false;
|
|
625
|
+
}
|
|
568
626
|
const tests = [];
|
|
569
627
|
const warnings = [];
|
|
628
|
+
if (!serverReachable) {
|
|
629
|
+
warnings.push(
|
|
630
|
+
`Server at ${url} is unreachable \u2014 all tests will fail. Check the URL and ensure the server is running.`
|
|
631
|
+
);
|
|
632
|
+
}
|
|
570
633
|
const nextId = createIdCounter(1e3);
|
|
571
634
|
const timeout = options.timeout || 15e3;
|
|
572
635
|
const retries = options.retries || 0;
|
|
@@ -611,7 +674,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
611
674
|
if (lastResult.passed) break;
|
|
612
675
|
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
|
|
613
676
|
} catch (err) {
|
|
614
|
-
|
|
677
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
678
|
+
lastResult = { passed: false, details: `Error: ${message}` };
|
|
615
679
|
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
|
|
616
680
|
}
|
|
617
681
|
}
|
|
@@ -640,10 +704,23 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
640
704
|
body: JSON.stringify({ jsonrpc: "2.0", id: 99901, method: "ping" }),
|
|
641
705
|
signal: AbortSignal.timeout(timeout)
|
|
642
706
|
});
|
|
643
|
-
await res.body.text();
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
707
|
+
const text = await res.body.text();
|
|
708
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
709
|
+
return { passed: true, details: `HTTP ${res.statusCode}` };
|
|
710
|
+
}
|
|
711
|
+
if (res.statusCode === 401 || res.statusCode === 403) {
|
|
712
|
+
return { passed: false, details: `HTTP ${res.statusCode} (auth required \u2014 pass --auth)` };
|
|
713
|
+
}
|
|
714
|
+
if (res.statusCode === 400) {
|
|
715
|
+
try {
|
|
716
|
+
const body = JSON.parse(text);
|
|
717
|
+
if (body?.error || body?.jsonrpc) {
|
|
718
|
+
return { passed: true, details: "HTTP 400 with JSON-RPC response (server requires initialization first)" };
|
|
719
|
+
}
|
|
720
|
+
} catch {
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
647
724
|
}
|
|
648
725
|
);
|
|
649
726
|
await test(
|
|
@@ -673,18 +750,29 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
673
750
|
false,
|
|
674
751
|
"basic/transports#streamable-http",
|
|
675
752
|
async () => {
|
|
753
|
+
const getHeaders = { Accept: "text/event-stream", ...buildHeaders() };
|
|
676
754
|
const res = await request(backendUrl, {
|
|
677
755
|
method: "GET",
|
|
678
|
-
headers:
|
|
756
|
+
headers: getHeaders,
|
|
679
757
|
signal: AbortSignal.timeout(timeout)
|
|
680
758
|
});
|
|
681
|
-
await res.body.text();
|
|
759
|
+
const body = await res.body.text();
|
|
682
760
|
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
683
761
|
if (res.statusCode === 405) {
|
|
684
762
|
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
685
763
|
}
|
|
686
764
|
if (ct.includes("text/event-stream")) {
|
|
687
|
-
|
|
765
|
+
if (body.trim().length > 0) {
|
|
766
|
+
const hasDataFields = body.includes("data:");
|
|
767
|
+
const hasEventFields = body.includes("event:");
|
|
768
|
+
if (!hasDataFields && !hasEventFields) {
|
|
769
|
+
return {
|
|
770
|
+
passed: false,
|
|
771
|
+
details: "Content-Type is text/event-stream but body has no SSE data: or event: fields"
|
|
772
|
+
};
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
return { passed: true, details: "Returns text/event-stream with valid SSE format" };
|
|
688
776
|
}
|
|
689
777
|
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
690
778
|
return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
|
|
@@ -692,31 +780,6 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
692
780
|
return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
|
|
693
781
|
}
|
|
694
782
|
);
|
|
695
|
-
await test(
|
|
696
|
-
"transport-delete",
|
|
697
|
-
"DELETE accepted or returns 405",
|
|
698
|
-
"transport",
|
|
699
|
-
false,
|
|
700
|
-
"basic/transports#streamable-http",
|
|
701
|
-
async () => {
|
|
702
|
-
const res = await request(backendUrl, {
|
|
703
|
-
method: "DELETE",
|
|
704
|
-
headers: { ...userHeaders },
|
|
705
|
-
signal: AbortSignal.timeout(timeout)
|
|
706
|
-
});
|
|
707
|
-
await res.body.text();
|
|
708
|
-
if (res.statusCode === 405) {
|
|
709
|
-
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
710
|
-
}
|
|
711
|
-
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
712
|
-
return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
|
|
713
|
-
}
|
|
714
|
-
if (res.statusCode === 400 || res.statusCode === 404) {
|
|
715
|
-
return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
|
|
716
|
-
}
|
|
717
|
-
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
718
|
-
}
|
|
719
|
-
);
|
|
720
783
|
await test(
|
|
721
784
|
"transport-batch-reject",
|
|
722
785
|
"Rejects JSON-RPC batch requests",
|
|
@@ -754,7 +817,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
754
817
|
try {
|
|
755
818
|
initRes = await rpc("initialize", {
|
|
756
819
|
protocolVersion: SPEC_VERSION,
|
|
757
|
-
capabilities: {
|
|
820
|
+
capabilities: {},
|
|
758
821
|
clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
|
|
759
822
|
});
|
|
760
823
|
const result = initRes?.body?.result;
|
|
@@ -767,7 +830,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
767
830
|
if (sid) sessionId = sid;
|
|
768
831
|
if (result.protocolVersion) negotiatedProtocolVersion = result.protocolVersion;
|
|
769
832
|
}
|
|
770
|
-
} catch
|
|
833
|
+
} catch {
|
|
771
834
|
}
|
|
772
835
|
try {
|
|
773
836
|
await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders(), timeout);
|
|
@@ -886,7 +949,17 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
886
949
|
if (res.body?.error) {
|
|
887
950
|
return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
|
|
888
951
|
}
|
|
889
|
-
|
|
952
|
+
const invalidRes = await rpc("logging/setLevel", { level: "__invalid_level__" });
|
|
953
|
+
const validatesInput = !!invalidRes.body?.error;
|
|
954
|
+
const validLevels = ["debug", "warning", "error"];
|
|
955
|
+
const accepted = [];
|
|
956
|
+
for (const level of validLevels) {
|
|
957
|
+
const r = await rpc("logging/setLevel", { level });
|
|
958
|
+
if (!r.body?.error) accepted.push(level);
|
|
959
|
+
}
|
|
960
|
+
const details = validatesInput ? `logging/setLevel accepted (validates levels, ${accepted.length + 1} levels accepted)` : "logging/setLevel accepted (warning: server does not reject invalid log levels)";
|
|
961
|
+
if (!validatesInput) warnings.push("Server accepts invalid log levels without error");
|
|
962
|
+
return { passed: true, details };
|
|
890
963
|
}
|
|
891
964
|
);
|
|
892
965
|
const hasCompletions = !!serverInfo.capabilities.completions;
|
|
@@ -915,6 +988,59 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
915
988
|
return { passed: true, details: "completion/complete accepted" };
|
|
916
989
|
}
|
|
917
990
|
);
|
|
991
|
+
await test(
|
|
992
|
+
"lifecycle-cancellation",
|
|
993
|
+
"Handles cancellation notifications",
|
|
994
|
+
"lifecycle",
|
|
995
|
+
false,
|
|
996
|
+
"basic/utilities#cancellation",
|
|
997
|
+
async () => {
|
|
998
|
+
const res = await mcpNotification(
|
|
999
|
+
backendUrl,
|
|
1000
|
+
"notifications/cancelled",
|
|
1001
|
+
{ requestId: 99999, reason: "compliance test" },
|
|
1002
|
+
buildHeaders(),
|
|
1003
|
+
timeout
|
|
1004
|
+
);
|
|
1005
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1006
|
+
return { passed: true, details: `HTTP ${res.statusCode} (cancellation accepted)` };
|
|
1007
|
+
}
|
|
1008
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should accept cancellation notifications` };
|
|
1009
|
+
}
|
|
1010
|
+
);
|
|
1011
|
+
await test(
|
|
1012
|
+
"lifecycle-progress",
|
|
1013
|
+
"Accepts progress notifications",
|
|
1014
|
+
"lifecycle",
|
|
1015
|
+
false,
|
|
1016
|
+
"basic/utilities#progress",
|
|
1017
|
+
async () => {
|
|
1018
|
+
const res = await mcpNotification(
|
|
1019
|
+
backendUrl,
|
|
1020
|
+
"notifications/progress",
|
|
1021
|
+
{ progressToken: "compliance-test-token", progress: 50, total: 100 },
|
|
1022
|
+
buildHeaders(),
|
|
1023
|
+
timeout
|
|
1024
|
+
);
|
|
1025
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1026
|
+
return { passed: true, details: `HTTP ${res.statusCode} (progress notification accepted)` };
|
|
1027
|
+
}
|
|
1028
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should accept progress notifications` };
|
|
1029
|
+
}
|
|
1030
|
+
);
|
|
1031
|
+
await test(
|
|
1032
|
+
"transport-content-type-init",
|
|
1033
|
+
"Initialize response has valid content type",
|
|
1034
|
+
"transport",
|
|
1035
|
+
false,
|
|
1036
|
+
"basic/transports#streamable-http",
|
|
1037
|
+
async () => {
|
|
1038
|
+
if (!initRes) return { passed: false, details: "No init response to check" };
|
|
1039
|
+
const ct = (initRes.headers["content-type"] || "").toLowerCase();
|
|
1040
|
+
const valid = ct.includes("application/json") || ct.includes("text/event-stream");
|
|
1041
|
+
return { passed: valid, details: `Content-Type: ${ct || "missing"}` };
|
|
1042
|
+
}
|
|
1043
|
+
);
|
|
918
1044
|
await test(
|
|
919
1045
|
"transport-notification-202",
|
|
920
1046
|
"Notification returns 202 Accepted",
|
|
@@ -972,6 +1098,88 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
972
1098
|
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
973
1099
|
}
|
|
974
1100
|
);
|
|
1101
|
+
await test(
|
|
1102
|
+
"transport-get-stream",
|
|
1103
|
+
"GET with session returns SSE or 405",
|
|
1104
|
+
"transport",
|
|
1105
|
+
false,
|
|
1106
|
+
"basic/transports#streamable-http",
|
|
1107
|
+
async () => {
|
|
1108
|
+
if (!sessionId) {
|
|
1109
|
+
return { passed: true, details: "No session ID \u2014 server-initiated messages not applicable" };
|
|
1110
|
+
}
|
|
1111
|
+
const res = await request(backendUrl, {
|
|
1112
|
+
method: "GET",
|
|
1113
|
+
headers: { Accept: "text/event-stream", ...buildHeaders() },
|
|
1114
|
+
signal: AbortSignal.timeout(Math.min(timeout, 3e3))
|
|
1115
|
+
});
|
|
1116
|
+
const body = await res.body.text();
|
|
1117
|
+
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
1118
|
+
if (res.statusCode === 405) {
|
|
1119
|
+
return { passed: true, details: "HTTP 405 (server does not support server-initiated messages)" };
|
|
1120
|
+
}
|
|
1121
|
+
if (ct.includes("text/event-stream")) {
|
|
1122
|
+
if (body.trim().length > 0) {
|
|
1123
|
+
const hasSSEFields = body.includes("data:") || body.includes("event:");
|
|
1124
|
+
if (!hasSSEFields) {
|
|
1125
|
+
return { passed: false, details: "Content-Type is text/event-stream but body has no SSE fields" };
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
return { passed: true, details: "GET with session returns SSE stream for server-initiated messages" };
|
|
1129
|
+
}
|
|
1130
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1131
|
+
return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
|
|
1132
|
+
}
|
|
1133
|
+
return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
|
|
1134
|
+
}
|
|
1135
|
+
);
|
|
1136
|
+
await test(
|
|
1137
|
+
"transport-concurrent",
|
|
1138
|
+
"Handles concurrent requests",
|
|
1139
|
+
"transport",
|
|
1140
|
+
false,
|
|
1141
|
+
"basic/transports#streamable-http",
|
|
1142
|
+
async () => {
|
|
1143
|
+
const ids = [createIdCounter(99930)(), createIdCounter(99931)(), createIdCounter(99932)()];
|
|
1144
|
+
const promises = ids.map(
|
|
1145
|
+
(id) => request(backendUrl, {
|
|
1146
|
+
method: "POST",
|
|
1147
|
+
headers: {
|
|
1148
|
+
"Content-Type": "application/json",
|
|
1149
|
+
Accept: "application/json, text/event-stream",
|
|
1150
|
+
...buildHeaders()
|
|
1151
|
+
},
|
|
1152
|
+
body: JSON.stringify({ jsonrpc: "2.0", id, method: "ping" }),
|
|
1153
|
+
signal: AbortSignal.timeout(timeout)
|
|
1154
|
+
}).then(async (res) => {
|
|
1155
|
+
const text = await res.body.text();
|
|
1156
|
+
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
1157
|
+
let body;
|
|
1158
|
+
if (ct.includes("text/event-stream")) {
|
|
1159
|
+
body = parseSSEResponse(text);
|
|
1160
|
+
}
|
|
1161
|
+
if (!body) {
|
|
1162
|
+
try {
|
|
1163
|
+
body = JSON.parse(text);
|
|
1164
|
+
} catch {
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
return { statusCode: res.statusCode, body, requestId: id };
|
|
1168
|
+
})
|
|
1169
|
+
);
|
|
1170
|
+
const results = await Promise.all(promises);
|
|
1171
|
+
const issues = [];
|
|
1172
|
+
for (const r of results) {
|
|
1173
|
+
if (r.statusCode < 200 || r.statusCode >= 300) {
|
|
1174
|
+
issues.push(`Request id=${r.requestId}: HTTP ${r.statusCode}`);
|
|
1175
|
+
} else if (r.body?.id !== r.requestId) {
|
|
1176
|
+
issues.push(`Request id=${r.requestId}: response id=${r.body?.id} (mismatch)`);
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1180
|
+
return { passed: true, details: `${results.length} concurrent requests handled correctly` };
|
|
1181
|
+
}
|
|
1182
|
+
);
|
|
975
1183
|
const hasTools = !!serverInfo.capabilities.tools;
|
|
976
1184
|
let cachedToolsList = null;
|
|
977
1185
|
await test(
|
|
@@ -993,6 +1201,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
993
1201
|
};
|
|
994
1202
|
}
|
|
995
1203
|
);
|
|
1204
|
+
const toolsListOk = cachedToolsList !== null;
|
|
996
1205
|
await test(
|
|
997
1206
|
"tools-schema",
|
|
998
1207
|
"All tools have name and inputSchema",
|
|
@@ -1000,7 +1209,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1000
1209
|
hasTools,
|
|
1001
1210
|
"server/tools#data-types",
|
|
1002
1211
|
async () => {
|
|
1003
|
-
|
|
1212
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1213
|
+
const tools = cachedToolsList ?? [];
|
|
1004
1214
|
const issues = [];
|
|
1005
1215
|
for (const tool of tools) {
|
|
1006
1216
|
if (!tool.name) {
|
|
@@ -1032,7 +1242,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1032
1242
|
false,
|
|
1033
1243
|
"server/tools#annotations",
|
|
1034
1244
|
async () => {
|
|
1035
|
-
|
|
1245
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1246
|
+
const tools = cachedToolsList ?? [];
|
|
1036
1247
|
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1037
1248
|
const issues = [];
|
|
1038
1249
|
let annotatedCount = 0;
|
|
@@ -1062,7 +1273,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1062
1273
|
}
|
|
1063
1274
|
);
|
|
1064
1275
|
await test("tools-title-field", "Tools include title field", "schema", false, "server/tools#data-types", async () => {
|
|
1065
|
-
|
|
1276
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1277
|
+
const tools = cachedToolsList ?? [];
|
|
1066
1278
|
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1067
1279
|
const withTitle = tools.filter((t) => typeof t.title === "string");
|
|
1068
1280
|
const issues = [];
|
|
@@ -1084,7 +1296,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1084
1296
|
false,
|
|
1085
1297
|
"server/tools#structured-content",
|
|
1086
1298
|
async () => {
|
|
1087
|
-
|
|
1299
|
+
if (!toolsListOk) return { passed: false, details: "Skipped: tools/list failed" };
|
|
1300
|
+
const tools = cachedToolsList ?? [];
|
|
1088
1301
|
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
1089
1302
|
const issues = [];
|
|
1090
1303
|
let withSchema = 0;
|
|
@@ -1125,14 +1338,14 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1125
1338
|
return { passed: true, details: `Protocol error: code ${code} \u2014 ${error.message}` };
|
|
1126
1339
|
}
|
|
1127
1340
|
if (result?.content && Array.isArray(result.content)) {
|
|
1341
|
+
if (result.isError) {
|
|
1342
|
+
return { passed: true, details: "Tool returned execution error with content (valid)" };
|
|
1343
|
+
}
|
|
1128
1344
|
const badItems = result.content.filter((c) => !c.type);
|
|
1129
1345
|
if (badItems.length > 0)
|
|
1130
1346
|
return { passed: false, details: `${badItems.length} content item(s) missing 'type' field` };
|
|
1131
1347
|
return { passed: true, details: `Returned ${result.content.length} content item(s)` };
|
|
1132
1348
|
}
|
|
1133
|
-
if (result?.isError && result?.content && Array.isArray(result.content)) {
|
|
1134
|
-
return { passed: true, details: "Tool returned execution error with content (valid)" };
|
|
1135
|
-
}
|
|
1136
1349
|
return { passed: false, details: "Response missing content array" };
|
|
1137
1350
|
}
|
|
1138
1351
|
);
|
|
@@ -1234,6 +1447,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1234
1447
|
return { passed: true, details: `${resourceCount} resource(s)` };
|
|
1235
1448
|
}
|
|
1236
1449
|
);
|
|
1450
|
+
const resourcesListOk = cachedResourcesList !== null;
|
|
1237
1451
|
await test(
|
|
1238
1452
|
"resources-schema",
|
|
1239
1453
|
"Resources have uri and name",
|
|
@@ -1241,7 +1455,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1241
1455
|
true,
|
|
1242
1456
|
"server/resources#data-types",
|
|
1243
1457
|
async () => {
|
|
1244
|
-
|
|
1458
|
+
if (!resourcesListOk) return { passed: false, details: "Skipped: resources/list failed" };
|
|
1459
|
+
const resources = cachedResourcesList ?? [];
|
|
1245
1460
|
const issues = [];
|
|
1246
1461
|
for (const r of resources) {
|
|
1247
1462
|
if (!r.uri) issues.push("Resource missing uri");
|
|
@@ -1270,7 +1485,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1270
1485
|
false,
|
|
1271
1486
|
"server/resources#reading-resources",
|
|
1272
1487
|
async () => {
|
|
1273
|
-
const resources = cachedResourcesList ??
|
|
1488
|
+
const resources = cachedResourcesList ?? [];
|
|
1274
1489
|
const firstUri = resources[0]?.uri;
|
|
1275
1490
|
if (!firstUri) return { passed: false, details: "No resource URI to test" };
|
|
1276
1491
|
const readRes = await rpc("resources/read", { uri: firstUri });
|
|
@@ -1303,8 +1518,15 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1303
1518
|
if (!Array.isArray(templates)) return { passed: false, details: "No resourceTemplates array" };
|
|
1304
1519
|
const issues = [];
|
|
1305
1520
|
for (const t of templates) {
|
|
1306
|
-
if (!t.uriTemplate)
|
|
1521
|
+
if (!t.uriTemplate) {
|
|
1522
|
+
issues.push("Template missing uriTemplate");
|
|
1523
|
+
} else if (typeof t.uriTemplate !== "string") {
|
|
1524
|
+
issues.push(`uriTemplate should be a string, got ${typeof t.uriTemplate}`);
|
|
1525
|
+
} else if (!t.uriTemplate.includes("{") || !t.uriTemplate.includes("}")) {
|
|
1526
|
+
warnings.push(`Template "${t.name || t.uriTemplate}" has no URI template parameters (e.g., {id})`);
|
|
1527
|
+
}
|
|
1307
1528
|
if (!t.name) issues.push(`${t.uriTemplate || "?"}: missing name`);
|
|
1529
|
+
if (!t.description) warnings.push(`Template "${t.name || t.uriTemplate || "?"}" missing description`);
|
|
1308
1530
|
}
|
|
1309
1531
|
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
1310
1532
|
return { passed: true, details: `${templates.length} resource template(s)` };
|
|
@@ -1346,7 +1568,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1346
1568
|
true,
|
|
1347
1569
|
"server/resources#subscriptions",
|
|
1348
1570
|
async () => {
|
|
1349
|
-
const resources = cachedResourcesList ??
|
|
1571
|
+
const resources = cachedResourcesList ?? [];
|
|
1350
1572
|
const firstUri = resources[0]?.uri;
|
|
1351
1573
|
if (!firstUri) return { passed: false, details: "No resource URI for subscribe test" };
|
|
1352
1574
|
const subRes = await rpc("resources/subscribe", { uri: firstUri });
|
|
@@ -1390,8 +1612,10 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1390
1612
|
};
|
|
1391
1613
|
}
|
|
1392
1614
|
);
|
|
1615
|
+
const promptsListOk = cachedPromptsList !== null;
|
|
1393
1616
|
await test("prompts-schema", "Prompts have name field", "schema", true, "server/prompts#data-types", async () => {
|
|
1394
|
-
|
|
1617
|
+
if (!promptsListOk) return { passed: false, details: "Skipped: prompts/list failed" };
|
|
1618
|
+
const prompts = cachedPromptsList ?? [];
|
|
1395
1619
|
const issues = [];
|
|
1396
1620
|
for (const p of prompts) {
|
|
1397
1621
|
if (!p.name) issues.push("Prompt missing name");
|
|
@@ -1587,6 +1811,60 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
1587
1811
|
}
|
|
1588
1812
|
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error code -32600` };
|
|
1589
1813
|
});
|
|
1814
|
+
await test(
|
|
1815
|
+
"transport-delete",
|
|
1816
|
+
"DELETE accepted or returns 405",
|
|
1817
|
+
"transport",
|
|
1818
|
+
false,
|
|
1819
|
+
"basic/transports#streamable-http",
|
|
1820
|
+
async () => {
|
|
1821
|
+
const deleteHeaders = { ...buildHeaders() };
|
|
1822
|
+
const res = await request(backendUrl, {
|
|
1823
|
+
method: "DELETE",
|
|
1824
|
+
headers: deleteHeaders,
|
|
1825
|
+
signal: AbortSignal.timeout(timeout)
|
|
1826
|
+
});
|
|
1827
|
+
await res.body.text();
|
|
1828
|
+
if (res.statusCode === 405) {
|
|
1829
|
+
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
1830
|
+
}
|
|
1831
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
1832
|
+
if (sessionId) {
|
|
1833
|
+
try {
|
|
1834
|
+
const verifyRes = await mcpRequest(
|
|
1835
|
+
backendUrl,
|
|
1836
|
+
"ping",
|
|
1837
|
+
void 0,
|
|
1838
|
+
createIdCounter(99920),
|
|
1839
|
+
deleteHeaders,
|
|
1840
|
+
timeout
|
|
1841
|
+
);
|
|
1842
|
+
if (verifyRes.statusCode === 400 || verifyRes.statusCode === 404 || verifyRes.statusCode === 409) {
|
|
1843
|
+
return {
|
|
1844
|
+
passed: true,
|
|
1845
|
+
details: `HTTP ${res.statusCode} (session terminated, post-delete request correctly rejected with ${verifyRes.statusCode})`
|
|
1846
|
+
};
|
|
1847
|
+
}
|
|
1848
|
+
} catch {
|
|
1849
|
+
return {
|
|
1850
|
+
passed: true,
|
|
1851
|
+
details: `HTTP ${res.statusCode} (session terminated, post-delete request rejected)`
|
|
1852
|
+
};
|
|
1853
|
+
}
|
|
1854
|
+
}
|
|
1855
|
+
return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
|
|
1856
|
+
}
|
|
1857
|
+
if (res.statusCode === 400 || res.statusCode === 404) {
|
|
1858
|
+
return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
|
|
1859
|
+
}
|
|
1860
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
1861
|
+
}
|
|
1862
|
+
);
|
|
1863
|
+
const MAX_WARNINGS = 50;
|
|
1864
|
+
if (warnings.length > MAX_WARNINGS) {
|
|
1865
|
+
const truncated = warnings.length - MAX_WARNINGS;
|
|
1866
|
+
warnings.splice(MAX_WARNINGS, truncated, `... and ${truncated} more warning(s) suppressed`);
|
|
1867
|
+
}
|
|
1590
1868
|
const { score, grade, overall, summary, categories } = computeScore(tests);
|
|
1591
1869
|
const badge = generateBadge(url);
|
|
1592
1870
|
return {
|
|
@@ -1617,5 +1895,8 @@ export {
|
|
|
1617
1895
|
computeGrade,
|
|
1618
1896
|
computeScore,
|
|
1619
1897
|
TEST_DEFINITIONS,
|
|
1898
|
+
SPEC_VERSION,
|
|
1899
|
+
SPEC_BASE,
|
|
1900
|
+
parseSSEResponse,
|
|
1620
1901
|
runComplianceSuite
|
|
1621
1902
|
};
|