@yawlabs/mcp-compliance 0.1.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -28
- package/dist/chunk-SP24UFRC.js +987 -0
- package/dist/index.js +695 -85
- package/dist/mcp/server.js +110 -101
- package/dist/runner.d.ts +13 -1
- package/dist/runner.js +1 -1
- package/package.json +2 -2
- package/dist/chunk-OOJ4PMF7.js +0 -563
package/dist/index.js
CHANGED
|
@@ -3,10 +3,11 @@
|
|
|
3
3
|
// src/index.ts
|
|
4
4
|
import { Command } from "commander";
|
|
5
5
|
import chalk2 from "chalk";
|
|
6
|
-
import { createRequire } from "module";
|
|
6
|
+
import { createRequire as createRequire2 } from "module";
|
|
7
7
|
|
|
8
8
|
// src/runner.ts
|
|
9
9
|
import { request } from "undici";
|
|
10
|
+
import { createRequire } from "module";
|
|
10
11
|
|
|
11
12
|
// src/grader.ts
|
|
12
13
|
function computeGrade(score) {
|
|
@@ -62,33 +63,99 @@ function generateBadge(url) {
|
|
|
62
63
|
};
|
|
63
64
|
}
|
|
64
65
|
|
|
66
|
+
// src/types.ts
|
|
67
|
+
var TEST_DEFINITIONS = [
|
|
68
|
+
// ── Transport (7 tests) ──────────────────────────────────────────
|
|
69
|
+
{ id: "transport-post", name: "HTTP POST accepted", category: "transport", required: true, specRef: "basic/transports#streamable-http", description: "Verifies the server accepts HTTP POST requests and returns a 2xx status code. This is the fundamental transport requirement for Streamable HTTP MCP servers." },
|
|
70
|
+
{ id: "transport-content-type", name: "Responds with JSON or SSE", category: "transport", required: true, specRef: "basic/transports#streamable-http", description: "Checks that the server responds with Content-Type application/json or text/event-stream. MCP servers must use one of these two content types." },
|
|
71
|
+
{ id: "transport-notification-202", name: "Notification returns 202 Accepted", category: "transport", required: false, specRef: "basic/transports#streamable-http", description: "Verifies that sending a JSON-RPC notification (no id field) returns HTTP 202 Accepted with no body. Per spec, servers MUST return 202 for notifications." },
|
|
72
|
+
{ id: "transport-session-id", name: "Enforces MCP-Session-Id after init", category: "transport", required: false, specRef: "basic/transports#streamable-http", description: "Tests that the server returns HTTP 400 when MCP-Session-Id header is missing on requests after initialization (when the server issued a session ID)." },
|
|
73
|
+
{ id: "transport-get", name: "GET returns SSE stream or 405", category: "transport", required: false, specRef: "basic/transports#streamable-http", description: "Tests the GET endpoint for server-initiated messages. Server should return text/event-stream or 405 Method Not Allowed." },
|
|
74
|
+
{ id: "transport-delete", name: "DELETE accepted or returns 405", category: "transport", required: false, specRef: "basic/transports#streamable-http", description: "Tests the DELETE endpoint for session termination. Server should accept the request or return 405 Method Not Allowed." },
|
|
75
|
+
{ id: "transport-batch-reject", name: "Rejects JSON-RPC batch requests", category: "transport", required: true, specRef: "basic/transports#streamable-http", description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests." },
|
|
76
|
+
// ── Lifecycle (10 tests) ─────────────────────────────────────────
|
|
77
|
+
{ id: "lifecycle-init", name: "Initialize handshake", category: "lifecycle", required: true, specRef: "basic/lifecycle#initialization", description: "Tests the initialize handshake by sending an initialize request with client capabilities. The server must return a result with protocolVersion." },
|
|
78
|
+
{ id: "lifecycle-proto-version", name: "Returns valid protocol version", category: "lifecycle", required: true, specRef: "basic/lifecycle#version-negotiation", description: "Validates that the protocolVersion returned by the server matches the YYYY-MM-DD date format required by the spec." },
|
|
79
|
+
{ id: "lifecycle-server-info", name: "Includes serverInfo", category: "lifecycle", required: false, specRef: "basic/lifecycle#initialization", description: "Checks that the server includes a serverInfo object with at least a name field in its initialize response. While recommended, this is not strictly required." },
|
|
80
|
+
{ id: "lifecycle-capabilities", name: "Returns capabilities object", category: "lifecycle", required: true, specRef: "basic/lifecycle#capability-negotiation", description: "Verifies the server returns a capabilities object in its initialize response. An empty object is valid (no optional features declared)." },
|
|
81
|
+
{ id: "lifecycle-jsonrpc", name: "Response is valid JSON-RPC 2.0", category: "lifecycle", required: true, specRef: "basic", description: 'Validates that the initialize response is a proper JSON-RPC 2.0 message with jsonrpc="2.0", an id field, and either a result or error field.' },
|
|
82
|
+
{ id: "lifecycle-ping", name: "Responds to ping", category: "lifecycle", required: true, specRef: "basic/utilities#ping", description: "Tests that the server responds to the ping method with an empty result object. This is a required utility method." },
|
|
83
|
+
{ id: "lifecycle-instructions", name: "Instructions field is valid", category: "lifecycle", required: false, specRef: "basic/lifecycle#initialization", description: "If the server includes an instructions field in the initialize response, validates it is a string. Instructions provide guidance for how the client should interact with the server." },
|
|
84
|
+
{ id: "lifecycle-id-match", name: "Response ID matches request ID", category: "lifecycle", required: true, specRef: "basic", description: "Verifies that the JSON-RPC response id matches the request id sent by the client. This is a fundamental JSON-RPC 2.0 requirement." },
|
|
85
|
+
{ id: "lifecycle-logging", name: "logging/setLevel accepted", category: "lifecycle", required: false, specRef: "server/utilities#logging", description: "If the server declares logging capability, tests that logging/setLevel method is accepted with a valid log level." },
|
|
86
|
+
{ id: "lifecycle-completions", name: "completion/complete accepted", category: "lifecycle", required: false, specRef: "server/utilities#completion", description: "If the server declares completions capability, tests that the completion/complete method is accepted." },
|
|
87
|
+
// ── Tools (4 tests) ──────────────────────────────────────────────
|
|
88
|
+
{ id: "tools-list", name: "tools/list returns valid response", category: "tools", required: false, specRef: "server/tools#listing-tools", description: "Calls tools/list and validates it returns an array of tool definitions. Required if the server declares tools capability." },
|
|
89
|
+
{ id: "tools-call", name: "tools/call responds correctly", category: "tools", required: false, specRef: "server/tools#calling-tools", description: "Calls the first tool with empty arguments and verifies the response format. Accepts both successful results and InvalidParams errors." },
|
|
90
|
+
{ id: "tools-pagination", name: "tools/list supports pagination", category: "tools", required: false, specRef: "server/tools#listing-tools", description: "Tests cursor-based pagination on tools/list. Validates nextCursor is a string if present and that fetching the next page returns a valid response." },
|
|
91
|
+
{ id: "tools-content-types", name: "Tool content items have valid types", category: "tools", required: false, specRef: "server/tools#calling-tools", description: "Validates that content items returned by tools/call have a recognized type field (text, image, audio, resource, resource_link)." },
|
|
92
|
+
// ── Resources (5 tests) ──────────────────────────────────────────
|
|
93
|
+
{ id: "resources-list", name: "resources/list returns valid response", category: "resources", required: false, specRef: "server/resources#listing-resources", description: "Calls resources/list and validates it returns an array. Required if the server declares resources capability." },
|
|
94
|
+
{ id: "resources-read", name: "resources/read returns content", category: "resources", required: false, specRef: "server/resources#reading-resources", description: "Reads the first resource and validates the response contains a contents array with proper uri and text/blob fields." },
|
|
95
|
+
{ id: "resources-templates", name: "resources/templates/list returns valid response", category: "resources", required: false, specRef: "server/resources#resource-templates", description: "Tests the resource templates endpoint. Accepts Method not found (-32601) since templates are optional." },
|
|
96
|
+
{ id: "resources-pagination", name: "resources/list supports pagination", category: "resources", required: false, specRef: "server/resources#listing-resources", description: "Tests cursor-based pagination on resources/list. Validates nextCursor is a string if present and that fetching the next page works." },
|
|
97
|
+
{ id: "resources-subscribe", name: "Resource subscribe/unsubscribe", category: "resources", required: false, specRef: "server/resources#subscriptions", description: "If the server declares resources.subscribe capability, tests that resources/subscribe and resources/unsubscribe methods are accepted." },
|
|
98
|
+
// ── Prompts (3 tests) ────────────────────────────────────────────
|
|
99
|
+
{ id: "prompts-list", name: "prompts/list returns valid response", category: "prompts", required: false, specRef: "server/prompts#listing-prompts", description: "Calls prompts/list and validates it returns an array. Required if the server declares prompts capability." },
|
|
100
|
+
{ id: "prompts-get", name: "prompts/get returns valid messages", category: "prompts", required: false, specRef: "server/prompts#getting-a-prompt", description: "Gets the first prompt and validates the response contains a messages array with proper role and content fields." },
|
|
101
|
+
{ id: "prompts-pagination", name: "prompts/list supports pagination", category: "prompts", required: false, specRef: "server/prompts#listing-prompts", description: "Tests cursor-based pagination on prompts/list. Validates nextCursor is a string if present and that fetching the next page works." },
|
|
102
|
+
// ── Error Handling (8 tests) ─────────────────────────────────────
|
|
103
|
+
{ id: "error-unknown-method", name: "Returns JSON-RPC error for unknown method", category: "errors", required: true, specRef: "basic", description: "Sends an unknown method and verifies the server returns a JSON-RPC error. The spec requires error code -32601 (Method not found)." },
|
|
104
|
+
{ id: "error-method-code", name: "Uses correct JSON-RPC error code for unknown method", category: "errors", required: false, specRef: "basic", description: "Checks the error code is specifically -32601 (Method not found) for unknown methods, as required by JSON-RPC 2.0." },
|
|
105
|
+
{ id: "error-invalid-jsonrpc", name: "Handles malformed JSON-RPC", category: "errors", required: true, specRef: "basic", description: "Sends a malformed JSON-RPC message (missing required fields) and verifies the server returns an error or 4xx status." },
|
|
106
|
+
{ id: "error-invalid-json", name: "Handles invalid JSON body", category: "errors", required: false, specRef: "basic", description: "Sends invalid JSON and verifies the server returns a parse error (-32700) or 4xx status code." },
|
|
107
|
+
{ id: "error-missing-params", name: "Returns error for tools/call without name", category: "errors", required: false, specRef: "server/tools#error-handling", description: "Calls tools/call with an empty params object (missing required name field) and verifies an error is returned." },
|
|
108
|
+
{ id: "error-parse-code", name: "Returns -32700 for invalid JSON", category: "errors", required: false, specRef: "basic", description: "Checks that the server returns the specific JSON-RPC error code -32700 (Parse error) when receiving invalid JSON, as required by the JSON-RPC 2.0 specification." },
|
|
109
|
+
{ id: "error-invalid-request-code", name: "Returns -32600 for invalid request", category: "errors", required: false, specRef: "basic", description: "Checks that the server returns the specific JSON-RPC error code -32600 (Invalid Request) for malformed JSON-RPC messages missing required fields." },
|
|
110
|
+
{ id: "tools-call-unknown", name: "Returns error for unknown tool name", category: "errors", required: false, specRef: "server/tools#error-handling", description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response." },
|
|
111
|
+
// ── Schema Validation (6 tests) ──────────────────────────────────
|
|
112
|
+
{ id: "tools-schema", name: "All tools have name and inputSchema", category: "schema", required: false, specRef: "server/tools#data-types", description: 'Validates every tool has a valid name (1-128 chars, alphanumeric/underscore/hyphen/dot) and a required inputSchema of type "object".' },
|
|
113
|
+
{ id: "tools-annotations", name: "Tool annotations are valid", category: "schema", required: false, specRef: "server/tools#annotations", description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans; title should be a string." },
|
|
114
|
+
{ id: "tools-title-field", name: "Tools include title field", category: "schema", required: false, specRef: "server/tools#data-types", description: "Checks if tools include the optional title field for human-readable display names. Added in spec version 2025-11-25." },
|
|
115
|
+
{ id: "tools-output-schema", name: "Tools with outputSchema are valid", category: "schema", required: false, specRef: "server/tools#structured-content", description: 'If tools declare an outputSchema, validates it is a valid JSON Schema object with type "object". Used for structured output validation.' },
|
|
116
|
+
{ id: "prompts-schema", name: "Prompts have name field", category: "schema", required: false, specRef: "server/prompts#data-types", description: "Validates every prompt has a name and that any arguments array contains items with name fields." },
|
|
117
|
+
{ id: "resources-schema", name: "Resources have uri and name", category: "schema", required: false, specRef: "server/resources#data-types", description: "Validates every resource has a valid URI (parseable as a URL) and a name field." }
|
|
118
|
+
];
|
|
119
|
+
|
|
65
120
|
// src/runner.ts
|
|
121
|
+
var _require = createRequire(import.meta.url);
|
|
122
|
+
var { version: TOOL_VERSION } = _require("../package.json");
|
|
66
123
|
var SPEC_VERSION = "2025-11-25";
|
|
67
124
|
var SPEC_BASE = `https://modelcontextprotocol.io/specification/${SPEC_VERSION}`;
|
|
68
|
-
|
|
69
|
-
|
|
125
|
+
var VALID_CONTENT_TYPES = ["text", "image", "audio", "resource", "resource_link"];
|
|
126
|
+
function createIdCounter(start = 0) {
|
|
127
|
+
let id = start;
|
|
70
128
|
return () => ++id;
|
|
71
129
|
}
|
|
72
130
|
function parseSSEResponse(text) {
|
|
73
131
|
const lines = text.split("\n");
|
|
74
132
|
let lastJsonRpcResponse = null;
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
133
|
+
let currentData = [];
|
|
134
|
+
function flushEvent() {
|
|
135
|
+
if (currentData.length === 0) return;
|
|
136
|
+
const data = currentData.join("\n");
|
|
137
|
+
currentData = [];
|
|
138
|
+
if (!data.trim()) return;
|
|
139
|
+
try {
|
|
140
|
+
const parsed = JSON.parse(data);
|
|
141
|
+
if (parsed.jsonrpc === "2.0" && parsed.id !== void 0) {
|
|
142
|
+
lastJsonRpcResponse = parsed;
|
|
85
143
|
}
|
|
144
|
+
} catch {
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
for (const line of lines) {
|
|
148
|
+
if (line.startsWith("data:")) {
|
|
149
|
+
const content = line.slice(5);
|
|
150
|
+
currentData.push(content.startsWith(" ") ? content.slice(1) : content);
|
|
151
|
+
} else if (line.trim() === "") {
|
|
152
|
+
flushEvent();
|
|
86
153
|
}
|
|
87
154
|
}
|
|
155
|
+
flushEvent();
|
|
88
156
|
return lastJsonRpcResponse;
|
|
89
157
|
}
|
|
90
|
-
|
|
91
|
-
async function mcpRequest(backendUrl, method, params, nextId = _defaultNextId, extraHeaders) {
|
|
158
|
+
async function mcpRequest(backendUrl, method, params, nextId, extraHeaders, timeout) {
|
|
92
159
|
const id = nextId();
|
|
93
160
|
const body = JSON.stringify({
|
|
94
161
|
jsonrpc: "2.0",
|
|
@@ -105,7 +172,7 @@ async function mcpRequest(backendUrl, method, params, nextId = _defaultNextId, e
|
|
|
105
172
|
method: "POST",
|
|
106
173
|
headers,
|
|
107
174
|
body,
|
|
108
|
-
signal: AbortSignal.timeout(
|
|
175
|
+
signal: AbortSignal.timeout(timeout)
|
|
109
176
|
});
|
|
110
177
|
const text = await res.body.text();
|
|
111
178
|
const responseHeaders = {};
|
|
@@ -116,32 +183,38 @@ async function mcpRequest(backendUrl, method, params, nextId = _defaultNextId, e
|
|
|
116
183
|
if (contentType.includes("text/event-stream")) {
|
|
117
184
|
const parsed = parseSSEResponse(text);
|
|
118
185
|
if (parsed) {
|
|
119
|
-
return { statusCode: res.statusCode, body: parsed, headers: responseHeaders };
|
|
186
|
+
return { statusCode: res.statusCode, body: parsed, headers: responseHeaders, requestId: id };
|
|
120
187
|
}
|
|
121
188
|
try {
|
|
122
|
-
return { statusCode: res.statusCode, body: JSON.parse(text), headers: responseHeaders };
|
|
189
|
+
return { statusCode: res.statusCode, body: JSON.parse(text), headers: responseHeaders, requestId: id };
|
|
123
190
|
} catch {
|
|
124
|
-
return { statusCode: res.statusCode, body: { _raw: text }, headers: responseHeaders };
|
|
191
|
+
return { statusCode: res.statusCode, body: { _raw: text }, headers: responseHeaders, requestId: id };
|
|
125
192
|
}
|
|
126
193
|
}
|
|
127
194
|
try {
|
|
128
|
-
return { statusCode: res.statusCode, body: JSON.parse(text), headers: responseHeaders };
|
|
195
|
+
return { statusCode: res.statusCode, body: JSON.parse(text), headers: responseHeaders, requestId: id };
|
|
129
196
|
} catch {
|
|
130
|
-
return { statusCode: res.statusCode, body: { _raw: text }, headers: responseHeaders };
|
|
197
|
+
return { statusCode: res.statusCode, body: { _raw: text }, headers: responseHeaders, requestId: id };
|
|
131
198
|
}
|
|
132
199
|
}
|
|
133
|
-
async function mcpNotification(backendUrl, method, params, extraHeaders) {
|
|
200
|
+
async function mcpNotification(backendUrl, method, params, extraHeaders, timeout) {
|
|
134
201
|
const headers = {
|
|
135
202
|
"Content-Type": "application/json",
|
|
203
|
+
"Accept": "application/json, text/event-stream",
|
|
136
204
|
...extraHeaders
|
|
137
205
|
};
|
|
138
|
-
await request(backendUrl, {
|
|
206
|
+
const res = await request(backendUrl, {
|
|
139
207
|
method: "POST",
|
|
140
208
|
headers,
|
|
141
209
|
body: JSON.stringify({ jsonrpc: "2.0", method, ...params ? { params } : {} }),
|
|
142
|
-
signal: AbortSignal.timeout(
|
|
143
|
-
}).then((r) => r.body.text()).catch(() => {
|
|
210
|
+
signal: AbortSignal.timeout(timeout)
|
|
144
211
|
});
|
|
212
|
+
await res.body.text();
|
|
213
|
+
const responseHeaders = {};
|
|
214
|
+
for (const [k, v] of Object.entries(res.headers)) {
|
|
215
|
+
if (typeof v === "string") responseHeaders[k] = v;
|
|
216
|
+
}
|
|
217
|
+
return { statusCode: res.statusCode, headers: responseHeaders };
|
|
145
218
|
}
|
|
146
219
|
async function runComplianceSuite(url, options = {}) {
|
|
147
220
|
let parsed;
|
|
@@ -156,7 +229,10 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
156
229
|
}
|
|
157
230
|
const backendUrl = url;
|
|
158
231
|
const tests = [];
|
|
159
|
-
const
|
|
232
|
+
const warnings = [];
|
|
233
|
+
const nextId = createIdCounter(1e3);
|
|
234
|
+
const timeout = options.timeout || 15e3;
|
|
235
|
+
const retries = options.retries || 0;
|
|
160
236
|
let sessionId = null;
|
|
161
237
|
let negotiatedProtocolVersion = null;
|
|
162
238
|
const userHeaders = options.headers || {};
|
|
@@ -166,7 +242,16 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
166
242
|
if (negotiatedProtocolVersion) h["mcp-protocol-version"] = negotiatedProtocolVersion;
|
|
167
243
|
return h;
|
|
168
244
|
}
|
|
169
|
-
const rpc = (method, params) => mcpRequest(backendUrl, method, params, nextId, buildHeaders());
|
|
245
|
+
const rpc = (method, params) => mcpRequest(backendUrl, method, params, nextId, buildHeaders(), timeout);
|
|
246
|
+
function shouldRun(id, category) {
|
|
247
|
+
if (options.only && options.only.length > 0) {
|
|
248
|
+
return options.only.includes(category) || options.only.includes(id);
|
|
249
|
+
}
|
|
250
|
+
if (options.skip && options.skip.length > 0) {
|
|
251
|
+
return !options.skip.includes(category) && !options.skip.includes(id);
|
|
252
|
+
}
|
|
253
|
+
return true;
|
|
254
|
+
}
|
|
170
255
|
let serverInfo = {
|
|
171
256
|
protocolVersion: null,
|
|
172
257
|
name: null,
|
|
@@ -176,42 +261,41 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
176
261
|
let toolCount = 0;
|
|
177
262
|
let toolNames = [];
|
|
178
263
|
let resourceCount = 0;
|
|
264
|
+
let resourceNames = [];
|
|
179
265
|
let promptCount = 0;
|
|
266
|
+
let promptNames = [];
|
|
180
267
|
async function test(id, name, category, required, specRef, fn) {
|
|
268
|
+
if (!shouldRun(id, category)) return;
|
|
181
269
|
const start = Date.now();
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
passed:
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
specRef: `${SPEC_BASE}/${specRef}`
|
|
193
|
-
});
|
|
194
|
-
options.onProgress?.(id, result.passed, result.details);
|
|
195
|
-
} catch (err) {
|
|
196
|
-
tests.push({
|
|
197
|
-
id,
|
|
198
|
-
name,
|
|
199
|
-
category,
|
|
200
|
-
required,
|
|
201
|
-
passed: false,
|
|
202
|
-
details: `Error: ${err.message}`,
|
|
203
|
-
durationMs: Date.now() - start,
|
|
204
|
-
specRef: `${SPEC_BASE}/${specRef}`
|
|
205
|
-
});
|
|
206
|
-
options.onProgress?.(id, false, `Error: ${err.message}`);
|
|
270
|
+
let lastResult = { passed: false, details: "" };
|
|
271
|
+
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
272
|
+
try {
|
|
273
|
+
lastResult = await fn();
|
|
274
|
+
if (lastResult.passed) break;
|
|
275
|
+
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
|
|
276
|
+
} catch (err) {
|
|
277
|
+
lastResult = { passed: false, details: `Error: ${err.message}` };
|
|
278
|
+
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
|
|
279
|
+
}
|
|
207
280
|
}
|
|
281
|
+
tests.push({
|
|
282
|
+
id,
|
|
283
|
+
name,
|
|
284
|
+
category,
|
|
285
|
+
required,
|
|
286
|
+
passed: lastResult.passed,
|
|
287
|
+
details: lastResult.details,
|
|
288
|
+
durationMs: Date.now() - start,
|
|
289
|
+
specRef: `${SPEC_BASE}/${specRef}`
|
|
290
|
+
});
|
|
291
|
+
options.onProgress?.(id, lastResult.passed, lastResult.details);
|
|
208
292
|
}
|
|
209
293
|
await test("transport-post", "HTTP POST accepted", "transport", true, "basic/transports#streamable-http", async () => {
|
|
210
294
|
const res = await request(backendUrl, {
|
|
211
295
|
method: "POST",
|
|
212
296
|
headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...userHeaders },
|
|
213
|
-
body: JSON.stringify({ jsonrpc: "2.0", id:
|
|
214
|
-
signal: AbortSignal.timeout(
|
|
297
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 99901, method: "ping" }),
|
|
298
|
+
signal: AbortSignal.timeout(timeout)
|
|
215
299
|
});
|
|
216
300
|
await res.body.text();
|
|
217
301
|
const passed = res.statusCode >= 200 && res.statusCode < 300;
|
|
@@ -222,8 +306,8 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
222
306
|
const res = await request(backendUrl, {
|
|
223
307
|
method: "POST",
|
|
224
308
|
headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...userHeaders },
|
|
225
|
-
body: JSON.stringify({ jsonrpc: "2.0", id:
|
|
226
|
-
signal: AbortSignal.timeout(
|
|
309
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 99902, method: "ping" }),
|
|
310
|
+
signal: AbortSignal.timeout(timeout)
|
|
227
311
|
});
|
|
228
312
|
await res.body.text();
|
|
229
313
|
const rawCt = res.headers["content-type"];
|
|
@@ -231,28 +315,105 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
231
315
|
const valid = ct.includes("application/json") || ct.includes("text/event-stream");
|
|
232
316
|
return { passed: valid, details: `Content-Type: ${ct}` };
|
|
233
317
|
});
|
|
318
|
+
await test("transport-get", "GET returns SSE stream or 405", "transport", false, "basic/transports#streamable-http", async () => {
|
|
319
|
+
const res = await request(backendUrl, {
|
|
320
|
+
method: "GET",
|
|
321
|
+
headers: { "Accept": "text/event-stream", ...userHeaders },
|
|
322
|
+
signal: AbortSignal.timeout(timeout)
|
|
323
|
+
});
|
|
324
|
+
await res.body.text();
|
|
325
|
+
const ct = (res.headers["content-type"] || "").toLowerCase();
|
|
326
|
+
if (res.statusCode === 405) {
|
|
327
|
+
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
328
|
+
}
|
|
329
|
+
if (ct.includes("text/event-stream")) {
|
|
330
|
+
return { passed: true, details: "Returns text/event-stream for SSE" };
|
|
331
|
+
}
|
|
332
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
333
|
+
return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
|
|
334
|
+
}
|
|
335
|
+
return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
|
|
336
|
+
});
|
|
337
|
+
await test("transport-delete", "DELETE accepted or returns 405", "transport", false, "basic/transports#streamable-http", async () => {
|
|
338
|
+
const res = await request(backendUrl, {
|
|
339
|
+
method: "DELETE",
|
|
340
|
+
headers: { ...userHeaders },
|
|
341
|
+
signal: AbortSignal.timeout(timeout)
|
|
342
|
+
});
|
|
343
|
+
await res.body.text();
|
|
344
|
+
if (res.statusCode === 405) {
|
|
345
|
+
return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
|
|
346
|
+
}
|
|
347
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
348
|
+
return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
|
|
349
|
+
}
|
|
350
|
+
if (res.statusCode === 400 || res.statusCode === 404) {
|
|
351
|
+
return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
|
|
352
|
+
}
|
|
353
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
354
|
+
});
|
|
355
|
+
await test("transport-batch-reject", "Rejects JSON-RPC batch requests", "transport", true, "basic/transports#streamable-http", async () => {
|
|
356
|
+
const res = await request(backendUrl, {
|
|
357
|
+
method: "POST",
|
|
358
|
+
headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...userHeaders },
|
|
359
|
+
body: JSON.stringify([
|
|
360
|
+
{ jsonrpc: "2.0", id: 99903, method: "ping" },
|
|
361
|
+
{ jsonrpc: "2.0", id: 99904, method: "ping" }
|
|
362
|
+
]),
|
|
363
|
+
signal: AbortSignal.timeout(timeout)
|
|
364
|
+
});
|
|
365
|
+
const text = await res.body.text();
|
|
366
|
+
if (res.statusCode >= 400 && res.statusCode < 500) {
|
|
367
|
+
return { passed: true, details: `HTTP ${res.statusCode} (batch rejected)` };
|
|
368
|
+
}
|
|
369
|
+
try {
|
|
370
|
+
const body = JSON.parse(text);
|
|
371
|
+
if (body?.error) {
|
|
372
|
+
return { passed: true, details: `JSON-RPC error: ${body.error.code} \u2014 ${body.error.message}` };
|
|
373
|
+
}
|
|
374
|
+
if (Array.isArray(body)) {
|
|
375
|
+
return { passed: false, details: "Server processed batch request (MCP forbids batch)" };
|
|
376
|
+
}
|
|
377
|
+
} catch {
|
|
378
|
+
}
|
|
379
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error or 4xx for batch request` };
|
|
380
|
+
});
|
|
234
381
|
let initRes = null;
|
|
235
|
-
|
|
382
|
+
try {
|
|
236
383
|
initRes = await rpc("initialize", {
|
|
237
384
|
protocolVersion: SPEC_VERSION,
|
|
238
385
|
capabilities: { roots: { listChanged: true }, sampling: {} },
|
|
239
|
-
clientInfo: { name: "mcp-compliance", version:
|
|
386
|
+
clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
|
|
240
387
|
});
|
|
388
|
+
const result = initRes?.body?.result;
|
|
389
|
+
if (result) {
|
|
390
|
+
serverInfo.protocolVersion = result.protocolVersion || null;
|
|
391
|
+
serverInfo.name = result.serverInfo?.name || null;
|
|
392
|
+
serverInfo.version = result.serverInfo?.version || null;
|
|
393
|
+
serverInfo.capabilities = result.capabilities || {};
|
|
394
|
+
const sid = initRes.headers["mcp-session-id"];
|
|
395
|
+
if (sid) sessionId = sid;
|
|
396
|
+
if (result.protocolVersion) negotiatedProtocolVersion = result.protocolVersion;
|
|
397
|
+
}
|
|
398
|
+
} catch (err) {
|
|
399
|
+
}
|
|
400
|
+
try {
|
|
401
|
+
await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders(), timeout);
|
|
402
|
+
} catch {
|
|
403
|
+
}
|
|
404
|
+
await test("lifecycle-init", "Initialize handshake", "lifecycle", true, "basic/lifecycle#initialization", async () => {
|
|
405
|
+
if (!initRes) return { passed: false, details: "Initialize request failed" };
|
|
241
406
|
const result = initRes.body?.result;
|
|
242
407
|
if (!result) return { passed: false, details: "No result in response" };
|
|
243
|
-
serverInfo.protocolVersion = result.protocolVersion || null;
|
|
244
|
-
serverInfo.name = result.serverInfo?.name || null;
|
|
245
|
-
serverInfo.version = result.serverInfo?.version || null;
|
|
246
|
-
serverInfo.capabilities = result.capabilities || {};
|
|
247
|
-
const sid = initRes.headers["mcp-session-id"];
|
|
248
|
-
if (sid) sessionId = sid;
|
|
249
|
-
if (result.protocolVersion) negotiatedProtocolVersion = result.protocolVersion;
|
|
250
408
|
return { passed: !!result.protocolVersion, details: `Protocol: ${result.protocolVersion || "missing"}` };
|
|
251
409
|
});
|
|
252
410
|
await test("lifecycle-proto-version", "Returns valid protocol version", "lifecycle", true, "basic/lifecycle#version-negotiation", async () => {
|
|
253
411
|
const version2 = initRes?.body?.result?.protocolVersion;
|
|
254
412
|
if (!version2) return { passed: false, details: "No protocolVersion" };
|
|
255
413
|
const valid = /^\d{4}-\d{2}-\d{2}$/.test(version2);
|
|
414
|
+
if (valid && version2 !== SPEC_VERSION) {
|
|
415
|
+
warnings.push(`Server negotiated protocol version ${version2} (latest is ${SPEC_VERSION})`);
|
|
416
|
+
}
|
|
256
417
|
return { passed: valid, details: `Version: ${version2}` };
|
|
257
418
|
});
|
|
258
419
|
await test("lifecycle-server-info", "Includes serverInfo", "lifecycle", false, "basic/lifecycle#initialization", async () => {
|
|
@@ -270,7 +431,6 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
270
431
|
const valid = body?.jsonrpc === "2.0" && body?.id !== void 0 && (body?.result !== void 0 || body?.error !== void 0);
|
|
271
432
|
return { passed: valid, details: valid ? "Valid JSON-RPC 2.0 response" : `Missing fields: jsonrpc=${body?.jsonrpc}, id=${body?.id}` };
|
|
272
433
|
});
|
|
273
|
-
await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders());
|
|
274
434
|
await test("lifecycle-ping", "Responds to ping", "lifecycle", true, "basic/utilities#ping", async () => {
|
|
275
435
|
const res = await rpc("ping");
|
|
276
436
|
const body = res.body;
|
|
@@ -278,6 +438,89 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
278
438
|
if (body?.result !== void 0) return { passed: true, details: "Ping responded successfully" };
|
|
279
439
|
return { passed: false, details: "No result in ping response" };
|
|
280
440
|
});
|
|
441
|
+
await test("lifecycle-instructions", "Instructions field is valid", "lifecycle", false, "basic/lifecycle#initialization", async () => {
|
|
442
|
+
const result = initRes?.body?.result;
|
|
443
|
+
if (!result) return { passed: false, details: "No init result" };
|
|
444
|
+
if (result.instructions === void 0) {
|
|
445
|
+
return { passed: true, details: "No instructions field (optional)" };
|
|
446
|
+
}
|
|
447
|
+
if (typeof result.instructions === "string") {
|
|
448
|
+
const preview = result.instructions.length > 80 ? result.instructions.slice(0, 80) + "..." : result.instructions;
|
|
449
|
+
return { passed: true, details: `Instructions: "${preview}"` };
|
|
450
|
+
}
|
|
451
|
+
return { passed: false, details: `instructions should be a string, got ${typeof result.instructions}` };
|
|
452
|
+
});
|
|
453
|
+
await test("lifecycle-id-match", "Response ID matches request ID", "lifecycle", true, "basic", async () => {
|
|
454
|
+
const res = await rpc("ping");
|
|
455
|
+
const body = res.body;
|
|
456
|
+
if (body?.id === void 0) return { passed: false, details: "No id in response" };
|
|
457
|
+
const match = body.id === res.requestId;
|
|
458
|
+
return { passed: match, details: match ? `Request id=${res.requestId}, response id=${body.id} (match)` : `Request id=${res.requestId}, response id=${body.id} (MISMATCH)` };
|
|
459
|
+
});
|
|
460
|
+
const hasLogging = !!serverInfo.capabilities.logging;
|
|
461
|
+
await test("lifecycle-logging", "logging/setLevel accepted", "lifecycle", hasLogging, "server/utilities#logging", async () => {
|
|
462
|
+
if (!hasLogging) return { passed: true, details: "Server does not declare logging capability (skipped)" };
|
|
463
|
+
const res = await rpc("logging/setLevel", { level: "info" });
|
|
464
|
+
if (res.body?.error) {
|
|
465
|
+
return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
|
|
466
|
+
}
|
|
467
|
+
return { passed: true, details: "logging/setLevel accepted" };
|
|
468
|
+
});
|
|
469
|
+
const hasCompletions = !!serverInfo.capabilities.completions;
|
|
470
|
+
await test("lifecycle-completions", "completion/complete accepted", "lifecycle", hasCompletions, "server/utilities#completion", async () => {
|
|
471
|
+
if (!hasCompletions) return { passed: true, details: "Server does not declare completions capability (skipped)" };
|
|
472
|
+
const res = await rpc("completion/complete", {
|
|
473
|
+
ref: { type: "ref/prompt", name: "__test__" },
|
|
474
|
+
argument: { name: "test", value: "" }
|
|
475
|
+
});
|
|
476
|
+
if (res.body?.error) {
|
|
477
|
+
if (res.body.error.code === -32602) {
|
|
478
|
+
return { passed: true, details: "InvalidParams for test ref (acceptable)" };
|
|
479
|
+
}
|
|
480
|
+
return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
|
|
481
|
+
}
|
|
482
|
+
const values = res.body?.result?.completion?.values;
|
|
483
|
+
if (Array.isArray(values)) {
|
|
484
|
+
return { passed: true, details: `Returned ${values.length} completion(s)` };
|
|
485
|
+
}
|
|
486
|
+
return { passed: true, details: "completion/complete accepted" };
|
|
487
|
+
});
|
|
488
|
+
await test("transport-notification-202", "Notification returns 202 Accepted", "transport", false, "basic/transports#streamable-http", async () => {
|
|
489
|
+
const res = await request(backendUrl, {
|
|
490
|
+
method: "POST",
|
|
491
|
+
headers: {
|
|
492
|
+
"Content-Type": "application/json",
|
|
493
|
+
"Accept": "application/json, text/event-stream",
|
|
494
|
+
...buildHeaders()
|
|
495
|
+
},
|
|
496
|
+
body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/cancelled", params: { requestId: "nonexistent", reason: "compliance test" } }),
|
|
497
|
+
signal: AbortSignal.timeout(timeout)
|
|
498
|
+
});
|
|
499
|
+
await res.body.text();
|
|
500
|
+
if (res.statusCode === 202) {
|
|
501
|
+
return { passed: true, details: "HTTP 202 Accepted (correct)" };
|
|
502
|
+
}
|
|
503
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
504
|
+
return { passed: true, details: `HTTP ${res.statusCode} (accepted, but 202 is preferred)` };
|
|
505
|
+
}
|
|
506
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected 202 Accepted for notifications` };
|
|
507
|
+
});
|
|
508
|
+
await test("transport-session-id", "Enforces MCP-Session-Id after init", "transport", false, "basic/transports#streamable-http", async () => {
|
|
509
|
+
if (!sessionId) {
|
|
510
|
+
warnings.push("Server did not issue MCP-Session-Id header");
|
|
511
|
+
return { passed: true, details: "Server did not issue session ID (test not applicable)" };
|
|
512
|
+
}
|
|
513
|
+
const headersWithout = { ...userHeaders };
|
|
514
|
+
if (negotiatedProtocolVersion) headersWithout["mcp-protocol-version"] = negotiatedProtocolVersion;
|
|
515
|
+
const res = await mcpRequest(backendUrl, "ping", void 0, createIdCounter(99910), headersWithout, timeout);
|
|
516
|
+
if (res.statusCode === 400) {
|
|
517
|
+
return { passed: true, details: "HTTP 400 for missing session ID (correct)" };
|
|
518
|
+
}
|
|
519
|
+
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
520
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should return 400 when session ID is missing` };
|
|
521
|
+
}
|
|
522
|
+
return { passed: false, details: `HTTP ${res.statusCode}` };
|
|
523
|
+
});
|
|
281
524
|
const hasTools = !!serverInfo.capabilities.tools;
|
|
282
525
|
let cachedToolsList = null;
|
|
283
526
|
await test("tools-list", "tools/list returns valid response", "tools", hasTools, "server/tools#listing-tools", async () => {
|
|
@@ -292,7 +535,6 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
292
535
|
await test("tools-schema", "All tools have name and inputSchema", "schema", hasTools, "server/tools#data-types", async () => {
|
|
293
536
|
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
294
537
|
const issues = [];
|
|
295
|
-
const warnings = [];
|
|
296
538
|
for (const tool of tools) {
|
|
297
539
|
if (!tool.name) {
|
|
298
540
|
issues.push("Tool missing name");
|
|
@@ -301,7 +543,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
301
543
|
if (tool.name.length > 128 || !/^[A-Za-z0-9_.\-]+$/.test(tool.name)) {
|
|
302
544
|
issues.push(`${tool.name}: name format invalid`);
|
|
303
545
|
}
|
|
304
|
-
if (!tool.description) warnings.push(
|
|
546
|
+
if (!tool.description) warnings.push(`Tool "${tool.name}" missing description`);
|
|
305
547
|
if (!tool.inputSchema) {
|
|
306
548
|
issues.push(`${tool.name}: missing inputSchema (required)`);
|
|
307
549
|
} else if (typeof tool.inputSchema !== "object" || tool.inputSchema === null) {
|
|
@@ -310,9 +552,68 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
310
552
|
issues.push(`${tool.name}: inputSchema.type must be "object" (got "${tool.inputSchema.type || "undefined"}")`);
|
|
311
553
|
}
|
|
312
554
|
}
|
|
313
|
-
const detail = issues.length === 0 ?
|
|
555
|
+
const detail = issues.length === 0 ? "All tools have valid schemas" : issues.join("; ");
|
|
314
556
|
return { passed: issues.length === 0, details: detail };
|
|
315
557
|
});
|
|
558
|
+
await test("tools-annotations", "Tool annotations are valid", "schema", false, "server/tools#annotations", async () => {
|
|
559
|
+
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
560
|
+
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
561
|
+
const issues = [];
|
|
562
|
+
let annotatedCount = 0;
|
|
563
|
+
for (const tool of tools) {
|
|
564
|
+
const ann = tool.annotations;
|
|
565
|
+
if (!ann) continue;
|
|
566
|
+
annotatedCount++;
|
|
567
|
+
if (typeof ann !== "object" || ann === null) {
|
|
568
|
+
issues.push(`${tool.name}: annotations must be an object`);
|
|
569
|
+
continue;
|
|
570
|
+
}
|
|
571
|
+
const boolFields = ["readOnlyHint", "destructiveHint", "idempotentHint", "openWorldHint"];
|
|
572
|
+
for (const field of boolFields) {
|
|
573
|
+
if (ann[field] !== void 0 && typeof ann[field] !== "boolean") {
|
|
574
|
+
issues.push(`${tool.name}: annotations.${field} should be boolean, got ${typeof ann[field]}`);
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
if (ann.title !== void 0 && typeof ann.title !== "string") {
|
|
578
|
+
issues.push(`${tool.name}: annotations.title should be string`);
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
582
|
+
return { passed: true, details: annotatedCount > 0 ? `${annotatedCount} tool(s) with valid annotations` : "No tools have annotations (optional)" };
|
|
583
|
+
});
|
|
584
|
+
await test("tools-title-field", "Tools include title field", "schema", false, "server/tools#data-types", async () => {
|
|
585
|
+
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
586
|
+
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
587
|
+
const withTitle = tools.filter((t) => typeof t.title === "string");
|
|
588
|
+
const issues = [];
|
|
589
|
+
for (const tool of tools) {
|
|
590
|
+
if (tool.title !== void 0 && typeof tool.title !== "string") {
|
|
591
|
+
issues.push(`${tool.name}: title should be a string`);
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
595
|
+
if (withTitle.length === 0) {
|
|
596
|
+
return { passed: true, details: "No tools have title field (optional, added in 2025-11-25)" };
|
|
597
|
+
}
|
|
598
|
+
return { passed: true, details: `${withTitle.length}/${tools.length} tool(s) have title field` };
|
|
599
|
+
});
|
|
600
|
+
await test("tools-output-schema", "Tools with outputSchema are valid", "schema", false, "server/tools#structured-content", async () => {
|
|
601
|
+
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
602
|
+
if (tools.length === 0) return { passed: true, details: "No tools to validate" };
|
|
603
|
+
const issues = [];
|
|
604
|
+
let withSchema = 0;
|
|
605
|
+
for (const tool of tools) {
|
|
606
|
+
if (tool.outputSchema === void 0) continue;
|
|
607
|
+
withSchema++;
|
|
608
|
+
if (typeof tool.outputSchema !== "object" || tool.outputSchema === null) {
|
|
609
|
+
issues.push(`${tool.name}: outputSchema must be a JSON Schema object`);
|
|
610
|
+
} else if (tool.outputSchema.type !== "object") {
|
|
611
|
+
issues.push(`${tool.name}: outputSchema.type must be "object" (got "${tool.outputSchema.type || "undefined"}")`);
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
615
|
+
return { passed: true, details: withSchema > 0 ? `${withSchema} tool(s) with valid outputSchema` : "No tools have outputSchema (optional)" };
|
|
616
|
+
});
|
|
316
617
|
if (toolNames.length > 0) {
|
|
317
618
|
await test("tools-call", "tools/call responds correctly", "tools", false, "server/tools#calling-tools", async () => {
|
|
318
619
|
const res = await rpc("tools/call", { name: toolNames[0], arguments: {} });
|
|
@@ -335,6 +636,51 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
335
636
|
}
|
|
336
637
|
return { passed: false, details: "Response missing content array" };
|
|
337
638
|
});
|
|
639
|
+
await test("tools-content-types", "Tool content items have valid types", "tools", false, "server/tools#calling-tools", async () => {
|
|
640
|
+
const res = await rpc("tools/call", { name: toolNames[0], arguments: {} });
|
|
641
|
+
const result = res.body?.result;
|
|
642
|
+
const error = res.body?.error;
|
|
643
|
+
if (error) {
|
|
644
|
+
return { passed: true, details: `Tool returned error (content types not applicable): code ${error.code}` };
|
|
645
|
+
}
|
|
646
|
+
const content = result?.content;
|
|
647
|
+
if (!Array.isArray(content) || content.length === 0) {
|
|
648
|
+
return { passed: true, details: "No content items to validate" };
|
|
649
|
+
}
|
|
650
|
+
const issues = [];
|
|
651
|
+
const types = /* @__PURE__ */ new Set();
|
|
652
|
+
for (const item of content) {
|
|
653
|
+
if (!item.type) {
|
|
654
|
+
issues.push("Content item missing type field");
|
|
655
|
+
} else if (!VALID_CONTENT_TYPES.includes(item.type)) {
|
|
656
|
+
issues.push(`Unknown content type: "${item.type}"`);
|
|
657
|
+
} else {
|
|
658
|
+
types.add(item.type);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
662
|
+
return { passed: true, details: `Content types: ${[...types].join(", ")}` };
|
|
663
|
+
});
|
|
664
|
+
}
|
|
665
|
+
if (hasTools) {
|
|
666
|
+
await test("tools-pagination", "tools/list supports pagination", "tools", false, "server/tools#listing-tools", async () => {
|
|
667
|
+
const res = await rpc("tools/list");
|
|
668
|
+
const result = res.body?.result;
|
|
669
|
+
if (!result) return { passed: false, details: "No result from tools/list" };
|
|
670
|
+
if (!Array.isArray(result.tools)) return { passed: false, details: "No tools array" };
|
|
671
|
+
if (result.nextCursor !== void 0) {
|
|
672
|
+
if (typeof result.nextCursor !== "string") {
|
|
673
|
+
return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
|
|
674
|
+
}
|
|
675
|
+
const nextRes = await rpc("tools/list", { cursor: result.nextCursor });
|
|
676
|
+
const nextResult = nextRes.body?.result;
|
|
677
|
+
if (!nextResult || !Array.isArray(nextResult.tools)) {
|
|
678
|
+
return { passed: false, details: "Next page failed to return tools array" };
|
|
679
|
+
}
|
|
680
|
+
return { passed: true, details: `Pagination works: page 1 had ${result.tools.length} tools, page 2 had ${nextResult.tools.length} tools` };
|
|
681
|
+
}
|
|
682
|
+
return { passed: true, details: `${result.tools.length} tool(s), no nextCursor (single page)` };
|
|
683
|
+
});
|
|
338
684
|
await test("tools-call-unknown", "Returns error for unknown tool name", "errors", false, "server/tools#error-handling", async () => {
|
|
339
685
|
const res = await rpc("tools/call", { name: "__nonexistent_tool_compliance_test__", arguments: {} });
|
|
340
686
|
const error = res.body?.error;
|
|
@@ -345,6 +691,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
345
691
|
});
|
|
346
692
|
}
|
|
347
693
|
const hasResources = !!serverInfo.capabilities.resources;
|
|
694
|
+
const hasSubscribe = !!serverInfo.capabilities.resources?.subscribe;
|
|
348
695
|
if (hasResources) {
|
|
349
696
|
let cachedResourcesList = null;
|
|
350
697
|
await test("resources-list", "resources/list returns valid response", "resources", true, "server/resources#listing-resources", async () => {
|
|
@@ -353,6 +700,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
353
700
|
if (!Array.isArray(resources)) return { passed: false, details: "No resources array" };
|
|
354
701
|
cachedResourcesList = resources;
|
|
355
702
|
resourceCount = resources.length;
|
|
703
|
+
resourceNames = resources.map((r) => r.name).filter(Boolean);
|
|
356
704
|
return { passed: true, details: `${resourceCount} resource(s)` };
|
|
357
705
|
});
|
|
358
706
|
await test("resources-schema", "Resources have uri and name", "schema", true, "server/resources#data-types", async () => {
|
|
@@ -368,13 +716,15 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
368
716
|
}
|
|
369
717
|
}
|
|
370
718
|
if (!r.name) issues.push(`${r.uri || "?"}: missing name`);
|
|
719
|
+
if (!r.description) warnings.push(`Resource "${r.name || r.uri}" missing description`);
|
|
720
|
+
if (!r.mimeType) warnings.push(`Resource "${r.name || r.uri}" missing mimeType`);
|
|
371
721
|
}
|
|
372
722
|
return { passed: issues.length === 0, details: issues.length === 0 ? "All resources valid" : issues.join("; ") };
|
|
373
723
|
});
|
|
374
724
|
if (resourceCount > 0) {
|
|
375
725
|
await test("resources-read", "resources/read returns content", "resources", false, "server/resources#reading-resources", async () => {
|
|
376
|
-
const
|
|
377
|
-
const firstUri =
|
|
726
|
+
const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
|
|
727
|
+
const firstUri = resources[0]?.uri;
|
|
378
728
|
if (!firstUri) return { passed: false, details: "No resource URI to test" };
|
|
379
729
|
const readRes = await rpc("resources/read", { uri: firstUri });
|
|
380
730
|
const contents = readRes.body?.result?.contents;
|
|
@@ -405,10 +755,43 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
405
755
|
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
406
756
|
return { passed: true, details: `${templates.length} resource template(s)` };
|
|
407
757
|
});
|
|
758
|
+
await test("resources-pagination", "resources/list supports pagination", "resources", false, "server/resources#listing-resources", async () => {
|
|
759
|
+
const res = await rpc("resources/list");
|
|
760
|
+
const result = res.body?.result;
|
|
761
|
+
if (!result) return { passed: false, details: "No result from resources/list" };
|
|
762
|
+
if (!Array.isArray(result.resources)) return { passed: false, details: "No resources array" };
|
|
763
|
+
if (result.nextCursor !== void 0) {
|
|
764
|
+
if (typeof result.nextCursor !== "string") {
|
|
765
|
+
return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
|
|
766
|
+
}
|
|
767
|
+
const nextRes = await rpc("resources/list", { cursor: result.nextCursor });
|
|
768
|
+
const nextResult = nextRes.body?.result;
|
|
769
|
+
if (!nextResult || !Array.isArray(nextResult.resources)) {
|
|
770
|
+
return { passed: false, details: "Next page failed to return resources array" };
|
|
771
|
+
}
|
|
772
|
+
return { passed: true, details: `Pagination works: page 1 had ${result.resources.length}, page 2 had ${nextResult.resources.length}` };
|
|
773
|
+
}
|
|
774
|
+
return { passed: true, details: `${result.resources.length} resource(s), no nextCursor (single page)` };
|
|
775
|
+
});
|
|
776
|
+
if (hasSubscribe && resourceCount > 0) {
|
|
777
|
+
await test("resources-subscribe", "Resource subscribe/unsubscribe", "resources", true, "server/resources#subscriptions", async () => {
|
|
778
|
+
const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
|
|
779
|
+
const firstUri = resources[0]?.uri;
|
|
780
|
+
if (!firstUri) return { passed: false, details: "No resource URI for subscribe test" };
|
|
781
|
+
const subRes = await rpc("resources/subscribe", { uri: firstUri });
|
|
782
|
+
if (subRes.body?.error) {
|
|
783
|
+
return { passed: false, details: `Subscribe error: ${subRes.body.error.code} \u2014 ${subRes.body.error.message}` };
|
|
784
|
+
}
|
|
785
|
+
const unsubRes = await rpc("resources/unsubscribe", { uri: firstUri });
|
|
786
|
+
if (unsubRes.body?.error) {
|
|
787
|
+
return { passed: false, details: `Unsubscribe error: ${unsubRes.body.error.code} \u2014 ${unsubRes.body.error.message}` };
|
|
788
|
+
}
|
|
789
|
+
return { passed: true, details: `Subscribe/unsubscribe for ${firstUri} succeeded` };
|
|
790
|
+
});
|
|
791
|
+
}
|
|
408
792
|
}
|
|
409
793
|
const hasPrompts = !!serverInfo.capabilities.prompts;
|
|
410
794
|
if (hasPrompts) {
|
|
411
|
-
let promptNames = [];
|
|
412
795
|
let cachedPromptsList = null;
|
|
413
796
|
await test("prompts-list", "prompts/list returns valid response", "prompts", true, "server/prompts#listing-prompts", async () => {
|
|
414
797
|
const res = await rpc("prompts/list");
|
|
@@ -424,6 +807,7 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
424
807
|
const issues = [];
|
|
425
808
|
for (const p of prompts) {
|
|
426
809
|
if (!p.name) issues.push("Prompt missing name");
|
|
810
|
+
if (!p.description) warnings.push(`Prompt "${p.name || "?"}" missing description`);
|
|
427
811
|
if (p.arguments && !Array.isArray(p.arguments)) issues.push(`${p.name || "?"}: arguments must be an array`);
|
|
428
812
|
if (Array.isArray(p.arguments)) {
|
|
429
813
|
for (const arg of p.arguments) {
|
|
@@ -449,6 +833,24 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
449
833
|
return { passed: true, details: `${messages.length} message(s) from ${promptNames[0]}` };
|
|
450
834
|
});
|
|
451
835
|
}
|
|
836
|
+
await test("prompts-pagination", "prompts/list supports pagination", "prompts", false, "server/prompts#listing-prompts", async () => {
|
|
837
|
+
const res = await rpc("prompts/list");
|
|
838
|
+
const result = res.body?.result;
|
|
839
|
+
if (!result) return { passed: false, details: "No result from prompts/list" };
|
|
840
|
+
if (!Array.isArray(result.prompts)) return { passed: false, details: "No prompts array" };
|
|
841
|
+
if (result.nextCursor !== void 0) {
|
|
842
|
+
if (typeof result.nextCursor !== "string") {
|
|
843
|
+
return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
|
|
844
|
+
}
|
|
845
|
+
const nextRes = await rpc("prompts/list", { cursor: result.nextCursor });
|
|
846
|
+
const nextResult = nextRes.body?.result;
|
|
847
|
+
if (!nextResult || !Array.isArray(nextResult.prompts)) {
|
|
848
|
+
return { passed: false, details: "Next page failed to return prompts array" };
|
|
849
|
+
}
|
|
850
|
+
return { passed: true, details: `Pagination works: page 1 had ${result.prompts.length}, page 2 had ${nextResult.prompts.length}` };
|
|
851
|
+
}
|
|
852
|
+
return { passed: true, details: `${result.prompts.length} prompt(s), no nextCursor (single page)` };
|
|
853
|
+
});
|
|
452
854
|
}
|
|
453
855
|
await test("error-unknown-method", "Returns JSON-RPC error for unknown method", "errors", true, "basic", async () => {
|
|
454
856
|
const res = await rpc("nonexistent/method");
|
|
@@ -469,9 +871,9 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
469
871
|
await test("error-invalid-jsonrpc", "Handles malformed JSON-RPC", "errors", true, "basic", async () => {
|
|
470
872
|
const res = await request(backendUrl, {
|
|
471
873
|
method: "POST",
|
|
472
|
-
headers: { "Content-Type": "application/json", ...buildHeaders() },
|
|
874
|
+
headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...buildHeaders() },
|
|
473
875
|
body: JSON.stringify({ not: "a valid jsonrpc message" }),
|
|
474
|
-
signal: AbortSignal.timeout(
|
|
876
|
+
signal: AbortSignal.timeout(timeout)
|
|
475
877
|
});
|
|
476
878
|
const text = await res.body.text();
|
|
477
879
|
try {
|
|
@@ -488,9 +890,9 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
488
890
|
await test("error-invalid-json", "Handles invalid JSON body", "errors", false, "basic", async () => {
|
|
489
891
|
const res = await request(backendUrl, {
|
|
490
892
|
method: "POST",
|
|
491
|
-
headers: { "Content-Type": "application/json", ...buildHeaders() },
|
|
893
|
+
headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...buildHeaders() },
|
|
492
894
|
body: "{this is not valid json!!!",
|
|
493
|
-
signal: AbortSignal.timeout(
|
|
895
|
+
signal: AbortSignal.timeout(timeout)
|
|
494
896
|
});
|
|
495
897
|
const text = await res.body.text();
|
|
496
898
|
try {
|
|
@@ -512,10 +914,57 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
512
914
|
if (isError) return { passed: true, details: "Tool execution error (valid)" };
|
|
513
915
|
return { passed: false, details: "No error for tools/call without name" };
|
|
514
916
|
});
|
|
917
|
+
await test("error-parse-code", "Returns -32700 for invalid JSON", "errors", false, "basic", async () => {
|
|
918
|
+
const res = await request(backendUrl, {
|
|
919
|
+
method: "POST",
|
|
920
|
+
headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...buildHeaders() },
|
|
921
|
+
body: "<<<not json>>>",
|
|
922
|
+
signal: AbortSignal.timeout(timeout)
|
|
923
|
+
});
|
|
924
|
+
const text = await res.body.text();
|
|
925
|
+
try {
|
|
926
|
+
const body = JSON.parse(text);
|
|
927
|
+
if (body?.error?.code === -32700) {
|
|
928
|
+
return { passed: true, details: `Error code: -32700 (Parse error) \u2014 ${body.error.message}` };
|
|
929
|
+
}
|
|
930
|
+
if (body?.error) {
|
|
931
|
+
return { passed: false, details: `Expected -32700, got ${body.error.code} \u2014 ${body.error.message}` };
|
|
932
|
+
}
|
|
933
|
+
} catch {
|
|
934
|
+
}
|
|
935
|
+
if (res.statusCode >= 400 && res.statusCode < 500) {
|
|
936
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should return JSON-RPC error code -32700` };
|
|
937
|
+
}
|
|
938
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error code -32700` };
|
|
939
|
+
});
|
|
940
|
+
await test("error-invalid-request-code", "Returns -32600 for invalid request", "errors", false, "basic", async () => {
|
|
941
|
+
const res = await request(backendUrl, {
|
|
942
|
+
method: "POST",
|
|
943
|
+
headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...buildHeaders() },
|
|
944
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 99999 }),
|
|
945
|
+
signal: AbortSignal.timeout(timeout)
|
|
946
|
+
});
|
|
947
|
+
const text = await res.body.text();
|
|
948
|
+
try {
|
|
949
|
+
const body = JSON.parse(text);
|
|
950
|
+
if (body?.error?.code === -32600) {
|
|
951
|
+
return { passed: true, details: `Error code: -32600 (Invalid Request) \u2014 ${body.error.message}` };
|
|
952
|
+
}
|
|
953
|
+
if (body?.error) {
|
|
954
|
+
return { passed: false, details: `Expected -32600, got ${body.error.code} \u2014 ${body.error.message}` };
|
|
955
|
+
}
|
|
956
|
+
} catch {
|
|
957
|
+
}
|
|
958
|
+
if (res.statusCode >= 400 && res.statusCode < 500) {
|
|
959
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should return JSON-RPC error code -32600` };
|
|
960
|
+
}
|
|
961
|
+
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error code -32600` };
|
|
962
|
+
});
|
|
515
963
|
const { score, grade, overall, summary, categories } = computeScore(tests);
|
|
516
964
|
const badge = generateBadge(url);
|
|
517
965
|
return {
|
|
518
966
|
specVersion: SPEC_VERSION,
|
|
967
|
+
toolVersion: TOOL_VERSION,
|
|
519
968
|
url,
|
|
520
969
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
521
970
|
score,
|
|
@@ -524,11 +973,14 @@ async function runComplianceSuite(url, options = {}) {
|
|
|
524
973
|
summary,
|
|
525
974
|
categories,
|
|
526
975
|
tests,
|
|
976
|
+
warnings,
|
|
527
977
|
serverInfo,
|
|
528
978
|
toolCount,
|
|
529
979
|
toolNames,
|
|
530
980
|
resourceCount,
|
|
981
|
+
resourceNames,
|
|
531
982
|
promptCount,
|
|
983
|
+
promptNames,
|
|
532
984
|
badge
|
|
533
985
|
};
|
|
534
986
|
}
|
|
@@ -582,7 +1034,7 @@ function formatTerminal(report) {
|
|
|
582
1034
|
const lines = [];
|
|
583
1035
|
lines.push("");
|
|
584
1036
|
lines.push(chalk.bold("MCP Compliance Report"));
|
|
585
|
-
lines.push(chalk.dim(`Spec: ${report.specVersion} | ${report.timestamp}`));
|
|
1037
|
+
lines.push(chalk.dim(`Spec: ${report.specVersion} | Tool: v${report.toolVersion} | ${report.timestamp}`));
|
|
586
1038
|
lines.push(chalk.dim(`URL: ${report.url}`));
|
|
587
1039
|
if (report.serverInfo.name) {
|
|
588
1040
|
lines.push(chalk.dim(`Server: ${report.serverInfo.name} v${report.serverInfo.version || "?"} (protocol ${report.serverInfo.protocolVersion || "?"})`));
|
|
@@ -608,15 +1060,27 @@ function formatTerminal(report) {
|
|
|
608
1060
|
lines.push(testLine(t));
|
|
609
1061
|
}
|
|
610
1062
|
}
|
|
611
|
-
|
|
1063
|
+
const caps = report.serverInfo.capabilities;
|
|
1064
|
+
const declared = Object.keys(caps).filter((k) => caps[k] !== void 0);
|
|
1065
|
+
if (declared.length > 0) {
|
|
612
1066
|
lines.push("");
|
|
1067
|
+
lines.push(chalk.dim(` Capabilities: ${declared.join(", ")}`));
|
|
1068
|
+
}
|
|
1069
|
+
if (report.toolCount > 0) {
|
|
613
1070
|
lines.push(chalk.dim(` Tools (${report.toolCount}): ${report.toolNames.slice(0, 10).join(", ")}${report.toolCount > 10 ? "..." : ""}`));
|
|
614
1071
|
}
|
|
615
1072
|
if (report.resourceCount > 0) {
|
|
616
|
-
lines.push(chalk.dim(` Resources: ${report.resourceCount}`));
|
|
1073
|
+
lines.push(chalk.dim(` Resources (${report.resourceCount}): ${report.resourceNames.slice(0, 10).join(", ")}${report.resourceCount > 10 ? "..." : ""}`));
|
|
617
1074
|
}
|
|
618
1075
|
if (report.promptCount > 0) {
|
|
619
|
-
lines.push(chalk.dim(` Prompts: ${report.promptCount}`));
|
|
1076
|
+
lines.push(chalk.dim(` Prompts (${report.promptCount}): ${report.promptNames.slice(0, 10).join(", ")}${report.promptCount > 10 ? "..." : ""}`));
|
|
1077
|
+
}
|
|
1078
|
+
if (report.warnings.length > 0) {
|
|
1079
|
+
lines.push("");
|
|
1080
|
+
lines.push(chalk.yellow(` Warnings (${report.warnings.length}):`));
|
|
1081
|
+
for (const w of report.warnings) {
|
|
1082
|
+
lines.push(chalk.yellow(` - ${w}`));
|
|
1083
|
+
}
|
|
620
1084
|
}
|
|
621
1085
|
lines.push("");
|
|
622
1086
|
lines.push(chalk.dim(" Badge markdown:"));
|
|
@@ -629,7 +1093,128 @@ function formatJson(report) {
|
|
|
629
1093
|
}
|
|
630
1094
|
|
|
631
1095
|
// src/index.ts
|
|
632
|
-
|
|
1096
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
1097
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
1098
|
+
|
|
1099
|
+
// src/mcp/tools.ts
|
|
1100
|
+
import { z } from "zod";
|
|
1101
|
+
function registerTools(server) {
|
|
1102
|
+
server.tool(
|
|
1103
|
+
"mcp_compliance_test",
|
|
1104
|
+
"Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all 43 tests covering transport, lifecycle, tools, resources, prompts, errors, and schema validation.",
|
|
1105
|
+
{
|
|
1106
|
+
url: z.string().url().describe("The MCP server URL to test (must be HTTP or HTTPS)")
|
|
1107
|
+
},
|
|
1108
|
+
async ({ url }) => {
|
|
1109
|
+
try {
|
|
1110
|
+
const report = await runComplianceSuite(url);
|
|
1111
|
+
const summary = [
|
|
1112
|
+
`Grade: ${report.grade} (${report.score}%)`,
|
|
1113
|
+
`Overall: ${report.overall}`,
|
|
1114
|
+
`Tests: ${report.summary.passed}/${report.summary.total} passed (${report.summary.requiredPassed}/${report.summary.required} required)`,
|
|
1115
|
+
"",
|
|
1116
|
+
...report.tests.map(
|
|
1117
|
+
(t) => `${t.passed ? "PASS" : "FAIL"} ${t.name}${t.required ? " (required)" : ""} \u2014 ${t.details}`
|
|
1118
|
+
)
|
|
1119
|
+
];
|
|
1120
|
+
if (report.serverInfo.name) {
|
|
1121
|
+
summary.unshift(`Server: ${report.serverInfo.name} v${report.serverInfo.version || "?"}`);
|
|
1122
|
+
}
|
|
1123
|
+
if (report.warnings.length > 0) {
|
|
1124
|
+
summary.push("", `Warnings (${report.warnings.length}):`);
|
|
1125
|
+
for (const w of report.warnings) {
|
|
1126
|
+
summary.push(` - ${w}`);
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
return {
|
|
1130
|
+
content: [
|
|
1131
|
+
{ type: "text", text: summary.join("\n") },
|
|
1132
|
+
{ type: "text", text: `
|
|
1133
|
+
|
|
1134
|
+
Full report:
|
|
1135
|
+
${JSON.stringify(report, null, 2)}` }
|
|
1136
|
+
]
|
|
1137
|
+
};
|
|
1138
|
+
} catch (err) {
|
|
1139
|
+
return {
|
|
1140
|
+
content: [{ type: "text", text: `Error running compliance test: ${err.message}` }],
|
|
1141
|
+
isError: true
|
|
1142
|
+
};
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
);
|
|
1146
|
+
server.tool(
|
|
1147
|
+
"mcp_compliance_badge",
|
|
1148
|
+
"Get the badge markdown embed code for an MCP server. Runs the compliance test suite first to determine the grade.",
|
|
1149
|
+
{
|
|
1150
|
+
url: z.string().url().describe("The MCP server URL to test")
|
|
1151
|
+
},
|
|
1152
|
+
async ({ url }) => {
|
|
1153
|
+
try {
|
|
1154
|
+
const report = await runComplianceSuite(url);
|
|
1155
|
+
const badge = report.badge;
|
|
1156
|
+
return {
|
|
1157
|
+
content: [{
|
|
1158
|
+
type: "text",
|
|
1159
|
+
text: [
|
|
1160
|
+
`Grade: ${report.grade} (${report.score}%)`,
|
|
1161
|
+
"",
|
|
1162
|
+
"Markdown:",
|
|
1163
|
+
badge.markdown,
|
|
1164
|
+
"",
|
|
1165
|
+
"HTML:",
|
|
1166
|
+
badge.html
|
|
1167
|
+
].join("\n")
|
|
1168
|
+
}]
|
|
1169
|
+
};
|
|
1170
|
+
} catch (err) {
|
|
1171
|
+
return {
|
|
1172
|
+
content: [{ type: "text", text: `Error: ${err.message}` }],
|
|
1173
|
+
isError: true
|
|
1174
|
+
};
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
);
|
|
1178
|
+
server.tool(
|
|
1179
|
+
"mcp_compliance_explain",
|
|
1180
|
+
"Explain what a specific compliance test ID checks and why it matters.",
|
|
1181
|
+
{
|
|
1182
|
+
testId: z.string().describe('The test ID to explain (e.g., "transport-post", "lifecycle-init", "tools-schema")')
|
|
1183
|
+
},
|
|
1184
|
+
async ({ testId }) => {
|
|
1185
|
+
const def = TEST_DEFINITIONS.find((t) => t.id === testId);
|
|
1186
|
+
if (!def) {
|
|
1187
|
+
return {
|
|
1188
|
+
content: [{
|
|
1189
|
+
type: "text",
|
|
1190
|
+
text: `Unknown test ID: "${testId}"
|
|
1191
|
+
|
|
1192
|
+
Valid test IDs:
|
|
1193
|
+
${TEST_DEFINITIONS.map((t) => t.id).join(", ")}`
|
|
1194
|
+
}],
|
|
1195
|
+
isError: true
|
|
1196
|
+
};
|
|
1197
|
+
}
|
|
1198
|
+
return {
|
|
1199
|
+
content: [{
|
|
1200
|
+
type: "text",
|
|
1201
|
+
text: [
|
|
1202
|
+
`Test: ${def.id}`,
|
|
1203
|
+
`Name: ${def.name}`,
|
|
1204
|
+
`Category: ${def.category}`,
|
|
1205
|
+
`Required: ${def.required ? "Yes" : "No"}`,
|
|
1206
|
+
`Spec reference: https://modelcontextprotocol.io/specification/2025-11-25/${def.specRef}`,
|
|
1207
|
+
"",
|
|
1208
|
+
def.description
|
|
1209
|
+
].join("\n")
|
|
1210
|
+
}]
|
|
1211
|
+
};
|
|
1212
|
+
}
|
|
1213
|
+
);
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
// src/index.ts
|
|
1217
|
+
var require2 = createRequire2(import.meta.url);
|
|
633
1218
|
var { version } = require2("../package.json");
|
|
634
1219
|
function parseHeaderArg(value, prev) {
|
|
635
1220
|
const idx = value.indexOf(":");
|
|
@@ -641,9 +1226,12 @@ function parseHeaderArg(value, prev) {
|
|
|
641
1226
|
prev[key] = val;
|
|
642
1227
|
return prev;
|
|
643
1228
|
}
|
|
1229
|
+
function parseList(value) {
|
|
1230
|
+
return value.split(",").map((s) => s.trim()).filter(Boolean);
|
|
1231
|
+
}
|
|
644
1232
|
var program = new Command();
|
|
645
1233
|
program.name("mcp-compliance").description("Test MCP servers for spec compliance").version(version);
|
|
646
|
-
program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal or json", "terminal").option("--strict", "Exit with code 1 on any required test failure (for CI)").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').action(async (url, opts) => {
|
|
1234
|
+
program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal or json", "terminal").option("--strict", "Exit with code 1 on any required test failure (for CI)").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").option("--retries <n>", "Number of retries for failed tests", "0").option("--only <items>", "Only run tests matching these categories or test IDs (comma-separated)", parseList).option("--skip <items>", "Skip tests matching these categories or test IDs (comma-separated)", parseList).option("--verbose", "Print each test result as it runs").action(async (url, opts) => {
|
|
647
1235
|
try {
|
|
648
1236
|
const headers = { ...opts.header };
|
|
649
1237
|
if (opts.auth) headers["Authorization"] = opts.auth;
|
|
@@ -652,7 +1240,20 @@ program.command("test").description("Run the full compliance test suite against
|
|
|
652
1240
|
Testing ${url}...
|
|
653
1241
|
`));
|
|
654
1242
|
}
|
|
655
|
-
const report = await runComplianceSuite(url, {
|
|
1243
|
+
const report = await runComplianceSuite(url, {
|
|
1244
|
+
headers,
|
|
1245
|
+
timeout: parseInt(opts.timeout, 10) || 15e3,
|
|
1246
|
+
retries: parseInt(opts.retries, 10) || 0,
|
|
1247
|
+
only: opts.only,
|
|
1248
|
+
skip: opts.skip,
|
|
1249
|
+
onProgress: opts.verbose ? (testId, passed, details) => {
|
|
1250
|
+
const icon = passed ? chalk2.green("PASS") : chalk2.red("FAIL");
|
|
1251
|
+
console.log(` ${icon} ${testId} \u2014 ${details}`);
|
|
1252
|
+
} : void 0
|
|
1253
|
+
});
|
|
1254
|
+
if (opts.verbose && opts.format === "terminal") {
|
|
1255
|
+
console.log("");
|
|
1256
|
+
}
|
|
656
1257
|
if (opts.format === "json") {
|
|
657
1258
|
console.log(formatJson(report));
|
|
658
1259
|
} else {
|
|
@@ -672,14 +1273,17 @@ Error: ${err.message}
|
|
|
672
1273
|
process.exit(1);
|
|
673
1274
|
}
|
|
674
1275
|
});
|
|
675
|
-
program.command("badge").description("Run tests and output just the badge markdown embed code").argument("<url>", "MCP server URL to test").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').action(async (url, opts) => {
|
|
1276
|
+
program.command("badge").description("Run tests and output just the badge markdown embed code").argument("<url>", "MCP server URL to test").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").action(async (url, opts) => {
|
|
676
1277
|
try {
|
|
677
1278
|
const headers = { ...opts.header };
|
|
678
1279
|
if (opts.auth) headers["Authorization"] = opts.auth;
|
|
679
1280
|
console.log(chalk2.dim(`
|
|
680
1281
|
Testing ${url}...
|
|
681
1282
|
`));
|
|
682
|
-
const report = await runComplianceSuite(url, {
|
|
1283
|
+
const report = await runComplianceSuite(url, {
|
|
1284
|
+
headers,
|
|
1285
|
+
timeout: parseInt(opts.timeout, 10) || 15e3
|
|
1286
|
+
});
|
|
683
1287
|
console.log(`Grade: ${report.grade} (${report.score}%)
|
|
684
1288
|
`);
|
|
685
1289
|
console.log(report.badge.markdown);
|
|
@@ -691,4 +1295,10 @@ Error: ${err.message}
|
|
|
691
1295
|
process.exit(1);
|
|
692
1296
|
}
|
|
693
1297
|
});
|
|
1298
|
+
program.command("mcp").description("Start the MCP compliance server (stdio transport)").action(async () => {
|
|
1299
|
+
const server = new McpServer({ name: "mcp-compliance", version });
|
|
1300
|
+
registerTools(server);
|
|
1301
|
+
const transport = new StdioServerTransport();
|
|
1302
|
+
await server.connect(transport);
|
|
1303
|
+
});
|
|
694
1304
|
program.parse();
|