@yawlabs/mcp-compliance 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -3,10 +3,11 @@
3
3
  // src/index.ts
4
4
  import { Command } from "commander";
5
5
  import chalk2 from "chalk";
6
- import { createRequire } from "module";
6
+ import { createRequire as createRequire2 } from "module";
7
7
 
8
8
  // src/runner.ts
9
9
  import { request } from "undici";
10
+ import { createRequire } from "module";
10
11
 
11
12
  // src/grader.ts
12
13
  function computeGrade(score) {
@@ -62,33 +63,99 @@ function generateBadge(url) {
62
63
  };
63
64
  }
64
65
 
66
+ // src/types.ts
67
+ var TEST_DEFINITIONS = [
68
+ // ── Transport (7 tests) ──────────────────────────────────────────
69
+ { id: "transport-post", name: "HTTP POST accepted", category: "transport", required: true, specRef: "basic/transports#streamable-http", description: "Verifies the server accepts HTTP POST requests and returns a 2xx status code. This is the fundamental transport requirement for Streamable HTTP MCP servers." },
70
+ { id: "transport-content-type", name: "Responds with JSON or SSE", category: "transport", required: true, specRef: "basic/transports#streamable-http", description: "Checks that the server responds with Content-Type application/json or text/event-stream. MCP servers must use one of these two content types." },
71
+ { id: "transport-notification-202", name: "Notification returns 202 Accepted", category: "transport", required: false, specRef: "basic/transports#streamable-http", description: "Verifies that sending a JSON-RPC notification (no id field) returns HTTP 202 Accepted with no body. Per spec, servers MUST return 202 for notifications." },
72
+ { id: "transport-session-id", name: "Enforces MCP-Session-Id after init", category: "transport", required: false, specRef: "basic/transports#streamable-http", description: "Tests that the server returns HTTP 400 when MCP-Session-Id header is missing on requests after initialization (when the server issued a session ID)." },
73
+ { id: "transport-get", name: "GET returns SSE stream or 405", category: "transport", required: false, specRef: "basic/transports#streamable-http", description: "Tests the GET endpoint for server-initiated messages. Server should return text/event-stream or 405 Method Not Allowed." },
74
+ { id: "transport-delete", name: "DELETE accepted or returns 405", category: "transport", required: false, specRef: "basic/transports#streamable-http", description: "Tests the DELETE endpoint for session termination. Server should accept the request or return 405 Method Not Allowed." },
75
+ { id: "transport-batch-reject", name: "Rejects JSON-RPC batch requests", category: "transport", required: true, specRef: "basic/transports#streamable-http", description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests." },
76
+ // ── Lifecycle (10 tests) ─────────────────────────────────────────
77
+ { id: "lifecycle-init", name: "Initialize handshake", category: "lifecycle", required: true, specRef: "basic/lifecycle#initialization", description: "Tests the initialize handshake by sending an initialize request with client capabilities. The server must return a result with protocolVersion." },
78
+ { id: "lifecycle-proto-version", name: "Returns valid protocol version", category: "lifecycle", required: true, specRef: "basic/lifecycle#version-negotiation", description: "Validates that the protocolVersion returned by the server matches the YYYY-MM-DD date format required by the spec." },
79
+ { id: "lifecycle-server-info", name: "Includes serverInfo", category: "lifecycle", required: false, specRef: "basic/lifecycle#initialization", description: "Checks that the server includes a serverInfo object with at least a name field in its initialize response. While recommended, this is not strictly required." },
80
+ { id: "lifecycle-capabilities", name: "Returns capabilities object", category: "lifecycle", required: true, specRef: "basic/lifecycle#capability-negotiation", description: "Verifies the server returns a capabilities object in its initialize response. An empty object is valid (no optional features declared)." },
81
+ { id: "lifecycle-jsonrpc", name: "Response is valid JSON-RPC 2.0", category: "lifecycle", required: true, specRef: "basic", description: 'Validates that the initialize response is a proper JSON-RPC 2.0 message with jsonrpc="2.0", an id field, and either a result or error field.' },
82
+ { id: "lifecycle-ping", name: "Responds to ping", category: "lifecycle", required: true, specRef: "basic/utilities#ping", description: "Tests that the server responds to the ping method with an empty result object. This is a required utility method." },
83
+ { id: "lifecycle-instructions", name: "Instructions field is valid", category: "lifecycle", required: false, specRef: "basic/lifecycle#initialization", description: "If the server includes an instructions field in the initialize response, validates it is a string. Instructions provide guidance for how the client should interact with the server." },
84
+ { id: "lifecycle-id-match", name: "Response ID matches request ID", category: "lifecycle", required: true, specRef: "basic", description: "Verifies that the JSON-RPC response id matches the request id sent by the client. This is a fundamental JSON-RPC 2.0 requirement." },
85
+ { id: "lifecycle-logging", name: "logging/setLevel accepted", category: "lifecycle", required: false, specRef: "server/utilities#logging", description: "If the server declares logging capability, tests that logging/setLevel method is accepted with a valid log level." },
86
+ { id: "lifecycle-completions", name: "completion/complete accepted", category: "lifecycle", required: false, specRef: "server/utilities#completion", description: "If the server declares completions capability, tests that the completion/complete method is accepted." },
87
+ // ── Tools (4 tests) ──────────────────────────────────────────────
88
+ { id: "tools-list", name: "tools/list returns valid response", category: "tools", required: false, specRef: "server/tools#listing-tools", description: "Calls tools/list and validates it returns an array of tool definitions. Required if the server declares tools capability." },
89
+ { id: "tools-call", name: "tools/call responds correctly", category: "tools", required: false, specRef: "server/tools#calling-tools", description: "Calls the first tool with empty arguments and verifies the response format. Accepts both successful results and InvalidParams errors." },
90
+ { id: "tools-pagination", name: "tools/list supports pagination", category: "tools", required: false, specRef: "server/tools#listing-tools", description: "Tests cursor-based pagination on tools/list. Validates nextCursor is a string if present and that fetching the next page returns a valid response." },
91
+ { id: "tools-content-types", name: "Tool content items have valid types", category: "tools", required: false, specRef: "server/tools#calling-tools", description: "Validates that content items returned by tools/call have a recognized type field (text, image, audio, resource, resource_link)." },
92
+ // ── Resources (5 tests) ──────────────────────────────────────────
93
+ { id: "resources-list", name: "resources/list returns valid response", category: "resources", required: false, specRef: "server/resources#listing-resources", description: "Calls resources/list and validates it returns an array. Required if the server declares resources capability." },
94
+ { id: "resources-read", name: "resources/read returns content", category: "resources", required: false, specRef: "server/resources#reading-resources", description: "Reads the first resource and validates the response contains a contents array with proper uri and text/blob fields." },
95
+ { id: "resources-templates", name: "resources/templates/list returns valid response", category: "resources", required: false, specRef: "server/resources#resource-templates", description: "Tests the resource templates endpoint. Accepts Method not found (-32601) since templates are optional." },
96
+ { id: "resources-pagination", name: "resources/list supports pagination", category: "resources", required: false, specRef: "server/resources#listing-resources", description: "Tests cursor-based pagination on resources/list. Validates nextCursor is a string if present and that fetching the next page works." },
97
+ { id: "resources-subscribe", name: "Resource subscribe/unsubscribe", category: "resources", required: false, specRef: "server/resources#subscriptions", description: "If the server declares resources.subscribe capability, tests that resources/subscribe and resources/unsubscribe methods are accepted." },
98
+ // ── Prompts (3 tests) ────────────────────────────────────────────
99
+ { id: "prompts-list", name: "prompts/list returns valid response", category: "prompts", required: false, specRef: "server/prompts#listing-prompts", description: "Calls prompts/list and validates it returns an array. Required if the server declares prompts capability." },
100
+ { id: "prompts-get", name: "prompts/get returns valid messages", category: "prompts", required: false, specRef: "server/prompts#getting-a-prompt", description: "Gets the first prompt and validates the response contains a messages array with proper role and content fields." },
101
+ { id: "prompts-pagination", name: "prompts/list supports pagination", category: "prompts", required: false, specRef: "server/prompts#listing-prompts", description: "Tests cursor-based pagination on prompts/list. Validates nextCursor is a string if present and that fetching the next page works." },
102
+ // ── Error Handling (8 tests) ─────────────────────────────────────
103
+ { id: "error-unknown-method", name: "Returns JSON-RPC error for unknown method", category: "errors", required: true, specRef: "basic", description: "Sends an unknown method and verifies the server returns a JSON-RPC error. The spec requires error code -32601 (Method not found)." },
104
+ { id: "error-method-code", name: "Uses correct JSON-RPC error code for unknown method", category: "errors", required: false, specRef: "basic", description: "Checks the error code is specifically -32601 (Method not found) for unknown methods, as required by JSON-RPC 2.0." },
105
+ { id: "error-invalid-jsonrpc", name: "Handles malformed JSON-RPC", category: "errors", required: true, specRef: "basic", description: "Sends a malformed JSON-RPC message (missing required fields) and verifies the server returns an error or 4xx status." },
106
+ { id: "error-invalid-json", name: "Handles invalid JSON body", category: "errors", required: false, specRef: "basic", description: "Sends invalid JSON and verifies the server returns a parse error (-32700) or 4xx status code." },
107
+ { id: "error-missing-params", name: "Returns error for tools/call without name", category: "errors", required: false, specRef: "server/tools#error-handling", description: "Calls tools/call with an empty params object (missing required name field) and verifies an error is returned." },
108
+ { id: "error-parse-code", name: "Returns -32700 for invalid JSON", category: "errors", required: false, specRef: "basic", description: "Checks that the server returns the specific JSON-RPC error code -32700 (Parse error) when receiving invalid JSON, as required by the JSON-RPC 2.0 specification." },
109
+ { id: "error-invalid-request-code", name: "Returns -32600 for invalid request", category: "errors", required: false, specRef: "basic", description: "Checks that the server returns the specific JSON-RPC error code -32600 (Invalid Request) for malformed JSON-RPC messages missing required fields." },
110
+ { id: "tools-call-unknown", name: "Returns error for unknown tool name", category: "errors", required: false, specRef: "server/tools#error-handling", description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response." },
111
+ // ── Schema Validation (6 tests) ──────────────────────────────────
112
+ { id: "tools-schema", name: "All tools have name and inputSchema", category: "schema", required: false, specRef: "server/tools#data-types", description: 'Validates every tool has a valid name (1-128 chars, alphanumeric/underscore/hyphen/dot) and a required inputSchema of type "object".' },
113
+ { id: "tools-annotations", name: "Tool annotations are valid", category: "schema", required: false, specRef: "server/tools#annotations", description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans; title should be a string." },
114
+ { id: "tools-title-field", name: "Tools include title field", category: "schema", required: false, specRef: "server/tools#data-types", description: "Checks if tools include the optional title field for human-readable display names. Added in spec version 2025-11-25." },
115
+ { id: "tools-output-schema", name: "Tools with outputSchema are valid", category: "schema", required: false, specRef: "server/tools#structured-content", description: 'If tools declare an outputSchema, validates it is a valid JSON Schema object with type "object". Used for structured output validation.' },
116
+ { id: "prompts-schema", name: "Prompts have name field", category: "schema", required: false, specRef: "server/prompts#data-types", description: "Validates every prompt has a name and that any arguments array contains items with name fields." },
117
+ { id: "resources-schema", name: "Resources have uri and name", category: "schema", required: false, specRef: "server/resources#data-types", description: "Validates every resource has a valid URI (parseable as a URL) and a name field." }
118
+ ];
119
+
65
120
  // src/runner.ts
121
+ var _require = createRequire(import.meta.url);
122
+ var { version: TOOL_VERSION } = _require("../package.json");
66
123
  var SPEC_VERSION = "2025-11-25";
67
124
  var SPEC_BASE = `https://modelcontextprotocol.io/specification/${SPEC_VERSION}`;
68
- function createIdCounter() {
69
- let id = 0;
125
+ var VALID_CONTENT_TYPES = ["text", "image", "audio", "resource", "resource_link"];
126
+ function createIdCounter(start = 0) {
127
+ let id = start;
70
128
  return () => ++id;
71
129
  }
72
130
  function parseSSEResponse(text) {
73
131
  const lines = text.split("\n");
74
132
  let lastJsonRpcResponse = null;
75
- for (const line of lines) {
76
- if (line.startsWith("data: ")) {
77
- const data = line.slice(6).trim();
78
- if (!data) continue;
79
- try {
80
- const parsed = JSON.parse(data);
81
- if (parsed.jsonrpc === "2.0" && parsed.id !== void 0) {
82
- lastJsonRpcResponse = parsed;
83
- }
84
- } catch {
133
+ let currentData = [];
134
+ function flushEvent() {
135
+ if (currentData.length === 0) return;
136
+ const data = currentData.join("\n");
137
+ currentData = [];
138
+ if (!data.trim()) return;
139
+ try {
140
+ const parsed = JSON.parse(data);
141
+ if (parsed.jsonrpc === "2.0" && parsed.id !== void 0) {
142
+ lastJsonRpcResponse = parsed;
85
143
  }
144
+ } catch {
145
+ }
146
+ }
147
+ for (const line of lines) {
148
+ if (line.startsWith("data:")) {
149
+ const content = line.slice(5);
150
+ currentData.push(content.startsWith(" ") ? content.slice(1) : content);
151
+ } else if (line.trim() === "") {
152
+ flushEvent();
86
153
  }
87
154
  }
155
+ flushEvent();
88
156
  return lastJsonRpcResponse;
89
157
  }
90
- var _defaultNextId = createIdCounter();
91
- async function mcpRequest(backendUrl, method, params, nextId = _defaultNextId, extraHeaders) {
158
+ async function mcpRequest(backendUrl, method, params, nextId, extraHeaders, timeout) {
92
159
  const id = nextId();
93
160
  const body = JSON.stringify({
94
161
  jsonrpc: "2.0",
@@ -105,7 +172,7 @@ async function mcpRequest(backendUrl, method, params, nextId = _defaultNextId, e
105
172
  method: "POST",
106
173
  headers,
107
174
  body,
108
- signal: AbortSignal.timeout(15e3)
175
+ signal: AbortSignal.timeout(timeout)
109
176
  });
110
177
  const text = await res.body.text();
111
178
  const responseHeaders = {};
@@ -116,32 +183,38 @@ async function mcpRequest(backendUrl, method, params, nextId = _defaultNextId, e
116
183
  if (contentType.includes("text/event-stream")) {
117
184
  const parsed = parseSSEResponse(text);
118
185
  if (parsed) {
119
- return { statusCode: res.statusCode, body: parsed, headers: responseHeaders };
186
+ return { statusCode: res.statusCode, body: parsed, headers: responseHeaders, requestId: id };
120
187
  }
121
188
  try {
122
- return { statusCode: res.statusCode, body: JSON.parse(text), headers: responseHeaders };
189
+ return { statusCode: res.statusCode, body: JSON.parse(text), headers: responseHeaders, requestId: id };
123
190
  } catch {
124
- return { statusCode: res.statusCode, body: { _raw: text }, headers: responseHeaders };
191
+ return { statusCode: res.statusCode, body: { _raw: text }, headers: responseHeaders, requestId: id };
125
192
  }
126
193
  }
127
194
  try {
128
- return { statusCode: res.statusCode, body: JSON.parse(text), headers: responseHeaders };
195
+ return { statusCode: res.statusCode, body: JSON.parse(text), headers: responseHeaders, requestId: id };
129
196
  } catch {
130
- return { statusCode: res.statusCode, body: { _raw: text }, headers: responseHeaders };
197
+ return { statusCode: res.statusCode, body: { _raw: text }, headers: responseHeaders, requestId: id };
131
198
  }
132
199
  }
133
- async function mcpNotification(backendUrl, method, params, extraHeaders) {
200
+ async function mcpNotification(backendUrl, method, params, extraHeaders, timeout) {
134
201
  const headers = {
135
202
  "Content-Type": "application/json",
203
+ "Accept": "application/json, text/event-stream",
136
204
  ...extraHeaders
137
205
  };
138
- await request(backendUrl, {
206
+ const res = await request(backendUrl, {
139
207
  method: "POST",
140
208
  headers,
141
209
  body: JSON.stringify({ jsonrpc: "2.0", method, ...params ? { params } : {} }),
142
- signal: AbortSignal.timeout(5e3)
143
- }).then((r) => r.body.text()).catch(() => {
210
+ signal: AbortSignal.timeout(timeout)
144
211
  });
212
+ await res.body.text();
213
+ const responseHeaders = {};
214
+ for (const [k, v] of Object.entries(res.headers)) {
215
+ if (typeof v === "string") responseHeaders[k] = v;
216
+ }
217
+ return { statusCode: res.statusCode, headers: responseHeaders };
145
218
  }
146
219
  async function runComplianceSuite(url, options = {}) {
147
220
  let parsed;
@@ -156,7 +229,10 @@ async function runComplianceSuite(url, options = {}) {
156
229
  }
157
230
  const backendUrl = url;
158
231
  const tests = [];
159
- const nextId = createIdCounter();
232
+ const warnings = [];
233
+ const nextId = createIdCounter(1e3);
234
+ const timeout = options.timeout || 15e3;
235
+ const retries = options.retries || 0;
160
236
  let sessionId = null;
161
237
  let negotiatedProtocolVersion = null;
162
238
  const userHeaders = options.headers || {};
@@ -166,7 +242,16 @@ async function runComplianceSuite(url, options = {}) {
166
242
  if (negotiatedProtocolVersion) h["mcp-protocol-version"] = negotiatedProtocolVersion;
167
243
  return h;
168
244
  }
169
- const rpc = (method, params) => mcpRequest(backendUrl, method, params, nextId, buildHeaders());
245
+ const rpc = (method, params) => mcpRequest(backendUrl, method, params, nextId, buildHeaders(), timeout);
246
+ function shouldRun(id, category) {
247
+ if (options.only && options.only.length > 0) {
248
+ return options.only.includes(category) || options.only.includes(id);
249
+ }
250
+ if (options.skip && options.skip.length > 0) {
251
+ return !options.skip.includes(category) && !options.skip.includes(id);
252
+ }
253
+ return true;
254
+ }
170
255
  let serverInfo = {
171
256
  protocolVersion: null,
172
257
  name: null,
@@ -176,42 +261,41 @@ async function runComplianceSuite(url, options = {}) {
176
261
  let toolCount = 0;
177
262
  let toolNames = [];
178
263
  let resourceCount = 0;
264
+ let resourceNames = [];
179
265
  let promptCount = 0;
266
+ let promptNames = [];
180
267
  async function test(id, name, category, required, specRef, fn) {
268
+ if (!shouldRun(id, category)) return;
181
269
  const start = Date.now();
182
- try {
183
- const result = await fn();
184
- tests.push({
185
- id,
186
- name,
187
- category,
188
- required,
189
- passed: result.passed,
190
- details: result.details,
191
- durationMs: Date.now() - start,
192
- specRef: `${SPEC_BASE}/${specRef}`
193
- });
194
- options.onProgress?.(id, result.passed, result.details);
195
- } catch (err) {
196
- tests.push({
197
- id,
198
- name,
199
- category,
200
- required,
201
- passed: false,
202
- details: `Error: ${err.message}`,
203
- durationMs: Date.now() - start,
204
- specRef: `${SPEC_BASE}/${specRef}`
205
- });
206
- options.onProgress?.(id, false, `Error: ${err.message}`);
270
+ let lastResult = { passed: false, details: "" };
271
+ for (let attempt = 0; attempt <= retries; attempt++) {
272
+ try {
273
+ lastResult = await fn();
274
+ if (lastResult.passed) break;
275
+ if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
276
+ } catch (err) {
277
+ lastResult = { passed: false, details: `Error: ${err.message}` };
278
+ if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
279
+ }
207
280
  }
281
+ tests.push({
282
+ id,
283
+ name,
284
+ category,
285
+ required,
286
+ passed: lastResult.passed,
287
+ details: lastResult.details,
288
+ durationMs: Date.now() - start,
289
+ specRef: `${SPEC_BASE}/${specRef}`
290
+ });
291
+ options.onProgress?.(id, lastResult.passed, lastResult.details);
208
292
  }
209
293
  await test("transport-post", "HTTP POST accepted", "transport", true, "basic/transports#streamable-http", async () => {
210
294
  const res = await request(backendUrl, {
211
295
  method: "POST",
212
296
  headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...userHeaders },
213
- body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "ping" }),
214
- signal: AbortSignal.timeout(1e4)
297
+ body: JSON.stringify({ jsonrpc: "2.0", id: 99901, method: "ping" }),
298
+ signal: AbortSignal.timeout(timeout)
215
299
  });
216
300
  await res.body.text();
217
301
  const passed = res.statusCode >= 200 && res.statusCode < 300;
@@ -222,8 +306,8 @@ async function runComplianceSuite(url, options = {}) {
222
306
  const res = await request(backendUrl, {
223
307
  method: "POST",
224
308
  headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...userHeaders },
225
- body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "ping" }),
226
- signal: AbortSignal.timeout(1e4)
309
+ body: JSON.stringify({ jsonrpc: "2.0", id: 99902, method: "ping" }),
310
+ signal: AbortSignal.timeout(timeout)
227
311
  });
228
312
  await res.body.text();
229
313
  const rawCt = res.headers["content-type"];
@@ -231,28 +315,105 @@ async function runComplianceSuite(url, options = {}) {
231
315
  const valid = ct.includes("application/json") || ct.includes("text/event-stream");
232
316
  return { passed: valid, details: `Content-Type: ${ct}` };
233
317
  });
318
+ await test("transport-get", "GET returns SSE stream or 405", "transport", false, "basic/transports#streamable-http", async () => {
319
+ const res = await request(backendUrl, {
320
+ method: "GET",
321
+ headers: { "Accept": "text/event-stream", ...userHeaders },
322
+ signal: AbortSignal.timeout(timeout)
323
+ });
324
+ await res.body.text();
325
+ const ct = (res.headers["content-type"] || "").toLowerCase();
326
+ if (res.statusCode === 405) {
327
+ return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
328
+ }
329
+ if (ct.includes("text/event-stream")) {
330
+ return { passed: true, details: "Returns text/event-stream for SSE" };
331
+ }
332
+ if (res.statusCode >= 200 && res.statusCode < 300) {
333
+ return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
334
+ }
335
+ return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
336
+ });
337
+ await test("transport-delete", "DELETE accepted or returns 405", "transport", false, "basic/transports#streamable-http", async () => {
338
+ const res = await request(backendUrl, {
339
+ method: "DELETE",
340
+ headers: { ...userHeaders },
341
+ signal: AbortSignal.timeout(timeout)
342
+ });
343
+ await res.body.text();
344
+ if (res.statusCode === 405) {
345
+ return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
346
+ }
347
+ if (res.statusCode >= 200 && res.statusCode < 300) {
348
+ return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
349
+ }
350
+ if (res.statusCode === 400 || res.statusCode === 404) {
351
+ return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
352
+ }
353
+ return { passed: false, details: `HTTP ${res.statusCode}` };
354
+ });
355
+ await test("transport-batch-reject", "Rejects JSON-RPC batch requests", "transport", true, "basic/transports#streamable-http", async () => {
356
+ const res = await request(backendUrl, {
357
+ method: "POST",
358
+ headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...userHeaders },
359
+ body: JSON.stringify([
360
+ { jsonrpc: "2.0", id: 99903, method: "ping" },
361
+ { jsonrpc: "2.0", id: 99904, method: "ping" }
362
+ ]),
363
+ signal: AbortSignal.timeout(timeout)
364
+ });
365
+ const text = await res.body.text();
366
+ if (res.statusCode >= 400 && res.statusCode < 500) {
367
+ return { passed: true, details: `HTTP ${res.statusCode} (batch rejected)` };
368
+ }
369
+ try {
370
+ const body = JSON.parse(text);
371
+ if (body?.error) {
372
+ return { passed: true, details: `JSON-RPC error: ${body.error.code} \u2014 ${body.error.message}` };
373
+ }
374
+ if (Array.isArray(body)) {
375
+ return { passed: false, details: "Server processed batch request (MCP forbids batch)" };
376
+ }
377
+ } catch {
378
+ }
379
+ return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error or 4xx for batch request` };
380
+ });
234
381
  let initRes = null;
235
- await test("lifecycle-init", "Initialize handshake", "lifecycle", true, "basic/lifecycle#initialization", async () => {
382
+ try {
236
383
  initRes = await rpc("initialize", {
237
384
  protocolVersion: SPEC_VERSION,
238
385
  capabilities: { roots: { listChanged: true }, sampling: {} },
239
- clientInfo: { name: "mcp-compliance", version: "1.0.0" }
386
+ clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
240
387
  });
388
+ const result = initRes?.body?.result;
389
+ if (result) {
390
+ serverInfo.protocolVersion = result.protocolVersion || null;
391
+ serverInfo.name = result.serverInfo?.name || null;
392
+ serverInfo.version = result.serverInfo?.version || null;
393
+ serverInfo.capabilities = result.capabilities || {};
394
+ const sid = initRes.headers["mcp-session-id"];
395
+ if (sid) sessionId = sid;
396
+ if (result.protocolVersion) negotiatedProtocolVersion = result.protocolVersion;
397
+ }
398
+ } catch (err) {
399
+ }
400
+ try {
401
+ await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders(), timeout);
402
+ } catch {
403
+ }
404
+ await test("lifecycle-init", "Initialize handshake", "lifecycle", true, "basic/lifecycle#initialization", async () => {
405
+ if (!initRes) return { passed: false, details: "Initialize request failed" };
241
406
  const result = initRes.body?.result;
242
407
  if (!result) return { passed: false, details: "No result in response" };
243
- serverInfo.protocolVersion = result.protocolVersion || null;
244
- serverInfo.name = result.serverInfo?.name || null;
245
- serverInfo.version = result.serverInfo?.version || null;
246
- serverInfo.capabilities = result.capabilities || {};
247
- const sid = initRes.headers["mcp-session-id"];
248
- if (sid) sessionId = sid;
249
- if (result.protocolVersion) negotiatedProtocolVersion = result.protocolVersion;
250
408
  return { passed: !!result.protocolVersion, details: `Protocol: ${result.protocolVersion || "missing"}` };
251
409
  });
252
410
  await test("lifecycle-proto-version", "Returns valid protocol version", "lifecycle", true, "basic/lifecycle#version-negotiation", async () => {
253
411
  const version2 = initRes?.body?.result?.protocolVersion;
254
412
  if (!version2) return { passed: false, details: "No protocolVersion" };
255
413
  const valid = /^\d{4}-\d{2}-\d{2}$/.test(version2);
414
+ if (valid && version2 !== SPEC_VERSION) {
415
+ warnings.push(`Server negotiated protocol version ${version2} (latest is ${SPEC_VERSION})`);
416
+ }
256
417
  return { passed: valid, details: `Version: ${version2}` };
257
418
  });
258
419
  await test("lifecycle-server-info", "Includes serverInfo", "lifecycle", false, "basic/lifecycle#initialization", async () => {
@@ -270,7 +431,6 @@ async function runComplianceSuite(url, options = {}) {
270
431
  const valid = body?.jsonrpc === "2.0" && body?.id !== void 0 && (body?.result !== void 0 || body?.error !== void 0);
271
432
  return { passed: valid, details: valid ? "Valid JSON-RPC 2.0 response" : `Missing fields: jsonrpc=${body?.jsonrpc}, id=${body?.id}` };
272
433
  });
273
- await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders());
274
434
  await test("lifecycle-ping", "Responds to ping", "lifecycle", true, "basic/utilities#ping", async () => {
275
435
  const res = await rpc("ping");
276
436
  const body = res.body;
@@ -278,6 +438,89 @@ async function runComplianceSuite(url, options = {}) {
278
438
  if (body?.result !== void 0) return { passed: true, details: "Ping responded successfully" };
279
439
  return { passed: false, details: "No result in ping response" };
280
440
  });
441
+ await test("lifecycle-instructions", "Instructions field is valid", "lifecycle", false, "basic/lifecycle#initialization", async () => {
442
+ const result = initRes?.body?.result;
443
+ if (!result) return { passed: false, details: "No init result" };
444
+ if (result.instructions === void 0) {
445
+ return { passed: true, details: "No instructions field (optional)" };
446
+ }
447
+ if (typeof result.instructions === "string") {
448
+ const preview = result.instructions.length > 80 ? result.instructions.slice(0, 80) + "..." : result.instructions;
449
+ return { passed: true, details: `Instructions: "${preview}"` };
450
+ }
451
+ return { passed: false, details: `instructions should be a string, got ${typeof result.instructions}` };
452
+ });
453
+ await test("lifecycle-id-match", "Response ID matches request ID", "lifecycle", true, "basic", async () => {
454
+ const res = await rpc("ping");
455
+ const body = res.body;
456
+ if (body?.id === void 0) return { passed: false, details: "No id in response" };
457
+ const match = body.id === res.requestId;
458
+ return { passed: match, details: match ? `Request id=${res.requestId}, response id=${body.id} (match)` : `Request id=${res.requestId}, response id=${body.id} (MISMATCH)` };
459
+ });
460
+ const hasLogging = !!serverInfo.capabilities.logging;
461
+ await test("lifecycle-logging", "logging/setLevel accepted", "lifecycle", hasLogging, "server/utilities#logging", async () => {
462
+ if (!hasLogging) return { passed: true, details: "Server does not declare logging capability (skipped)" };
463
+ const res = await rpc("logging/setLevel", { level: "info" });
464
+ if (res.body?.error) {
465
+ return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
466
+ }
467
+ return { passed: true, details: "logging/setLevel accepted" };
468
+ });
469
+ const hasCompletions = !!serverInfo.capabilities.completions;
470
+ await test("lifecycle-completions", "completion/complete accepted", "lifecycle", hasCompletions, "server/utilities#completion", async () => {
471
+ if (!hasCompletions) return { passed: true, details: "Server does not declare completions capability (skipped)" };
472
+ const res = await rpc("completion/complete", {
473
+ ref: { type: "ref/prompt", name: "__test__" },
474
+ argument: { name: "test", value: "" }
475
+ });
476
+ if (res.body?.error) {
477
+ if (res.body.error.code === -32602) {
478
+ return { passed: true, details: "InvalidParams for test ref (acceptable)" };
479
+ }
480
+ return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
481
+ }
482
+ const values = res.body?.result?.completion?.values;
483
+ if (Array.isArray(values)) {
484
+ return { passed: true, details: `Returned ${values.length} completion(s)` };
485
+ }
486
+ return { passed: true, details: "completion/complete accepted" };
487
+ });
488
+ await test("transport-notification-202", "Notification returns 202 Accepted", "transport", false, "basic/transports#streamable-http", async () => {
489
+ const res = await request(backendUrl, {
490
+ method: "POST",
491
+ headers: {
492
+ "Content-Type": "application/json",
493
+ "Accept": "application/json, text/event-stream",
494
+ ...buildHeaders()
495
+ },
496
+ body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/cancelled", params: { requestId: "nonexistent", reason: "compliance test" } }),
497
+ signal: AbortSignal.timeout(timeout)
498
+ });
499
+ await res.body.text();
500
+ if (res.statusCode === 202) {
501
+ return { passed: true, details: "HTTP 202 Accepted (correct)" };
502
+ }
503
+ if (res.statusCode >= 200 && res.statusCode < 300) {
504
+ return { passed: true, details: `HTTP ${res.statusCode} (accepted, but 202 is preferred)` };
505
+ }
506
+ return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected 202 Accepted for notifications` };
507
+ });
508
+ await test("transport-session-id", "Enforces MCP-Session-Id after init", "transport", false, "basic/transports#streamable-http", async () => {
509
+ if (!sessionId) {
510
+ warnings.push("Server did not issue MCP-Session-Id header");
511
+ return { passed: true, details: "Server did not issue session ID (test not applicable)" };
512
+ }
513
+ const headersWithout = { ...userHeaders };
514
+ if (negotiatedProtocolVersion) headersWithout["mcp-protocol-version"] = negotiatedProtocolVersion;
515
+ const res = await mcpRequest(backendUrl, "ping", void 0, createIdCounter(99910), headersWithout, timeout);
516
+ if (res.statusCode === 400) {
517
+ return { passed: true, details: "HTTP 400 for missing session ID (correct)" };
518
+ }
519
+ if (res.statusCode >= 200 && res.statusCode < 300) {
520
+ return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should return 400 when session ID is missing` };
521
+ }
522
+ return { passed: false, details: `HTTP ${res.statusCode}` };
523
+ });
281
524
  const hasTools = !!serverInfo.capabilities.tools;
282
525
  let cachedToolsList = null;
283
526
  await test("tools-list", "tools/list returns valid response", "tools", hasTools, "server/tools#listing-tools", async () => {
@@ -292,7 +535,6 @@ async function runComplianceSuite(url, options = {}) {
292
535
  await test("tools-schema", "All tools have name and inputSchema", "schema", hasTools, "server/tools#data-types", async () => {
293
536
  const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
294
537
  const issues = [];
295
- const warnings = [];
296
538
  for (const tool of tools) {
297
539
  if (!tool.name) {
298
540
  issues.push("Tool missing name");
@@ -301,7 +543,7 @@ async function runComplianceSuite(url, options = {}) {
301
543
  if (tool.name.length > 128 || !/^[A-Za-z0-9_.\-]+$/.test(tool.name)) {
302
544
  issues.push(`${tool.name}: name format invalid`);
303
545
  }
304
- if (!tool.description) warnings.push(`${tool.name}: missing description`);
546
+ if (!tool.description) warnings.push(`Tool "${tool.name}" missing description`);
305
547
  if (!tool.inputSchema) {
306
548
  issues.push(`${tool.name}: missing inputSchema (required)`);
307
549
  } else if (typeof tool.inputSchema !== "object" || tool.inputSchema === null) {
@@ -310,9 +552,68 @@ async function runComplianceSuite(url, options = {}) {
310
552
  issues.push(`${tool.name}: inputSchema.type must be "object" (got "${tool.inputSchema.type || "undefined"}")`);
311
553
  }
312
554
  }
313
- const detail = issues.length === 0 ? warnings.length > 0 ? `Schemas valid. Warnings: ${warnings.join("; ")}` : "All tools have valid schemas" : issues.join("; ");
555
+ const detail = issues.length === 0 ? "All tools have valid schemas" : issues.join("; ");
314
556
  return { passed: issues.length === 0, details: detail };
315
557
  });
558
+ await test("tools-annotations", "Tool annotations are valid", "schema", false, "server/tools#annotations", async () => {
559
+ const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
560
+ if (tools.length === 0) return { passed: true, details: "No tools to validate" };
561
+ const issues = [];
562
+ let annotatedCount = 0;
563
+ for (const tool of tools) {
564
+ const ann = tool.annotations;
565
+ if (!ann) continue;
566
+ annotatedCount++;
567
+ if (typeof ann !== "object" || ann === null) {
568
+ issues.push(`${tool.name}: annotations must be an object`);
569
+ continue;
570
+ }
571
+ const boolFields = ["readOnlyHint", "destructiveHint", "idempotentHint", "openWorldHint"];
572
+ for (const field of boolFields) {
573
+ if (ann[field] !== void 0 && typeof ann[field] !== "boolean") {
574
+ issues.push(`${tool.name}: annotations.${field} should be boolean, got ${typeof ann[field]}`);
575
+ }
576
+ }
577
+ if (ann.title !== void 0 && typeof ann.title !== "string") {
578
+ issues.push(`${tool.name}: annotations.title should be string`);
579
+ }
580
+ }
581
+ if (issues.length > 0) return { passed: false, details: issues.join("; ") };
582
+ return { passed: true, details: annotatedCount > 0 ? `${annotatedCount} tool(s) with valid annotations` : "No tools have annotations (optional)" };
583
+ });
584
+ await test("tools-title-field", "Tools include title field", "schema", false, "server/tools#data-types", async () => {
585
+ const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
586
+ if (tools.length === 0) return { passed: true, details: "No tools to validate" };
587
+ const withTitle = tools.filter((t) => typeof t.title === "string");
588
+ const issues = [];
589
+ for (const tool of tools) {
590
+ if (tool.title !== void 0 && typeof tool.title !== "string") {
591
+ issues.push(`${tool.name}: title should be a string`);
592
+ }
593
+ }
594
+ if (issues.length > 0) return { passed: false, details: issues.join("; ") };
595
+ if (withTitle.length === 0) {
596
+ return { passed: true, details: "No tools have title field (optional, added in 2025-11-25)" };
597
+ }
598
+ return { passed: true, details: `${withTitle.length}/${tools.length} tool(s) have title field` };
599
+ });
600
+ await test("tools-output-schema", "Tools with outputSchema are valid", "schema", false, "server/tools#structured-content", async () => {
601
+ const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
602
+ if (tools.length === 0) return { passed: true, details: "No tools to validate" };
603
+ const issues = [];
604
+ let withSchema = 0;
605
+ for (const tool of tools) {
606
+ if (tool.outputSchema === void 0) continue;
607
+ withSchema++;
608
+ if (typeof tool.outputSchema !== "object" || tool.outputSchema === null) {
609
+ issues.push(`${tool.name}: outputSchema must be a JSON Schema object`);
610
+ } else if (tool.outputSchema.type !== "object") {
611
+ issues.push(`${tool.name}: outputSchema.type must be "object" (got "${tool.outputSchema.type || "undefined"}")`);
612
+ }
613
+ }
614
+ if (issues.length > 0) return { passed: false, details: issues.join("; ") };
615
+ return { passed: true, details: withSchema > 0 ? `${withSchema} tool(s) with valid outputSchema` : "No tools have outputSchema (optional)" };
616
+ });
316
617
  if (toolNames.length > 0) {
317
618
  await test("tools-call", "tools/call responds correctly", "tools", false, "server/tools#calling-tools", async () => {
318
619
  const res = await rpc("tools/call", { name: toolNames[0], arguments: {} });
@@ -335,6 +636,51 @@ async function runComplianceSuite(url, options = {}) {
335
636
  }
336
637
  return { passed: false, details: "Response missing content array" };
337
638
  });
639
+ await test("tools-content-types", "Tool content items have valid types", "tools", false, "server/tools#calling-tools", async () => {
640
+ const res = await rpc("tools/call", { name: toolNames[0], arguments: {} });
641
+ const result = res.body?.result;
642
+ const error = res.body?.error;
643
+ if (error) {
644
+ return { passed: true, details: `Tool returned error (content types not applicable): code ${error.code}` };
645
+ }
646
+ const content = result?.content;
647
+ if (!Array.isArray(content) || content.length === 0) {
648
+ return { passed: true, details: "No content items to validate" };
649
+ }
650
+ const issues = [];
651
+ const types = /* @__PURE__ */ new Set();
652
+ for (const item of content) {
653
+ if (!item.type) {
654
+ issues.push("Content item missing type field");
655
+ } else if (!VALID_CONTENT_TYPES.includes(item.type)) {
656
+ issues.push(`Unknown content type: "${item.type}"`);
657
+ } else {
658
+ types.add(item.type);
659
+ }
660
+ }
661
+ if (issues.length > 0) return { passed: false, details: issues.join("; ") };
662
+ return { passed: true, details: `Content types: ${[...types].join(", ")}` };
663
+ });
664
+ }
665
+ if (hasTools) {
666
+ await test("tools-pagination", "tools/list supports pagination", "tools", false, "server/tools#listing-tools", async () => {
667
+ const res = await rpc("tools/list");
668
+ const result = res.body?.result;
669
+ if (!result) return { passed: false, details: "No result from tools/list" };
670
+ if (!Array.isArray(result.tools)) return { passed: false, details: "No tools array" };
671
+ if (result.nextCursor !== void 0) {
672
+ if (typeof result.nextCursor !== "string") {
673
+ return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
674
+ }
675
+ const nextRes = await rpc("tools/list", { cursor: result.nextCursor });
676
+ const nextResult = nextRes.body?.result;
677
+ if (!nextResult || !Array.isArray(nextResult.tools)) {
678
+ return { passed: false, details: "Next page failed to return tools array" };
679
+ }
680
+ return { passed: true, details: `Pagination works: page 1 had ${result.tools.length} tools, page 2 had ${nextResult.tools.length} tools` };
681
+ }
682
+ return { passed: true, details: `${result.tools.length} tool(s), no nextCursor (single page)` };
683
+ });
338
684
  await test("tools-call-unknown", "Returns error for unknown tool name", "errors", false, "server/tools#error-handling", async () => {
339
685
  const res = await rpc("tools/call", { name: "__nonexistent_tool_compliance_test__", arguments: {} });
340
686
  const error = res.body?.error;
@@ -345,6 +691,7 @@ async function runComplianceSuite(url, options = {}) {
345
691
  });
346
692
  }
347
693
  const hasResources = !!serverInfo.capabilities.resources;
694
+ const hasSubscribe = !!serverInfo.capabilities.resources?.subscribe;
348
695
  if (hasResources) {
349
696
  let cachedResourcesList = null;
350
697
  await test("resources-list", "resources/list returns valid response", "resources", true, "server/resources#listing-resources", async () => {
@@ -353,6 +700,7 @@ async function runComplianceSuite(url, options = {}) {
353
700
  if (!Array.isArray(resources)) return { passed: false, details: "No resources array" };
354
701
  cachedResourcesList = resources;
355
702
  resourceCount = resources.length;
703
+ resourceNames = resources.map((r) => r.name).filter(Boolean);
356
704
  return { passed: true, details: `${resourceCount} resource(s)` };
357
705
  });
358
706
  await test("resources-schema", "Resources have uri and name", "schema", true, "server/resources#data-types", async () => {
@@ -368,13 +716,15 @@ async function runComplianceSuite(url, options = {}) {
368
716
  }
369
717
  }
370
718
  if (!r.name) issues.push(`${r.uri || "?"}: missing name`);
719
+ if (!r.description) warnings.push(`Resource "${r.name || r.uri}" missing description`);
720
+ if (!r.mimeType) warnings.push(`Resource "${r.name || r.uri}" missing mimeType`);
371
721
  }
372
722
  return { passed: issues.length === 0, details: issues.length === 0 ? "All resources valid" : issues.join("; ") };
373
723
  });
374
724
  if (resourceCount > 0) {
375
725
  await test("resources-read", "resources/read returns content", "resources", false, "server/resources#reading-resources", async () => {
376
- const listRes = await rpc("resources/list");
377
- const firstUri = listRes.body?.result?.resources?.[0]?.uri;
726
+ const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
727
+ const firstUri = resources[0]?.uri;
378
728
  if (!firstUri) return { passed: false, details: "No resource URI to test" };
379
729
  const readRes = await rpc("resources/read", { uri: firstUri });
380
730
  const contents = readRes.body?.result?.contents;
@@ -405,10 +755,43 @@ async function runComplianceSuite(url, options = {}) {
405
755
  if (issues.length > 0) return { passed: false, details: issues.join("; ") };
406
756
  return { passed: true, details: `${templates.length} resource template(s)` };
407
757
  });
758
+ await test("resources-pagination", "resources/list supports pagination", "resources", false, "server/resources#listing-resources", async () => {
759
+ const res = await rpc("resources/list");
760
+ const result = res.body?.result;
761
+ if (!result) return { passed: false, details: "No result from resources/list" };
762
+ if (!Array.isArray(result.resources)) return { passed: false, details: "No resources array" };
763
+ if (result.nextCursor !== void 0) {
764
+ if (typeof result.nextCursor !== "string") {
765
+ return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
766
+ }
767
+ const nextRes = await rpc("resources/list", { cursor: result.nextCursor });
768
+ const nextResult = nextRes.body?.result;
769
+ if (!nextResult || !Array.isArray(nextResult.resources)) {
770
+ return { passed: false, details: "Next page failed to return resources array" };
771
+ }
772
+ return { passed: true, details: `Pagination works: page 1 had ${result.resources.length}, page 2 had ${nextResult.resources.length}` };
773
+ }
774
+ return { passed: true, details: `${result.resources.length} resource(s), no nextCursor (single page)` };
775
+ });
776
+ if (hasSubscribe && resourceCount > 0) {
777
+ await test("resources-subscribe", "Resource subscribe/unsubscribe", "resources", true, "server/resources#subscriptions", async () => {
778
+ const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
779
+ const firstUri = resources[0]?.uri;
780
+ if (!firstUri) return { passed: false, details: "No resource URI for subscribe test" };
781
+ const subRes = await rpc("resources/subscribe", { uri: firstUri });
782
+ if (subRes.body?.error) {
783
+ return { passed: false, details: `Subscribe error: ${subRes.body.error.code} \u2014 ${subRes.body.error.message}` };
784
+ }
785
+ const unsubRes = await rpc("resources/unsubscribe", { uri: firstUri });
786
+ if (unsubRes.body?.error) {
787
+ return { passed: false, details: `Unsubscribe error: ${unsubRes.body.error.code} \u2014 ${unsubRes.body.error.message}` };
788
+ }
789
+ return { passed: true, details: `Subscribe/unsubscribe for ${firstUri} succeeded` };
790
+ });
791
+ }
408
792
  }
409
793
  const hasPrompts = !!serverInfo.capabilities.prompts;
410
794
  if (hasPrompts) {
411
- let promptNames = [];
412
795
  let cachedPromptsList = null;
413
796
  await test("prompts-list", "prompts/list returns valid response", "prompts", true, "server/prompts#listing-prompts", async () => {
414
797
  const res = await rpc("prompts/list");
@@ -424,6 +807,7 @@ async function runComplianceSuite(url, options = {}) {
424
807
  const issues = [];
425
808
  for (const p of prompts) {
426
809
  if (!p.name) issues.push("Prompt missing name");
810
+ if (!p.description) warnings.push(`Prompt "${p.name || "?"}" missing description`);
427
811
  if (p.arguments && !Array.isArray(p.arguments)) issues.push(`${p.name || "?"}: arguments must be an array`);
428
812
  if (Array.isArray(p.arguments)) {
429
813
  for (const arg of p.arguments) {
@@ -449,6 +833,24 @@ async function runComplianceSuite(url, options = {}) {
449
833
  return { passed: true, details: `${messages.length} message(s) from ${promptNames[0]}` };
450
834
  });
451
835
  }
836
+ await test("prompts-pagination", "prompts/list supports pagination", "prompts", false, "server/prompts#listing-prompts", async () => {
837
+ const res = await rpc("prompts/list");
838
+ const result = res.body?.result;
839
+ if (!result) return { passed: false, details: "No result from prompts/list" };
840
+ if (!Array.isArray(result.prompts)) return { passed: false, details: "No prompts array" };
841
+ if (result.nextCursor !== void 0) {
842
+ if (typeof result.nextCursor !== "string") {
843
+ return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
844
+ }
845
+ const nextRes = await rpc("prompts/list", { cursor: result.nextCursor });
846
+ const nextResult = nextRes.body?.result;
847
+ if (!nextResult || !Array.isArray(nextResult.prompts)) {
848
+ return { passed: false, details: "Next page failed to return prompts array" };
849
+ }
850
+ return { passed: true, details: `Pagination works: page 1 had ${result.prompts.length}, page 2 had ${nextResult.prompts.length}` };
851
+ }
852
+ return { passed: true, details: `${result.prompts.length} prompt(s), no nextCursor (single page)` };
853
+ });
452
854
  }
453
855
  await test("error-unknown-method", "Returns JSON-RPC error for unknown method", "errors", true, "basic", async () => {
454
856
  const res = await rpc("nonexistent/method");
@@ -469,9 +871,9 @@ async function runComplianceSuite(url, options = {}) {
469
871
  await test("error-invalid-jsonrpc", "Handles malformed JSON-RPC", "errors", true, "basic", async () => {
470
872
  const res = await request(backendUrl, {
471
873
  method: "POST",
472
- headers: { "Content-Type": "application/json", ...buildHeaders() },
874
+ headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...buildHeaders() },
473
875
  body: JSON.stringify({ not: "a valid jsonrpc message" }),
474
- signal: AbortSignal.timeout(1e4)
876
+ signal: AbortSignal.timeout(timeout)
475
877
  });
476
878
  const text = await res.body.text();
477
879
  try {
@@ -488,9 +890,9 @@ async function runComplianceSuite(url, options = {}) {
488
890
  await test("error-invalid-json", "Handles invalid JSON body", "errors", false, "basic", async () => {
489
891
  const res = await request(backendUrl, {
490
892
  method: "POST",
491
- headers: { "Content-Type": "application/json", ...buildHeaders() },
893
+ headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...buildHeaders() },
492
894
  body: "{this is not valid json!!!",
493
- signal: AbortSignal.timeout(1e4)
895
+ signal: AbortSignal.timeout(timeout)
494
896
  });
495
897
  const text = await res.body.text();
496
898
  try {
@@ -512,10 +914,57 @@ async function runComplianceSuite(url, options = {}) {
512
914
  if (isError) return { passed: true, details: "Tool execution error (valid)" };
513
915
  return { passed: false, details: "No error for tools/call without name" };
514
916
  });
917
+ await test("error-parse-code", "Returns -32700 for invalid JSON", "errors", false, "basic", async () => {
918
+ const res = await request(backendUrl, {
919
+ method: "POST",
920
+ headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...buildHeaders() },
921
+ body: "<<<not json>>>",
922
+ signal: AbortSignal.timeout(timeout)
923
+ });
924
+ const text = await res.body.text();
925
+ try {
926
+ const body = JSON.parse(text);
927
+ if (body?.error?.code === -32700) {
928
+ return { passed: true, details: `Error code: -32700 (Parse error) \u2014 ${body.error.message}` };
929
+ }
930
+ if (body?.error) {
931
+ return { passed: false, details: `Expected -32700, got ${body.error.code} \u2014 ${body.error.message}` };
932
+ }
933
+ } catch {
934
+ }
935
+ if (res.statusCode >= 400 && res.statusCode < 500) {
936
+ return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should return JSON-RPC error code -32700` };
937
+ }
938
+ return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error code -32700` };
939
+ });
940
+ await test("error-invalid-request-code", "Returns -32600 for invalid request", "errors", false, "basic", async () => {
941
+ const res = await request(backendUrl, {
942
+ method: "POST",
943
+ headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...buildHeaders() },
944
+ body: JSON.stringify({ jsonrpc: "2.0", id: 99999 }),
945
+ signal: AbortSignal.timeout(timeout)
946
+ });
947
+ const text = await res.body.text();
948
+ try {
949
+ const body = JSON.parse(text);
950
+ if (body?.error?.code === -32600) {
951
+ return { passed: true, details: `Error code: -32600 (Invalid Request) \u2014 ${body.error.message}` };
952
+ }
953
+ if (body?.error) {
954
+ return { passed: false, details: `Expected -32600, got ${body.error.code} \u2014 ${body.error.message}` };
955
+ }
956
+ } catch {
957
+ }
958
+ if (res.statusCode >= 400 && res.statusCode < 500) {
959
+ return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should return JSON-RPC error code -32600` };
960
+ }
961
+ return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error code -32600` };
962
+ });
515
963
  const { score, grade, overall, summary, categories } = computeScore(tests);
516
964
  const badge = generateBadge(url);
517
965
  return {
518
966
  specVersion: SPEC_VERSION,
967
+ toolVersion: TOOL_VERSION,
519
968
  url,
520
969
  timestamp: (/* @__PURE__ */ new Date()).toISOString(),
521
970
  score,
@@ -524,11 +973,14 @@ async function runComplianceSuite(url, options = {}) {
524
973
  summary,
525
974
  categories,
526
975
  tests,
976
+ warnings,
527
977
  serverInfo,
528
978
  toolCount,
529
979
  toolNames,
530
980
  resourceCount,
981
+ resourceNames,
531
982
  promptCount,
983
+ promptNames,
532
984
  badge
533
985
  };
534
986
  }
@@ -582,7 +1034,7 @@ function formatTerminal(report) {
582
1034
  const lines = [];
583
1035
  lines.push("");
584
1036
  lines.push(chalk.bold("MCP Compliance Report"));
585
- lines.push(chalk.dim(`Spec: ${report.specVersion} | ${report.timestamp}`));
1037
+ lines.push(chalk.dim(`Spec: ${report.specVersion} | Tool: v${report.toolVersion} | ${report.timestamp}`));
586
1038
  lines.push(chalk.dim(`URL: ${report.url}`));
587
1039
  if (report.serverInfo.name) {
588
1040
  lines.push(chalk.dim(`Server: ${report.serverInfo.name} v${report.serverInfo.version || "?"} (protocol ${report.serverInfo.protocolVersion || "?"})`));
@@ -608,15 +1060,27 @@ function formatTerminal(report) {
608
1060
  lines.push(testLine(t));
609
1061
  }
610
1062
  }
611
- if (report.toolCount > 0) {
1063
+ const caps = report.serverInfo.capabilities;
1064
+ const declared = Object.keys(caps).filter((k) => caps[k] !== void 0);
1065
+ if (declared.length > 0) {
612
1066
  lines.push("");
1067
+ lines.push(chalk.dim(` Capabilities: ${declared.join(", ")}`));
1068
+ }
1069
+ if (report.toolCount > 0) {
613
1070
  lines.push(chalk.dim(` Tools (${report.toolCount}): ${report.toolNames.slice(0, 10).join(", ")}${report.toolCount > 10 ? "..." : ""}`));
614
1071
  }
615
1072
  if (report.resourceCount > 0) {
616
- lines.push(chalk.dim(` Resources: ${report.resourceCount}`));
1073
+ lines.push(chalk.dim(` Resources (${report.resourceCount}): ${report.resourceNames.slice(0, 10).join(", ")}${report.resourceCount > 10 ? "..." : ""}`));
617
1074
  }
618
1075
  if (report.promptCount > 0) {
619
- lines.push(chalk.dim(` Prompts: ${report.promptCount}`));
1076
+ lines.push(chalk.dim(` Prompts (${report.promptCount}): ${report.promptNames.slice(0, 10).join(", ")}${report.promptCount > 10 ? "..." : ""}`));
1077
+ }
1078
+ if (report.warnings.length > 0) {
1079
+ lines.push("");
1080
+ lines.push(chalk.yellow(` Warnings (${report.warnings.length}):`));
1081
+ for (const w of report.warnings) {
1082
+ lines.push(chalk.yellow(` - ${w}`));
1083
+ }
620
1084
  }
621
1085
  lines.push("");
622
1086
  lines.push(chalk.dim(" Badge markdown:"));
@@ -629,7 +1093,10 @@ function formatJson(report) {
629
1093
  }
630
1094
 
631
1095
  // src/index.ts
632
- var require2 = createRequire(import.meta.url);
1096
+ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
1097
+ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
1098
+ import { z } from "zod";
1099
+ var require2 = createRequire2(import.meta.url);
633
1100
  var { version } = require2("../package.json");
634
1101
  function parseHeaderArg(value, prev) {
635
1102
  const idx = value.indexOf(":");
@@ -641,9 +1108,12 @@ function parseHeaderArg(value, prev) {
641
1108
  prev[key] = val;
642
1109
  return prev;
643
1110
  }
1111
+ function parseList(value) {
1112
+ return value.split(",").map((s) => s.trim()).filter(Boolean);
1113
+ }
644
1114
  var program = new Command();
645
1115
  program.name("mcp-compliance").description("Test MCP servers for spec compliance").version(version);
646
- program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal or json", "terminal").option("--strict", "Exit with code 1 on any required test failure (for CI)").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').action(async (url, opts) => {
1116
+ program.command("test").description("Run the full compliance test suite against an MCP server").argument("<url>", "MCP server URL to test").option("--format <format>", "Output format: terminal or json", "terminal").option("--strict", "Exit with code 1 on any required test failure (for CI)").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").option("--retries <n>", "Number of retries for failed tests", "0").option("--only <items>", "Only run tests matching these categories or test IDs (comma-separated)", parseList).option("--skip <items>", "Skip tests matching these categories or test IDs (comma-separated)", parseList).option("--verbose", "Print each test result as it runs").action(async (url, opts) => {
647
1117
  try {
648
1118
  const headers = { ...opts.header };
649
1119
  if (opts.auth) headers["Authorization"] = opts.auth;
@@ -652,7 +1122,20 @@ program.command("test").description("Run the full compliance test suite against
652
1122
  Testing ${url}...
653
1123
  `));
654
1124
  }
655
- const report = await runComplianceSuite(url, { headers });
1125
+ const report = await runComplianceSuite(url, {
1126
+ headers,
1127
+ timeout: parseInt(opts.timeout, 10) || 15e3,
1128
+ retries: parseInt(opts.retries, 10) || 0,
1129
+ only: opts.only,
1130
+ skip: opts.skip,
1131
+ onProgress: opts.verbose ? (testId, passed, details) => {
1132
+ const icon = passed ? chalk2.green("PASS") : chalk2.red("FAIL");
1133
+ console.log(` ${icon} ${testId} \u2014 ${details}`);
1134
+ } : void 0
1135
+ });
1136
+ if (opts.verbose && opts.format === "terminal") {
1137
+ console.log("");
1138
+ }
656
1139
  if (opts.format === "json") {
657
1140
  console.log(formatJson(report));
658
1141
  } else {
@@ -672,14 +1155,17 @@ Error: ${err.message}
672
1155
  process.exit(1);
673
1156
  }
674
1157
  });
675
- program.command("badge").description("Run tests and output just the badge markdown embed code").argument("<url>", "MCP server URL to test").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').action(async (url, opts) => {
1158
+ program.command("badge").description("Run tests and output just the badge markdown embed code").argument("<url>", "MCP server URL to test").option("-H, --header <header>", 'Add header to all requests (format: "Key: Value", repeatable)', parseHeaderArg, {}).option("--auth <token>", 'Shorthand for -H "Authorization: <token>"').option("--timeout <ms>", "Request timeout in milliseconds", "15000").action(async (url, opts) => {
676
1159
  try {
677
1160
  const headers = { ...opts.header };
678
1161
  if (opts.auth) headers["Authorization"] = opts.auth;
679
1162
  console.log(chalk2.dim(`
680
1163
  Testing ${url}...
681
1164
  `));
682
- const report = await runComplianceSuite(url, { headers });
1165
+ const report = await runComplianceSuite(url, {
1166
+ headers,
1167
+ timeout: parseInt(opts.timeout, 10) || 15e3
1168
+ });
683
1169
  console.log(`Grade: ${report.grade} (${report.score}%)
684
1170
  `);
685
1171
  console.log(report.badge.markdown);
@@ -691,4 +1177,88 @@ Error: ${err.message}
691
1177
  process.exit(1);
692
1178
  }
693
1179
  });
1180
+ program.command("mcp").description("Start the MCP compliance server (stdio transport)").action(async () => {
1181
+ const server = new McpServer({ name: "mcp-compliance", version });
1182
+ server.tool(
1183
+ "mcp_compliance_test",
1184
+ "Run the full MCP compliance test suite against a server URL. Returns grade (A-F), score, and detailed results for all 43 tests covering transport, lifecycle, tools, resources, prompts, errors, and schema validation.",
1185
+ { url: z.string().url().describe("The MCP server URL to test (must be HTTP or HTTPS)") },
1186
+ async ({ url }) => {
1187
+ try {
1188
+ const report = await runComplianceSuite(url);
1189
+ const summary = [
1190
+ `Grade: ${report.grade} (${report.score}%)`,
1191
+ `Overall: ${report.overall}`,
1192
+ `Tests: ${report.summary.passed}/${report.summary.total} passed (${report.summary.requiredPassed}/${report.summary.required} required)`,
1193
+ "",
1194
+ ...report.tests.map(
1195
+ (t) => `${t.passed ? "PASS" : "FAIL"} ${t.name}${t.required ? " (required)" : ""} \u2014 ${t.details}`
1196
+ )
1197
+ ];
1198
+ if (report.serverInfo.name) {
1199
+ summary.unshift(`Server: ${report.serverInfo.name} v${report.serverInfo.version || "?"}`);
1200
+ }
1201
+ if (report.warnings.length > 0) {
1202
+ summary.push("", `Warnings (${report.warnings.length}):`);
1203
+ for (const w of report.warnings) summary.push(` - ${w}`);
1204
+ }
1205
+ return {
1206
+ content: [
1207
+ { type: "text", text: summary.join("\n") },
1208
+ { type: "text", text: `
1209
+
1210
+ Full report:
1211
+ ${JSON.stringify(report, null, 2)}` }
1212
+ ]
1213
+ };
1214
+ } catch (err) {
1215
+ return { content: [{ type: "text", text: `Error running compliance test: ${err.message}` }], isError: true };
1216
+ }
1217
+ }
1218
+ );
1219
+ server.tool(
1220
+ "mcp_compliance_badge",
1221
+ "Get the badge markdown embed code for an MCP server. Runs the compliance test suite first to determine the grade.",
1222
+ { url: z.string().url().describe("The MCP server URL to test") },
1223
+ async ({ url }) => {
1224
+ try {
1225
+ const report = await runComplianceSuite(url);
1226
+ const badge = report.badge;
1227
+ return {
1228
+ content: [{
1229
+ type: "text",
1230
+ text: [`Grade: ${report.grade} (${report.score}%)`, "", "Markdown:", badge.markdown, "", "HTML:", badge.html].join("\n")
1231
+ }]
1232
+ };
1233
+ } catch (err) {
1234
+ return { content: [{ type: "text", text: `Error: ${err.message}` }], isError: true };
1235
+ }
1236
+ }
1237
+ );
1238
+ server.tool(
1239
+ "mcp_compliance_explain",
1240
+ "Explain what a specific compliance test ID checks and why it matters.",
1241
+ { testId: z.string().describe('The test ID to explain (e.g., "transport-post", "lifecycle-init", "tools-schema")') },
1242
+ async ({ testId }) => {
1243
+ const def = TEST_DEFINITIONS.find((t) => t.id === testId);
1244
+ if (!def) {
1245
+ return {
1246
+ content: [{ type: "text", text: `Unknown test ID: "${testId}"
1247
+
1248
+ Valid test IDs:
1249
+ ${TEST_DEFINITIONS.map((t) => t.id).join(", ")}` }],
1250
+ isError: true
1251
+ };
1252
+ }
1253
+ return {
1254
+ content: [{
1255
+ type: "text",
1256
+ text: [`Test: ${def.id}`, `Name: ${def.name}`, `Category: ${def.category}`, `Required: ${def.required ? "Yes" : "No"}`, `Spec reference: https://modelcontextprotocol.io/specification/2025-11-25/${def.specRef}`, "", def.description].join("\n")
1257
+ }]
1258
+ };
1259
+ }
1260
+ );
1261
+ const transport = new StdioServerTransport();
1262
+ await server.connect(transport);
1263
+ });
694
1264
  program.parse();