@yawlabs/mcp-compliance 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,987 +0,0 @@
1
- // src/runner.ts
2
- import { request } from "undici";
3
- import { createRequire } from "module";
4
-
5
- // src/grader.ts
6
- function computeGrade(score) {
7
- if (score >= 90) return "A";
8
- if (score >= 75) return "B";
9
- if (score >= 60) return "C";
10
- if (score >= 40) return "D";
11
- return "F";
12
- }
13
- function computeScore(tests) {
14
- const total = tests.length;
15
- const passed = tests.filter((t) => t.passed).length;
16
- const failed = total - passed;
17
- const requiredTests = tests.filter((t) => t.required);
18
- const requiredPassed = requiredTests.filter((t) => t.passed).length;
19
- const requiredScore = requiredTests.length > 0 ? requiredPassed / requiredTests.length * 70 : 70;
20
- const optionalTests = tests.filter((t) => !t.required);
21
- const optionalPassed = optionalTests.filter((t) => t.passed).length;
22
- const optionalScore = optionalTests.length > 0 ? optionalPassed / optionalTests.length * 30 : 30;
23
- const score = Math.round(requiredScore + optionalScore);
24
- const overall = requiredPassed === requiredTests.length ? passed === total ? "pass" : "partial" : "fail";
25
- const categories = {};
26
- for (const t of tests) {
27
- if (!categories[t.category]) categories[t.category] = { passed: 0, total: 0 };
28
- categories[t.category].total++;
29
- if (t.passed) categories[t.category].passed++;
30
- }
31
- return {
32
- score,
33
- grade: computeGrade(score),
34
- overall,
35
- summary: { total, passed, failed, required: requiredTests.length, requiredPassed },
36
- categories
37
- };
38
- }
39
-
40
- // src/badge.ts
41
- function generateBadge(url) {
42
- let parsed;
43
- try {
44
- parsed = new URL(url);
45
- } catch {
46
- parsed = new URL("https://unknown");
47
- }
48
- const encoded = encodeURIComponent(parsed.href);
49
- const imageUrl = `https://mcp.hosting/api/compliance/${encoded}/badge`;
50
- const reportUrl = `https://mcp.hosting/compliance/${encoded}`;
51
- return {
52
- imageUrl,
53
- reportUrl,
54
- markdown: `[![MCP Compliant](${imageUrl})](${reportUrl})`,
55
- html: `<a href="${reportUrl}"><img src="${imageUrl}" alt="MCP Compliant"></a>`
56
- };
57
- }
58
-
59
- // src/types.ts
60
- var TEST_DEFINITIONS = [
61
- // ── Transport (7 tests) ──────────────────────────────────────────
62
- { id: "transport-post", name: "HTTP POST accepted", category: "transport", required: true, specRef: "basic/transports#streamable-http", description: "Verifies the server accepts HTTP POST requests and returns a 2xx status code. This is the fundamental transport requirement for Streamable HTTP MCP servers." },
63
- { id: "transport-content-type", name: "Responds with JSON or SSE", category: "transport", required: true, specRef: "basic/transports#streamable-http", description: "Checks that the server responds with Content-Type application/json or text/event-stream. MCP servers must use one of these two content types." },
64
- { id: "transport-notification-202", name: "Notification returns 202 Accepted", category: "transport", required: false, specRef: "basic/transports#streamable-http", description: "Verifies that sending a JSON-RPC notification (no id field) returns HTTP 202 Accepted with no body. Per spec, servers MUST return 202 for notifications." },
65
- { id: "transport-session-id", name: "Enforces MCP-Session-Id after init", category: "transport", required: false, specRef: "basic/transports#streamable-http", description: "Tests that the server returns HTTP 400 when MCP-Session-Id header is missing on requests after initialization (when the server issued a session ID)." },
66
- { id: "transport-get", name: "GET returns SSE stream or 405", category: "transport", required: false, specRef: "basic/transports#streamable-http", description: "Tests the GET endpoint for server-initiated messages. Server should return text/event-stream or 405 Method Not Allowed." },
67
- { id: "transport-delete", name: "DELETE accepted or returns 405", category: "transport", required: false, specRef: "basic/transports#streamable-http", description: "Tests the DELETE endpoint for session termination. Server should accept the request or return 405 Method Not Allowed." },
68
- { id: "transport-batch-reject", name: "Rejects JSON-RPC batch requests", category: "transport", required: true, specRef: "basic/transports#streamable-http", description: "Sends a JSON-RPC batch request (array of messages) and verifies the server rejects it with an error. MCP does not support JSON-RPC batch requests." },
69
- // ── Lifecycle (10 tests) ─────────────────────────────────────────
70
- { id: "lifecycle-init", name: "Initialize handshake", category: "lifecycle", required: true, specRef: "basic/lifecycle#initialization", description: "Tests the initialize handshake by sending an initialize request with client capabilities. The server must return a result with protocolVersion." },
71
- { id: "lifecycle-proto-version", name: "Returns valid protocol version", category: "lifecycle", required: true, specRef: "basic/lifecycle#version-negotiation", description: "Validates that the protocolVersion returned by the server matches the YYYY-MM-DD date format required by the spec." },
72
- { id: "lifecycle-server-info", name: "Includes serverInfo", category: "lifecycle", required: false, specRef: "basic/lifecycle#initialization", description: "Checks that the server includes a serverInfo object with at least a name field in its initialize response. While recommended, this is not strictly required." },
73
- { id: "lifecycle-capabilities", name: "Returns capabilities object", category: "lifecycle", required: true, specRef: "basic/lifecycle#capability-negotiation", description: "Verifies the server returns a capabilities object in its initialize response. An empty object is valid (no optional features declared)." },
74
- { id: "lifecycle-jsonrpc", name: "Response is valid JSON-RPC 2.0", category: "lifecycle", required: true, specRef: "basic", description: 'Validates that the initialize response is a proper JSON-RPC 2.0 message with jsonrpc="2.0", an id field, and either a result or error field.' },
75
- { id: "lifecycle-ping", name: "Responds to ping", category: "lifecycle", required: true, specRef: "basic/utilities#ping", description: "Tests that the server responds to the ping method with an empty result object. This is a required utility method." },
76
- { id: "lifecycle-instructions", name: "Instructions field is valid", category: "lifecycle", required: false, specRef: "basic/lifecycle#initialization", description: "If the server includes an instructions field in the initialize response, validates it is a string. Instructions provide guidance for how the client should interact with the server." },
77
- { id: "lifecycle-id-match", name: "Response ID matches request ID", category: "lifecycle", required: true, specRef: "basic", description: "Verifies that the JSON-RPC response id matches the request id sent by the client. This is a fundamental JSON-RPC 2.0 requirement." },
78
- { id: "lifecycle-logging", name: "logging/setLevel accepted", category: "lifecycle", required: false, specRef: "server/utilities#logging", description: "If the server declares logging capability, tests that logging/setLevel method is accepted with a valid log level." },
79
- { id: "lifecycle-completions", name: "completion/complete accepted", category: "lifecycle", required: false, specRef: "server/utilities#completion", description: "If the server declares completions capability, tests that the completion/complete method is accepted." },
80
- // ── Tools (4 tests) ──────────────────────────────────────────────
81
- { id: "tools-list", name: "tools/list returns valid response", category: "tools", required: false, specRef: "server/tools#listing-tools", description: "Calls tools/list and validates it returns an array of tool definitions. Required if the server declares tools capability." },
82
- { id: "tools-call", name: "tools/call responds correctly", category: "tools", required: false, specRef: "server/tools#calling-tools", description: "Calls the first tool with empty arguments and verifies the response format. Accepts both successful results and InvalidParams errors." },
83
- { id: "tools-pagination", name: "tools/list supports pagination", category: "tools", required: false, specRef: "server/tools#listing-tools", description: "Tests cursor-based pagination on tools/list. Validates nextCursor is a string if present and that fetching the next page returns a valid response." },
84
- { id: "tools-content-types", name: "Tool content items have valid types", category: "tools", required: false, specRef: "server/tools#calling-tools", description: "Validates that content items returned by tools/call have a recognized type field (text, image, audio, resource, resource_link)." },
85
- // ── Resources (5 tests) ──────────────────────────────────────────
86
- { id: "resources-list", name: "resources/list returns valid response", category: "resources", required: false, specRef: "server/resources#listing-resources", description: "Calls resources/list and validates it returns an array. Required if the server declares resources capability." },
87
- { id: "resources-read", name: "resources/read returns content", category: "resources", required: false, specRef: "server/resources#reading-resources", description: "Reads the first resource and validates the response contains a contents array with proper uri and text/blob fields." },
88
- { id: "resources-templates", name: "resources/templates/list returns valid response", category: "resources", required: false, specRef: "server/resources#resource-templates", description: "Tests the resource templates endpoint. Accepts Method not found (-32601) since templates are optional." },
89
- { id: "resources-pagination", name: "resources/list supports pagination", category: "resources", required: false, specRef: "server/resources#listing-resources", description: "Tests cursor-based pagination on resources/list. Validates nextCursor is a string if present and that fetching the next page works." },
90
- { id: "resources-subscribe", name: "Resource subscribe/unsubscribe", category: "resources", required: false, specRef: "server/resources#subscriptions", description: "If the server declares resources.subscribe capability, tests that resources/subscribe and resources/unsubscribe methods are accepted." },
91
- // ── Prompts (3 tests) ────────────────────────────────────────────
92
- { id: "prompts-list", name: "prompts/list returns valid response", category: "prompts", required: false, specRef: "server/prompts#listing-prompts", description: "Calls prompts/list and validates it returns an array. Required if the server declares prompts capability." },
93
- { id: "prompts-get", name: "prompts/get returns valid messages", category: "prompts", required: false, specRef: "server/prompts#getting-a-prompt", description: "Gets the first prompt and validates the response contains a messages array with proper role and content fields." },
94
- { id: "prompts-pagination", name: "prompts/list supports pagination", category: "prompts", required: false, specRef: "server/prompts#listing-prompts", description: "Tests cursor-based pagination on prompts/list. Validates nextCursor is a string if present and that fetching the next page works." },
95
- // ── Error Handling (8 tests) ─────────────────────────────────────
96
- { id: "error-unknown-method", name: "Returns JSON-RPC error for unknown method", category: "errors", required: true, specRef: "basic", description: "Sends an unknown method and verifies the server returns a JSON-RPC error. The spec requires error code -32601 (Method not found)." },
97
- { id: "error-method-code", name: "Uses correct JSON-RPC error code for unknown method", category: "errors", required: false, specRef: "basic", description: "Checks the error code is specifically -32601 (Method not found) for unknown methods, as required by JSON-RPC 2.0." },
98
- { id: "error-invalid-jsonrpc", name: "Handles malformed JSON-RPC", category: "errors", required: true, specRef: "basic", description: "Sends a malformed JSON-RPC message (missing required fields) and verifies the server returns an error or 4xx status." },
99
- { id: "error-invalid-json", name: "Handles invalid JSON body", category: "errors", required: false, specRef: "basic", description: "Sends invalid JSON and verifies the server returns a parse error (-32700) or 4xx status code." },
100
- { id: "error-missing-params", name: "Returns error for tools/call without name", category: "errors", required: false, specRef: "server/tools#error-handling", description: "Calls tools/call with an empty params object (missing required name field) and verifies an error is returned." },
101
- { id: "error-parse-code", name: "Returns -32700 for invalid JSON", category: "errors", required: false, specRef: "basic", description: "Checks that the server returns the specific JSON-RPC error code -32700 (Parse error) when receiving invalid JSON, as required by the JSON-RPC 2.0 specification." },
102
- { id: "error-invalid-request-code", name: "Returns -32600 for invalid request", category: "errors", required: false, specRef: "basic", description: "Checks that the server returns the specific JSON-RPC error code -32600 (Invalid Request) for malformed JSON-RPC messages missing required fields." },
103
- { id: "tools-call-unknown", name: "Returns error for unknown tool name", category: "errors", required: false, specRef: "server/tools#error-handling", description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response." },
104
- // ── Schema Validation (6 tests) ──────────────────────────────────
105
- { id: "tools-schema", name: "All tools have name and inputSchema", category: "schema", required: false, specRef: "server/tools#data-types", description: 'Validates every tool has a valid name (1-128 chars, alphanumeric/underscore/hyphen/dot) and a required inputSchema of type "object".' },
106
- { id: "tools-annotations", name: "Tool annotations are valid", category: "schema", required: false, specRef: "server/tools#annotations", description: "Validates tool annotation fields if present: readOnlyHint, destructiveHint, idempotentHint, openWorldHint should be booleans; title should be a string." },
107
- { id: "tools-title-field", name: "Tools include title field", category: "schema", required: false, specRef: "server/tools#data-types", description: "Checks if tools include the optional title field for human-readable display names. Added in spec version 2025-11-25." },
108
- { id: "tools-output-schema", name: "Tools with outputSchema are valid", category: "schema", required: false, specRef: "server/tools#structured-content", description: 'If tools declare an outputSchema, validates it is a valid JSON Schema object with type "object". Used for structured output validation.' },
109
- { id: "prompts-schema", name: "Prompts have name field", category: "schema", required: false, specRef: "server/prompts#data-types", description: "Validates every prompt has a name and that any arguments array contains items with name fields." },
110
- { id: "resources-schema", name: "Resources have uri and name", category: "schema", required: false, specRef: "server/resources#data-types", description: "Validates every resource has a valid URI (parseable as a URL) and a name field." }
111
- ];
112
-
113
- // src/runner.ts
114
- var _require = createRequire(import.meta.url);
115
- var { version: TOOL_VERSION } = _require("../package.json");
116
- var SPEC_VERSION = "2025-11-25";
117
- var SPEC_BASE = `https://modelcontextprotocol.io/specification/${SPEC_VERSION}`;
118
- var VALID_CONTENT_TYPES = ["text", "image", "audio", "resource", "resource_link"];
119
- function createIdCounter(start = 0) {
120
- let id = start;
121
- return () => ++id;
122
- }
123
- function parseSSEResponse(text) {
124
- const lines = text.split("\n");
125
- let lastJsonRpcResponse = null;
126
- let currentData = [];
127
- function flushEvent() {
128
- if (currentData.length === 0) return;
129
- const data = currentData.join("\n");
130
- currentData = [];
131
- if (!data.trim()) return;
132
- try {
133
- const parsed = JSON.parse(data);
134
- if (parsed.jsonrpc === "2.0" && parsed.id !== void 0) {
135
- lastJsonRpcResponse = parsed;
136
- }
137
- } catch {
138
- }
139
- }
140
- for (const line of lines) {
141
- if (line.startsWith("data:")) {
142
- const content = line.slice(5);
143
- currentData.push(content.startsWith(" ") ? content.slice(1) : content);
144
- } else if (line.trim() === "") {
145
- flushEvent();
146
- }
147
- }
148
- flushEvent();
149
- return lastJsonRpcResponse;
150
- }
151
- async function mcpRequest(backendUrl, method, params, nextId, extraHeaders, timeout) {
152
- const id = nextId();
153
- const body = JSON.stringify({
154
- jsonrpc: "2.0",
155
- id,
156
- method,
157
- params: params || {}
158
- });
159
- const headers = {
160
- "Content-Type": "application/json",
161
- "Accept": "application/json, text/event-stream",
162
- ...extraHeaders
163
- };
164
- const res = await request(backendUrl, {
165
- method: "POST",
166
- headers,
167
- body,
168
- signal: AbortSignal.timeout(timeout)
169
- });
170
- const text = await res.body.text();
171
- const responseHeaders = {};
172
- for (const [k, v] of Object.entries(res.headers)) {
173
- if (typeof v === "string") responseHeaders[k] = v;
174
- }
175
- const contentType = (responseHeaders["content-type"] || "").toLowerCase();
176
- if (contentType.includes("text/event-stream")) {
177
- const parsed = parseSSEResponse(text);
178
- if (parsed) {
179
- return { statusCode: res.statusCode, body: parsed, headers: responseHeaders, requestId: id };
180
- }
181
- try {
182
- return { statusCode: res.statusCode, body: JSON.parse(text), headers: responseHeaders, requestId: id };
183
- } catch {
184
- return { statusCode: res.statusCode, body: { _raw: text }, headers: responseHeaders, requestId: id };
185
- }
186
- }
187
- try {
188
- return { statusCode: res.statusCode, body: JSON.parse(text), headers: responseHeaders, requestId: id };
189
- } catch {
190
- return { statusCode: res.statusCode, body: { _raw: text }, headers: responseHeaders, requestId: id };
191
- }
192
- }
193
- async function mcpNotification(backendUrl, method, params, extraHeaders, timeout) {
194
- const headers = {
195
- "Content-Type": "application/json",
196
- "Accept": "application/json, text/event-stream",
197
- ...extraHeaders
198
- };
199
- const res = await request(backendUrl, {
200
- method: "POST",
201
- headers,
202
- body: JSON.stringify({ jsonrpc: "2.0", method, ...params ? { params } : {} }),
203
- signal: AbortSignal.timeout(timeout)
204
- });
205
- await res.body.text();
206
- const responseHeaders = {};
207
- for (const [k, v] of Object.entries(res.headers)) {
208
- if (typeof v === "string") responseHeaders[k] = v;
209
- }
210
- return { statusCode: res.statusCode, headers: responseHeaders };
211
- }
212
- async function runComplianceSuite(url, options = {}) {
213
- let parsed;
214
- try {
215
- parsed = new URL(url);
216
- if (!["http:", "https:"].includes(parsed.protocol)) {
217
- throw new Error("Only HTTP and HTTPS URLs are supported");
218
- }
219
- } catch (e) {
220
- if (e.message.includes("Only HTTP")) throw e;
221
- throw new Error(`Invalid URL: ${url}`);
222
- }
223
- const backendUrl = url;
224
- const tests = [];
225
- const warnings = [];
226
- const nextId = createIdCounter(1e3);
227
- const timeout = options.timeout || 15e3;
228
- const retries = options.retries || 0;
229
- let sessionId = null;
230
- let negotiatedProtocolVersion = null;
231
- const userHeaders = options.headers || {};
232
- function buildHeaders() {
233
- const h = { ...userHeaders };
234
- if (sessionId) h["mcp-session-id"] = sessionId;
235
- if (negotiatedProtocolVersion) h["mcp-protocol-version"] = negotiatedProtocolVersion;
236
- return h;
237
- }
238
- const rpc = (method, params) => mcpRequest(backendUrl, method, params, nextId, buildHeaders(), timeout);
239
- function shouldRun(id, category) {
240
- if (options.only && options.only.length > 0) {
241
- return options.only.includes(category) || options.only.includes(id);
242
- }
243
- if (options.skip && options.skip.length > 0) {
244
- return !options.skip.includes(category) && !options.skip.includes(id);
245
- }
246
- return true;
247
- }
248
- let serverInfo = {
249
- protocolVersion: null,
250
- name: null,
251
- version: null,
252
- capabilities: {}
253
- };
254
- let toolCount = 0;
255
- let toolNames = [];
256
- let resourceCount = 0;
257
- let resourceNames = [];
258
- let promptCount = 0;
259
- let promptNames = [];
260
- async function test(id, name, category, required, specRef, fn) {
261
- if (!shouldRun(id, category)) return;
262
- const start = Date.now();
263
- let lastResult = { passed: false, details: "" };
264
- for (let attempt = 0; attempt <= retries; attempt++) {
265
- try {
266
- lastResult = await fn();
267
- if (lastResult.passed) break;
268
- if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
269
- } catch (err) {
270
- lastResult = { passed: false, details: `Error: ${err.message}` };
271
- if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
272
- }
273
- }
274
- tests.push({
275
- id,
276
- name,
277
- category,
278
- required,
279
- passed: lastResult.passed,
280
- details: lastResult.details,
281
- durationMs: Date.now() - start,
282
- specRef: `${SPEC_BASE}/${specRef}`
283
- });
284
- options.onProgress?.(id, lastResult.passed, lastResult.details);
285
- }
286
- await test("transport-post", "HTTP POST accepted", "transport", true, "basic/transports#streamable-http", async () => {
287
- const res = await request(backendUrl, {
288
- method: "POST",
289
- headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...userHeaders },
290
- body: JSON.stringify({ jsonrpc: "2.0", id: 99901, method: "ping" }),
291
- signal: AbortSignal.timeout(timeout)
292
- });
293
- await res.body.text();
294
- const passed = res.statusCode >= 200 && res.statusCode < 300;
295
- const note = res.statusCode === 401 || res.statusCode === 403 ? " (auth required)" : "";
296
- return { passed, details: `HTTP ${res.statusCode}${note}` };
297
- });
298
- await test("transport-content-type", "Responds with JSON or SSE", "transport", true, "basic/transports#streamable-http", async () => {
299
- const res = await request(backendUrl, {
300
- method: "POST",
301
- headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...userHeaders },
302
- body: JSON.stringify({ jsonrpc: "2.0", id: 99902, method: "ping" }),
303
- signal: AbortSignal.timeout(timeout)
304
- });
305
- await res.body.text();
306
- const rawCt = res.headers["content-type"];
307
- const ct = (Array.isArray(rawCt) ? rawCt[0] : rawCt || "").toLowerCase();
308
- const valid = ct.includes("application/json") || ct.includes("text/event-stream");
309
- return { passed: valid, details: `Content-Type: ${ct}` };
310
- });
311
- await test("transport-get", "GET returns SSE stream or 405", "transport", false, "basic/transports#streamable-http", async () => {
312
- const res = await request(backendUrl, {
313
- method: "GET",
314
- headers: { "Accept": "text/event-stream", ...userHeaders },
315
- signal: AbortSignal.timeout(timeout)
316
- });
317
- await res.body.text();
318
- const ct = (res.headers["content-type"] || "").toLowerCase();
319
- if (res.statusCode === 405) {
320
- return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
321
- }
322
- if (ct.includes("text/event-stream")) {
323
- return { passed: true, details: "Returns text/event-stream for SSE" };
324
- }
325
- if (res.statusCode >= 200 && res.statusCode < 300) {
326
- return { passed: true, details: `HTTP ${res.statusCode} (accepted)` };
327
- }
328
- return { passed: false, details: `HTTP ${res.statusCode}, Content-Type: ${ct}` };
329
- });
330
- await test("transport-delete", "DELETE accepted or returns 405", "transport", false, "basic/transports#streamable-http", async () => {
331
- const res = await request(backendUrl, {
332
- method: "DELETE",
333
- headers: { ...userHeaders },
334
- signal: AbortSignal.timeout(timeout)
335
- });
336
- await res.body.text();
337
- if (res.statusCode === 405) {
338
- return { passed: true, details: "HTTP 405 Method Not Allowed (acceptable)" };
339
- }
340
- if (res.statusCode >= 200 && res.statusCode < 300) {
341
- return { passed: true, details: `HTTP ${res.statusCode} (session termination supported)` };
342
- }
343
- if (res.statusCode === 400 || res.statusCode === 404) {
344
- return { passed: true, details: `HTTP ${res.statusCode} (no active session, acceptable)` };
345
- }
346
- return { passed: false, details: `HTTP ${res.statusCode}` };
347
- });
348
- await test("transport-batch-reject", "Rejects JSON-RPC batch requests", "transport", true, "basic/transports#streamable-http", async () => {
349
- const res = await request(backendUrl, {
350
- method: "POST",
351
- headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...userHeaders },
352
- body: JSON.stringify([
353
- { jsonrpc: "2.0", id: 99903, method: "ping" },
354
- { jsonrpc: "2.0", id: 99904, method: "ping" }
355
- ]),
356
- signal: AbortSignal.timeout(timeout)
357
- });
358
- const text = await res.body.text();
359
- if (res.statusCode >= 400 && res.statusCode < 500) {
360
- return { passed: true, details: `HTTP ${res.statusCode} (batch rejected)` };
361
- }
362
- try {
363
- const body = JSON.parse(text);
364
- if (body?.error) {
365
- return { passed: true, details: `JSON-RPC error: ${body.error.code} \u2014 ${body.error.message}` };
366
- }
367
- if (Array.isArray(body)) {
368
- return { passed: false, details: "Server processed batch request (MCP forbids batch)" };
369
- }
370
- } catch {
371
- }
372
- return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error or 4xx for batch request` };
373
- });
374
- let initRes = null;
375
- try {
376
- initRes = await rpc("initialize", {
377
- protocolVersion: SPEC_VERSION,
378
- capabilities: { roots: { listChanged: true }, sampling: {} },
379
- clientInfo: { name: "mcp-compliance", version: TOOL_VERSION }
380
- });
381
- const result = initRes?.body?.result;
382
- if (result) {
383
- serverInfo.protocolVersion = result.protocolVersion || null;
384
- serverInfo.name = result.serverInfo?.name || null;
385
- serverInfo.version = result.serverInfo?.version || null;
386
- serverInfo.capabilities = result.capabilities || {};
387
- const sid = initRes.headers["mcp-session-id"];
388
- if (sid) sessionId = sid;
389
- if (result.protocolVersion) negotiatedProtocolVersion = result.protocolVersion;
390
- }
391
- } catch (err) {
392
- }
393
- try {
394
- await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders(), timeout);
395
- } catch {
396
- }
397
- await test("lifecycle-init", "Initialize handshake", "lifecycle", true, "basic/lifecycle#initialization", async () => {
398
- if (!initRes) return { passed: false, details: "Initialize request failed" };
399
- const result = initRes.body?.result;
400
- if (!result) return { passed: false, details: "No result in response" };
401
- return { passed: !!result.protocolVersion, details: `Protocol: ${result.protocolVersion || "missing"}` };
402
- });
403
- await test("lifecycle-proto-version", "Returns valid protocol version", "lifecycle", true, "basic/lifecycle#version-negotiation", async () => {
404
- const version = initRes?.body?.result?.protocolVersion;
405
- if (!version) return { passed: false, details: "No protocolVersion" };
406
- const valid = /^\d{4}-\d{2}-\d{2}$/.test(version);
407
- if (valid && version !== SPEC_VERSION) {
408
- warnings.push(`Server negotiated protocol version ${version} (latest is ${SPEC_VERSION})`);
409
- }
410
- return { passed: valid, details: `Version: ${version}` };
411
- });
412
- await test("lifecycle-server-info", "Includes serverInfo", "lifecycle", false, "basic/lifecycle#initialization", async () => {
413
- const info = initRes?.body?.result?.serverInfo;
414
- return { passed: !!info?.name, details: info ? `${info.name} v${info.version || "?"}` : "Missing serverInfo" };
415
- });
416
- await test("lifecycle-capabilities", "Returns capabilities object", "lifecycle", true, "basic/lifecycle#capability-negotiation", async () => {
417
- const caps = initRes?.body?.result?.capabilities;
418
- if (!caps || typeof caps !== "object") return { passed: false, details: "No capabilities object in response" };
419
- const declared = Object.keys(caps).filter((k) => caps[k] !== void 0);
420
- return { passed: true, details: declared.length > 0 ? `Capabilities: ${declared.join(", ")}` : "Empty capabilities (valid)" };
421
- });
422
- await test("lifecycle-jsonrpc", "Response is valid JSON-RPC 2.0", "lifecycle", true, "basic", async () => {
423
- const body = initRes?.body;
424
- const valid = body?.jsonrpc === "2.0" && body?.id !== void 0 && (body?.result !== void 0 || body?.error !== void 0);
425
- return { passed: valid, details: valid ? "Valid JSON-RPC 2.0 response" : `Missing fields: jsonrpc=${body?.jsonrpc}, id=${body?.id}` };
426
- });
427
- await test("lifecycle-ping", "Responds to ping", "lifecycle", true, "basic/utilities#ping", async () => {
428
- const res = await rpc("ping");
429
- const body = res.body;
430
- if (body?.error) return { passed: false, details: `Error: ${body.error.message}` };
431
- if (body?.result !== void 0) return { passed: true, details: "Ping responded successfully" };
432
- return { passed: false, details: "No result in ping response" };
433
- });
434
- await test("lifecycle-instructions", "Instructions field is valid", "lifecycle", false, "basic/lifecycle#initialization", async () => {
435
- const result = initRes?.body?.result;
436
- if (!result) return { passed: false, details: "No init result" };
437
- if (result.instructions === void 0) {
438
- return { passed: true, details: "No instructions field (optional)" };
439
- }
440
- if (typeof result.instructions === "string") {
441
- const preview = result.instructions.length > 80 ? result.instructions.slice(0, 80) + "..." : result.instructions;
442
- return { passed: true, details: `Instructions: "${preview}"` };
443
- }
444
- return { passed: false, details: `instructions should be a string, got ${typeof result.instructions}` };
445
- });
446
- await test("lifecycle-id-match", "Response ID matches request ID", "lifecycle", true, "basic", async () => {
447
- const res = await rpc("ping");
448
- const body = res.body;
449
- if (body?.id === void 0) return { passed: false, details: "No id in response" };
450
- const match = body.id === res.requestId;
451
- return { passed: match, details: match ? `Request id=${res.requestId}, response id=${body.id} (match)` : `Request id=${res.requestId}, response id=${body.id} (MISMATCH)` };
452
- });
453
- const hasLogging = !!serverInfo.capabilities.logging;
454
- await test("lifecycle-logging", "logging/setLevel accepted", "lifecycle", hasLogging, "server/utilities#logging", async () => {
455
- if (!hasLogging) return { passed: true, details: "Server does not declare logging capability (skipped)" };
456
- const res = await rpc("logging/setLevel", { level: "info" });
457
- if (res.body?.error) {
458
- return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
459
- }
460
- return { passed: true, details: "logging/setLevel accepted" };
461
- });
462
- const hasCompletions = !!serverInfo.capabilities.completions;
463
- await test("lifecycle-completions", "completion/complete accepted", "lifecycle", hasCompletions, "server/utilities#completion", async () => {
464
- if (!hasCompletions) return { passed: true, details: "Server does not declare completions capability (skipped)" };
465
- const res = await rpc("completion/complete", {
466
- ref: { type: "ref/prompt", name: "__test__" },
467
- argument: { name: "test", value: "" }
468
- });
469
- if (res.body?.error) {
470
- if (res.body.error.code === -32602) {
471
- return { passed: true, details: "InvalidParams for test ref (acceptable)" };
472
- }
473
- return { passed: false, details: `Error: ${res.body.error.code} \u2014 ${res.body.error.message}` };
474
- }
475
- const values = res.body?.result?.completion?.values;
476
- if (Array.isArray(values)) {
477
- return { passed: true, details: `Returned ${values.length} completion(s)` };
478
- }
479
- return { passed: true, details: "completion/complete accepted" };
480
- });
481
- await test("transport-notification-202", "Notification returns 202 Accepted", "transport", false, "basic/transports#streamable-http", async () => {
482
- const res = await request(backendUrl, {
483
- method: "POST",
484
- headers: {
485
- "Content-Type": "application/json",
486
- "Accept": "application/json, text/event-stream",
487
- ...buildHeaders()
488
- },
489
- body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/cancelled", params: { requestId: "nonexistent", reason: "compliance test" } }),
490
- signal: AbortSignal.timeout(timeout)
491
- });
492
- await res.body.text();
493
- if (res.statusCode === 202) {
494
- return { passed: true, details: "HTTP 202 Accepted (correct)" };
495
- }
496
- if (res.statusCode >= 200 && res.statusCode < 300) {
497
- return { passed: true, details: `HTTP ${res.statusCode} (accepted, but 202 is preferred)` };
498
- }
499
- return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected 202 Accepted for notifications` };
500
- });
501
- await test("transport-session-id", "Enforces MCP-Session-Id after init", "transport", false, "basic/transports#streamable-http", async () => {
502
- if (!sessionId) {
503
- warnings.push("Server did not issue MCP-Session-Id header");
504
- return { passed: true, details: "Server did not issue session ID (test not applicable)" };
505
- }
506
- const headersWithout = { ...userHeaders };
507
- if (negotiatedProtocolVersion) headersWithout["mcp-protocol-version"] = negotiatedProtocolVersion;
508
- const res = await mcpRequest(backendUrl, "ping", void 0, createIdCounter(99910), headersWithout, timeout);
509
- if (res.statusCode === 400) {
510
- return { passed: true, details: "HTTP 400 for missing session ID (correct)" };
511
- }
512
- if (res.statusCode >= 200 && res.statusCode < 300) {
513
- return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should return 400 when session ID is missing` };
514
- }
515
- return { passed: false, details: `HTTP ${res.statusCode}` };
516
- });
517
- const hasTools = !!serverInfo.capabilities.tools;
518
- let cachedToolsList = null;
519
- await test("tools-list", "tools/list returns valid response", "tools", hasTools, "server/tools#listing-tools", async () => {
520
- const res = await rpc("tools/list");
521
- const tools = res.body?.result?.tools;
522
- if (!Array.isArray(tools)) return { passed: false, details: "No tools array in result" };
523
- cachedToolsList = tools;
524
- toolCount = tools.length;
525
- toolNames = tools.map((t) => t.name).filter(Boolean);
526
- return { passed: true, details: `${toolCount} tool(s): ${toolNames.slice(0, 5).join(", ")}${toolCount > 5 ? "..." : ""}` };
527
- });
528
- await test("tools-schema", "All tools have name and inputSchema", "schema", hasTools, "server/tools#data-types", async () => {
529
- const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
530
- const issues = [];
531
- for (const tool of tools) {
532
- if (!tool.name) {
533
- issues.push("Tool missing name");
534
- continue;
535
- }
536
- if (tool.name.length > 128 || !/^[A-Za-z0-9_.\-]+$/.test(tool.name)) {
537
- issues.push(`${tool.name}: name format invalid`);
538
- }
539
- if (!tool.description) warnings.push(`Tool "${tool.name}" missing description`);
540
- if (!tool.inputSchema) {
541
- issues.push(`${tool.name}: missing inputSchema (required)`);
542
- } else if (typeof tool.inputSchema !== "object" || tool.inputSchema === null) {
543
- issues.push(`${tool.name}: inputSchema must be a valid JSON Schema object`);
544
- } else if (tool.inputSchema.type !== "object") {
545
- issues.push(`${tool.name}: inputSchema.type must be "object" (got "${tool.inputSchema.type || "undefined"}")`);
546
- }
547
- }
548
- const detail = issues.length === 0 ? "All tools have valid schemas" : issues.join("; ");
549
- return { passed: issues.length === 0, details: detail };
550
- });
551
- await test("tools-annotations", "Tool annotations are valid", "schema", false, "server/tools#annotations", async () => {
552
- const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
553
- if (tools.length === 0) return { passed: true, details: "No tools to validate" };
554
- const issues = [];
555
- let annotatedCount = 0;
556
- for (const tool of tools) {
557
- const ann = tool.annotations;
558
- if (!ann) continue;
559
- annotatedCount++;
560
- if (typeof ann !== "object" || ann === null) {
561
- issues.push(`${tool.name}: annotations must be an object`);
562
- continue;
563
- }
564
- const boolFields = ["readOnlyHint", "destructiveHint", "idempotentHint", "openWorldHint"];
565
- for (const field of boolFields) {
566
- if (ann[field] !== void 0 && typeof ann[field] !== "boolean") {
567
- issues.push(`${tool.name}: annotations.${field} should be boolean, got ${typeof ann[field]}`);
568
- }
569
- }
570
- if (ann.title !== void 0 && typeof ann.title !== "string") {
571
- issues.push(`${tool.name}: annotations.title should be string`);
572
- }
573
- }
574
- if (issues.length > 0) return { passed: false, details: issues.join("; ") };
575
- return { passed: true, details: annotatedCount > 0 ? `${annotatedCount} tool(s) with valid annotations` : "No tools have annotations (optional)" };
576
- });
577
- await test("tools-title-field", "Tools include title field", "schema", false, "server/tools#data-types", async () => {
578
- const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
579
- if (tools.length === 0) return { passed: true, details: "No tools to validate" };
580
- const withTitle = tools.filter((t) => typeof t.title === "string");
581
- const issues = [];
582
- for (const tool of tools) {
583
- if (tool.title !== void 0 && typeof tool.title !== "string") {
584
- issues.push(`${tool.name}: title should be a string`);
585
- }
586
- }
587
- if (issues.length > 0) return { passed: false, details: issues.join("; ") };
588
- if (withTitle.length === 0) {
589
- return { passed: true, details: "No tools have title field (optional, added in 2025-11-25)" };
590
- }
591
- return { passed: true, details: `${withTitle.length}/${tools.length} tool(s) have title field` };
592
- });
593
- await test("tools-output-schema", "Tools with outputSchema are valid", "schema", false, "server/tools#structured-content", async () => {
594
- const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
595
- if (tools.length === 0) return { passed: true, details: "No tools to validate" };
596
- const issues = [];
597
- let withSchema = 0;
598
- for (const tool of tools) {
599
- if (tool.outputSchema === void 0) continue;
600
- withSchema++;
601
- if (typeof tool.outputSchema !== "object" || tool.outputSchema === null) {
602
- issues.push(`${tool.name}: outputSchema must be a JSON Schema object`);
603
- } else if (tool.outputSchema.type !== "object") {
604
- issues.push(`${tool.name}: outputSchema.type must be "object" (got "${tool.outputSchema.type || "undefined"}")`);
605
- }
606
- }
607
- if (issues.length > 0) return { passed: false, details: issues.join("; ") };
608
- return { passed: true, details: withSchema > 0 ? `${withSchema} tool(s) with valid outputSchema` : "No tools have outputSchema (optional)" };
609
- });
610
- if (toolNames.length > 0) {
611
- await test("tools-call", "tools/call responds correctly", "tools", false, "server/tools#calling-tools", async () => {
612
- const res = await rpc("tools/call", { name: toolNames[0], arguments: {} });
613
- const result = res.body?.result;
614
- const error = res.body?.error;
615
- if (error) {
616
- const code = error.code;
617
- if (code === -32602 || code === -32600) {
618
- return { passed: true, details: `Invalid params error (acceptable): code ${code}` };
619
- }
620
- return { passed: true, details: `Protocol error: code ${code} \u2014 ${error.message}` };
621
- }
622
- if (result?.content && Array.isArray(result.content)) {
623
- const badItems = result.content.filter((c) => !c.type);
624
- if (badItems.length > 0) return { passed: false, details: `${badItems.length} content item(s) missing 'type' field` };
625
- return { passed: true, details: `Returned ${result.content.length} content item(s)` };
626
- }
627
- if (result?.isError && result?.content && Array.isArray(result.content)) {
628
- return { passed: true, details: "Tool returned execution error with content (valid)" };
629
- }
630
- return { passed: false, details: "Response missing content array" };
631
- });
632
- await test("tools-content-types", "Tool content items have valid types", "tools", false, "server/tools#calling-tools", async () => {
633
- const res = await rpc("tools/call", { name: toolNames[0], arguments: {} });
634
- const result = res.body?.result;
635
- const error = res.body?.error;
636
- if (error) {
637
- return { passed: true, details: `Tool returned error (content types not applicable): code ${error.code}` };
638
- }
639
- const content = result?.content;
640
- if (!Array.isArray(content) || content.length === 0) {
641
- return { passed: true, details: "No content items to validate" };
642
- }
643
- const issues = [];
644
- const types = /* @__PURE__ */ new Set();
645
- for (const item of content) {
646
- if (!item.type) {
647
- issues.push("Content item missing type field");
648
- } else if (!VALID_CONTENT_TYPES.includes(item.type)) {
649
- issues.push(`Unknown content type: "${item.type}"`);
650
- } else {
651
- types.add(item.type);
652
- }
653
- }
654
- if (issues.length > 0) return { passed: false, details: issues.join("; ") };
655
- return { passed: true, details: `Content types: ${[...types].join(", ")}` };
656
- });
657
- }
658
- if (hasTools) {
659
- await test("tools-pagination", "tools/list supports pagination", "tools", false, "server/tools#listing-tools", async () => {
660
- const res = await rpc("tools/list");
661
- const result = res.body?.result;
662
- if (!result) return { passed: false, details: "No result from tools/list" };
663
- if (!Array.isArray(result.tools)) return { passed: false, details: "No tools array" };
664
- if (result.nextCursor !== void 0) {
665
- if (typeof result.nextCursor !== "string") {
666
- return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
667
- }
668
- const nextRes = await rpc("tools/list", { cursor: result.nextCursor });
669
- const nextResult = nextRes.body?.result;
670
- if (!nextResult || !Array.isArray(nextResult.tools)) {
671
- return { passed: false, details: "Next page failed to return tools array" };
672
- }
673
- return { passed: true, details: `Pagination works: page 1 had ${result.tools.length} tools, page 2 had ${nextResult.tools.length} tools` };
674
- }
675
- return { passed: true, details: `${result.tools.length} tool(s), no nextCursor (single page)` };
676
- });
677
- await test("tools-call-unknown", "Returns error for unknown tool name", "errors", false, "server/tools#error-handling", async () => {
678
- const res = await rpc("tools/call", { name: "__nonexistent_tool_compliance_test__", arguments: {} });
679
- const error = res.body?.error;
680
- const isError = res.body?.result?.isError;
681
- if (error) return { passed: true, details: `Error code: ${error.code} \u2014 ${error.message}` };
682
- if (isError) return { passed: true, details: "Tool execution error with isError=true (valid)" };
683
- return { passed: false, details: "No error returned for nonexistent tool" };
684
- });
685
- }
686
- const hasResources = !!serverInfo.capabilities.resources;
687
- const hasSubscribe = !!serverInfo.capabilities.resources?.subscribe;
688
- if (hasResources) {
689
- let cachedResourcesList = null;
690
- await test("resources-list", "resources/list returns valid response", "resources", true, "server/resources#listing-resources", async () => {
691
- const res = await rpc("resources/list");
692
- const resources = res.body?.result?.resources;
693
- if (!Array.isArray(resources)) return { passed: false, details: "No resources array" };
694
- cachedResourcesList = resources;
695
- resourceCount = resources.length;
696
- resourceNames = resources.map((r) => r.name).filter(Boolean);
697
- return { passed: true, details: `${resourceCount} resource(s)` };
698
- });
699
- await test("resources-schema", "Resources have uri and name", "schema", true, "server/resources#data-types", async () => {
700
- const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
701
- const issues = [];
702
- for (const r of resources) {
703
- if (!r.uri) issues.push("Resource missing uri");
704
- else {
705
- try {
706
- new URL(r.uri);
707
- } catch {
708
- issues.push(`${r.uri}: invalid URI format`);
709
- }
710
- }
711
- if (!r.name) issues.push(`${r.uri || "?"}: missing name`);
712
- if (!r.description) warnings.push(`Resource "${r.name || r.uri}" missing description`);
713
- if (!r.mimeType) warnings.push(`Resource "${r.name || r.uri}" missing mimeType`);
714
- }
715
- return { passed: issues.length === 0, details: issues.length === 0 ? "All resources valid" : issues.join("; ") };
716
- });
717
- if (resourceCount > 0) {
718
- await test("resources-read", "resources/read returns content", "resources", false, "server/resources#reading-resources", async () => {
719
- const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
720
- const firstUri = resources[0]?.uri;
721
- if (!firstUri) return { passed: false, details: "No resource URI to test" };
722
- const readRes = await rpc("resources/read", { uri: firstUri });
723
- const contents = readRes.body?.result?.contents;
724
- if (!Array.isArray(contents)) return { passed: false, details: "No contents array" };
725
- const issues = [];
726
- for (const c of contents) {
727
- if (!c.uri) issues.push("Content item missing uri");
728
- if (!c.text && !c.blob) issues.push(`Content item for ${c.uri || "?"} missing both text and blob`);
729
- }
730
- if (issues.length > 0) return { passed: false, details: issues.join("; ") };
731
- return { passed: true, details: `Read ${contents.length} content item(s) from ${firstUri}` };
732
- });
733
- }
734
- await test("resources-templates", "resources/templates/list returns valid response", "resources", false, "server/resources#resource-templates", async () => {
735
- const res = await rpc("resources/templates/list");
736
- const error = res.body?.error;
737
- if (error) {
738
- if (error.code === -32601) return { passed: true, details: "Method not supported (acceptable)" };
739
- return { passed: false, details: `Error: ${error.message}` };
740
- }
741
- const templates = res.body?.result?.resourceTemplates;
742
- if (!Array.isArray(templates)) return { passed: false, details: "No resourceTemplates array" };
743
- const issues = [];
744
- for (const t of templates) {
745
- if (!t.uriTemplate) issues.push("Template missing uriTemplate");
746
- if (!t.name) issues.push(`${t.uriTemplate || "?"}: missing name`);
747
- }
748
- if (issues.length > 0) return { passed: false, details: issues.join("; ") };
749
- return { passed: true, details: `${templates.length} resource template(s)` };
750
- });
751
- await test("resources-pagination", "resources/list supports pagination", "resources", false, "server/resources#listing-resources", async () => {
752
- const res = await rpc("resources/list");
753
- const result = res.body?.result;
754
- if (!result) return { passed: false, details: "No result from resources/list" };
755
- if (!Array.isArray(result.resources)) return { passed: false, details: "No resources array" };
756
- if (result.nextCursor !== void 0) {
757
- if (typeof result.nextCursor !== "string") {
758
- return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
759
- }
760
- const nextRes = await rpc("resources/list", { cursor: result.nextCursor });
761
- const nextResult = nextRes.body?.result;
762
- if (!nextResult || !Array.isArray(nextResult.resources)) {
763
- return { passed: false, details: "Next page failed to return resources array" };
764
- }
765
- return { passed: true, details: `Pagination works: page 1 had ${result.resources.length}, page 2 had ${nextResult.resources.length}` };
766
- }
767
- return { passed: true, details: `${result.resources.length} resource(s), no nextCursor (single page)` };
768
- });
769
- if (hasSubscribe && resourceCount > 0) {
770
- await test("resources-subscribe", "Resource subscribe/unsubscribe", "resources", true, "server/resources#subscriptions", async () => {
771
- const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
772
- const firstUri = resources[0]?.uri;
773
- if (!firstUri) return { passed: false, details: "No resource URI for subscribe test" };
774
- const subRes = await rpc("resources/subscribe", { uri: firstUri });
775
- if (subRes.body?.error) {
776
- return { passed: false, details: `Subscribe error: ${subRes.body.error.code} \u2014 ${subRes.body.error.message}` };
777
- }
778
- const unsubRes = await rpc("resources/unsubscribe", { uri: firstUri });
779
- if (unsubRes.body?.error) {
780
- return { passed: false, details: `Unsubscribe error: ${unsubRes.body.error.code} \u2014 ${unsubRes.body.error.message}` };
781
- }
782
- return { passed: true, details: `Subscribe/unsubscribe for ${firstUri} succeeded` };
783
- });
784
- }
785
- }
786
- const hasPrompts = !!serverInfo.capabilities.prompts;
787
- if (hasPrompts) {
788
- let cachedPromptsList = null;
789
- await test("prompts-list", "prompts/list returns valid response", "prompts", true, "server/prompts#listing-prompts", async () => {
790
- const res = await rpc("prompts/list");
791
- const prompts = res.body?.result?.prompts;
792
- if (!Array.isArray(prompts)) return { passed: false, details: "No prompts array" };
793
- cachedPromptsList = prompts;
794
- promptCount = prompts.length;
795
- promptNames = prompts.map((p) => p.name).filter(Boolean);
796
- return { passed: true, details: `${promptCount} prompt(s): ${promptNames.slice(0, 5).join(", ")}${promptCount > 5 ? "..." : ""}` };
797
- });
798
- await test("prompts-schema", "Prompts have name field", "schema", true, "server/prompts#data-types", async () => {
799
- const prompts = cachedPromptsList ?? (await rpc("prompts/list")).body?.result?.prompts ?? [];
800
- const issues = [];
801
- for (const p of prompts) {
802
- if (!p.name) issues.push("Prompt missing name");
803
- if (!p.description) warnings.push(`Prompt "${p.name || "?"}" missing description`);
804
- if (p.arguments && !Array.isArray(p.arguments)) issues.push(`${p.name || "?"}: arguments must be an array`);
805
- if (Array.isArray(p.arguments)) {
806
- for (const arg of p.arguments) {
807
- if (!arg.name) issues.push(`${p.name}: argument missing name`);
808
- }
809
- }
810
- }
811
- return { passed: issues.length === 0, details: issues.length === 0 ? "All prompts valid" : issues.join("; ") };
812
- });
813
- if (promptNames.length > 0) {
814
- await test("prompts-get", "prompts/get returns valid messages", "prompts", false, "server/prompts#getting-a-prompt", async () => {
815
- const res = await rpc("prompts/get", { name: promptNames[0] });
816
- const error = res.body?.error;
817
- if (error) return { passed: true, details: `Error (may need arguments): code ${error.code}` };
818
- const messages = res.body?.result?.messages;
819
- if (!Array.isArray(messages)) return { passed: false, details: "No messages array in result" };
820
- const issues = [];
821
- for (const msg of messages) {
822
- if (!msg.role || !["user", "assistant"].includes(msg.role)) issues.push(`Invalid role: ${msg.role}`);
823
- if (!msg.content) issues.push("Message missing content");
824
- }
825
- if (issues.length > 0) return { passed: false, details: issues.join("; ") };
826
- return { passed: true, details: `${messages.length} message(s) from ${promptNames[0]}` };
827
- });
828
- }
829
- await test("prompts-pagination", "prompts/list supports pagination", "prompts", false, "server/prompts#listing-prompts", async () => {
830
- const res = await rpc("prompts/list");
831
- const result = res.body?.result;
832
- if (!result) return { passed: false, details: "No result from prompts/list" };
833
- if (!Array.isArray(result.prompts)) return { passed: false, details: "No prompts array" };
834
- if (result.nextCursor !== void 0) {
835
- if (typeof result.nextCursor !== "string") {
836
- return { passed: false, details: `nextCursor should be string, got ${typeof result.nextCursor}` };
837
- }
838
- const nextRes = await rpc("prompts/list", { cursor: result.nextCursor });
839
- const nextResult = nextRes.body?.result;
840
- if (!nextResult || !Array.isArray(nextResult.prompts)) {
841
- return { passed: false, details: "Next page failed to return prompts array" };
842
- }
843
- return { passed: true, details: `Pagination works: page 1 had ${result.prompts.length}, page 2 had ${nextResult.prompts.length}` };
844
- }
845
- return { passed: true, details: `${result.prompts.length} prompt(s), no nextCursor (single page)` };
846
- });
847
- }
848
- await test("error-unknown-method", "Returns JSON-RPC error for unknown method", "errors", true, "basic", async () => {
849
- const res = await rpc("nonexistent/method");
850
- const error = res.body?.error;
851
- if (!error) return { passed: false, details: "No JSON-RPC error returned for unknown method" };
852
- const correctCode = error.code === -32601;
853
- return {
854
- passed: true,
855
- details: `Error code: ${error.code}${correctCode ? " (correct: Method not found)" : " (expected -32601)"} \u2014 ${error.message}`
856
- };
857
- });
858
- await test("error-method-code", "Uses correct JSON-RPC error code for unknown method", "errors", false, "basic", async () => {
859
- const res = await rpc("nonexistent/method");
860
- const error = res.body?.error;
861
- if (!error) return { passed: false, details: "No error returned" };
862
- return { passed: error.code === -32601, details: `Expected -32601, got ${error.code}` };
863
- });
864
- await test("error-invalid-jsonrpc", "Handles malformed JSON-RPC", "errors", true, "basic", async () => {
865
- const res = await request(backendUrl, {
866
- method: "POST",
867
- headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...buildHeaders() },
868
- body: JSON.stringify({ not: "a valid jsonrpc message" }),
869
- signal: AbortSignal.timeout(timeout)
870
- });
871
- const text = await res.body.text();
872
- try {
873
- const body = JSON.parse(text);
874
- if (body?.error) {
875
- const correctCode = body.error.code === -32600;
876
- return { passed: true, details: `Error code: ${body.error.code}${correctCode ? " (correct: Invalid Request)" : ""} \u2014 ${body.error.message}` };
877
- }
878
- } catch {
879
- }
880
- if (res.statusCode >= 400 && res.statusCode < 500) return { passed: true, details: `HTTP ${res.statusCode} (acceptable)` };
881
- return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected JSON-RPC error or 4xx status` };
882
- });
883
- await test("error-invalid-json", "Handles invalid JSON body", "errors", false, "basic", async () => {
884
- const res = await request(backendUrl, {
885
- method: "POST",
886
- headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...buildHeaders() },
887
- body: "{this is not valid json!!!",
888
- signal: AbortSignal.timeout(timeout)
889
- });
890
- const text = await res.body.text();
891
- try {
892
- const body = JSON.parse(text);
893
- if (body?.error) return { passed: true, details: `Error code: ${body.error.code} \u2014 ${body.error.message}` };
894
- } catch {
895
- }
896
- if (res.statusCode >= 400 && res.statusCode < 500) return { passed: true, details: `HTTP ${res.statusCode} (acceptable)` };
897
- return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected parse error or 4xx status` };
898
- });
899
- await test("error-missing-params", "Returns error for tools/call without name", "errors", false, "server/tools#error-handling", async () => {
900
- const res = await rpc("tools/call", {});
901
- const error = res.body?.error;
902
- const isError = res.body?.result?.isError;
903
- if (error) {
904
- const correctCode = error.code === -32602;
905
- return { passed: true, details: `Error code: ${error.code}${correctCode ? " (correct: Invalid params)" : ""} \u2014 ${error.message}` };
906
- }
907
- if (isError) return { passed: true, details: "Tool execution error (valid)" };
908
- return { passed: false, details: "No error for tools/call without name" };
909
- });
910
- await test("error-parse-code", "Returns -32700 for invalid JSON", "errors", false, "basic", async () => {
911
- const res = await request(backendUrl, {
912
- method: "POST",
913
- headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...buildHeaders() },
914
- body: "<<<not json>>>",
915
- signal: AbortSignal.timeout(timeout)
916
- });
917
- const text = await res.body.text();
918
- try {
919
- const body = JSON.parse(text);
920
- if (body?.error?.code === -32700) {
921
- return { passed: true, details: `Error code: -32700 (Parse error) \u2014 ${body.error.message}` };
922
- }
923
- if (body?.error) {
924
- return { passed: false, details: `Expected -32700, got ${body.error.code} \u2014 ${body.error.message}` };
925
- }
926
- } catch {
927
- }
928
- if (res.statusCode >= 400 && res.statusCode < 500) {
929
- return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should return JSON-RPC error code -32700` };
930
- }
931
- return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error code -32700` };
932
- });
933
- await test("error-invalid-request-code", "Returns -32600 for invalid request", "errors", false, "basic", async () => {
934
- const res = await request(backendUrl, {
935
- method: "POST",
936
- headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...buildHeaders() },
937
- body: JSON.stringify({ jsonrpc: "2.0", id: 99999 }),
938
- signal: AbortSignal.timeout(timeout)
939
- });
940
- const text = await res.body.text();
941
- try {
942
- const body = JSON.parse(text);
943
- if (body?.error?.code === -32600) {
944
- return { passed: true, details: `Error code: -32600 (Invalid Request) \u2014 ${body.error.message}` };
945
- }
946
- if (body?.error) {
947
- return { passed: false, details: `Expected -32600, got ${body.error.code} \u2014 ${body.error.message}` };
948
- }
949
- } catch {
950
- }
951
- if (res.statusCode >= 400 && res.statusCode < 500) {
952
- return { passed: false, details: `HTTP ${res.statusCode} \u2014 server should return JSON-RPC error code -32600` };
953
- }
954
- return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected error code -32600` };
955
- });
956
- const { score, grade, overall, summary, categories } = computeScore(tests);
957
- const badge = generateBadge(url);
958
- return {
959
- specVersion: SPEC_VERSION,
960
- toolVersion: TOOL_VERSION,
961
- url,
962
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
963
- score,
964
- grade,
965
- overall,
966
- summary,
967
- categories,
968
- tests,
969
- warnings,
970
- serverInfo,
971
- toolCount,
972
- toolNames,
973
- resourceCount,
974
- resourceNames,
975
- promptCount,
976
- promptNames,
977
- badge
978
- };
979
- }
980
-
981
- export {
982
- computeGrade,
983
- computeScore,
984
- generateBadge,
985
- TEST_DEFINITIONS,
986
- runComplianceSuite
987
- };