@yawlabs/mcp-compliance 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -28
- package/dist/chunk-SP24UFRC.js +987 -0
- package/dist/index.js +655 -85
- package/dist/mcp/server.js +8 -2
- package/dist/runner.d.ts +13 -1
- package/dist/runner.js +1 -1
- package/package.json +2 -2
- package/dist/chunk-OOJ4PMF7.js +0 -563
package/dist/chunk-OOJ4PMF7.js
DELETED
|
@@ -1,563 +0,0 @@
|
|
|
1
|
-
// src/runner.ts
|
|
2
|
-
import { request } from "undici";
|
|
3
|
-
|
|
4
|
-
// src/grader.ts
|
|
5
|
-
function computeGrade(score) {
|
|
6
|
-
if (score >= 90) return "A";
|
|
7
|
-
if (score >= 75) return "B";
|
|
8
|
-
if (score >= 60) return "C";
|
|
9
|
-
if (score >= 40) return "D";
|
|
10
|
-
return "F";
|
|
11
|
-
}
|
|
12
|
-
function computeScore(tests) {
|
|
13
|
-
const total = tests.length;
|
|
14
|
-
const passed = tests.filter((t) => t.passed).length;
|
|
15
|
-
const failed = total - passed;
|
|
16
|
-
const requiredTests = tests.filter((t) => t.required);
|
|
17
|
-
const requiredPassed = requiredTests.filter((t) => t.passed).length;
|
|
18
|
-
const requiredScore = requiredTests.length > 0 ? requiredPassed / requiredTests.length * 70 : 70;
|
|
19
|
-
const optionalTests = tests.filter((t) => !t.required);
|
|
20
|
-
const optionalPassed = optionalTests.filter((t) => t.passed).length;
|
|
21
|
-
const optionalScore = optionalTests.length > 0 ? optionalPassed / optionalTests.length * 30 : 30;
|
|
22
|
-
const score = Math.round(requiredScore + optionalScore);
|
|
23
|
-
const overall = requiredPassed === requiredTests.length ? passed === total ? "pass" : "partial" : "fail";
|
|
24
|
-
const categories = {};
|
|
25
|
-
for (const t of tests) {
|
|
26
|
-
if (!categories[t.category]) categories[t.category] = { passed: 0, total: 0 };
|
|
27
|
-
categories[t.category].total++;
|
|
28
|
-
if (t.passed) categories[t.category].passed++;
|
|
29
|
-
}
|
|
30
|
-
return {
|
|
31
|
-
score,
|
|
32
|
-
grade: computeGrade(score),
|
|
33
|
-
overall,
|
|
34
|
-
summary: { total, passed, failed, required: requiredTests.length, requiredPassed },
|
|
35
|
-
categories
|
|
36
|
-
};
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
// src/badge.ts
|
|
40
|
-
function generateBadge(url) {
|
|
41
|
-
let parsed;
|
|
42
|
-
try {
|
|
43
|
-
parsed = new URL(url);
|
|
44
|
-
} catch {
|
|
45
|
-
parsed = new URL("https://unknown");
|
|
46
|
-
}
|
|
47
|
-
const encoded = encodeURIComponent(parsed.href);
|
|
48
|
-
const imageUrl = `https://mcp.hosting/api/compliance/${encoded}/badge`;
|
|
49
|
-
const reportUrl = `https://mcp.hosting/compliance/${encoded}`;
|
|
50
|
-
return {
|
|
51
|
-
imageUrl,
|
|
52
|
-
reportUrl,
|
|
53
|
-
markdown: `[](${reportUrl})`,
|
|
54
|
-
html: `<a href="${reportUrl}"><img src="${imageUrl}" alt="MCP Compliant"></a>`
|
|
55
|
-
};
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
// src/types.ts
|
|
59
|
-
var TEST_DEFINITIONS = [
|
|
60
|
-
{ id: "transport-post", name: "HTTP POST accepted", category: "transport", required: true, specRef: "basic/transports#streamable-http", description: "Verifies the server accepts HTTP POST requests and returns a 2xx status code. This is the fundamental transport requirement for Streamable HTTP MCP servers." },
|
|
61
|
-
{ id: "transport-content-type", name: "Responds with JSON or SSE", category: "transport", required: true, specRef: "basic/transports#streamable-http", description: "Checks that the server responds with Content-Type application/json or text/event-stream. MCP servers must use one of these two content types." },
|
|
62
|
-
{ id: "lifecycle-init", name: "Initialize handshake", category: "lifecycle", required: true, specRef: "basic/lifecycle#initialization", description: "Tests the initialize handshake by sending an initialize request with client capabilities. The server must return a result with protocolVersion." },
|
|
63
|
-
{ id: "lifecycle-proto-version", name: "Returns valid protocol version", category: "lifecycle", required: true, specRef: "basic/lifecycle#version-negotiation", description: "Validates that the protocolVersion returned by the server matches the YYYY-MM-DD date format required by the spec." },
|
|
64
|
-
{ id: "lifecycle-server-info", name: "Includes serverInfo", category: "lifecycle", required: false, specRef: "basic/lifecycle#initialization", description: "Checks that the server includes a serverInfo object with at least a name field in its initialize response. While recommended, this is not strictly required." },
|
|
65
|
-
{ id: "lifecycle-capabilities", name: "Returns capabilities object", category: "lifecycle", required: true, specRef: "basic/lifecycle#capability-negotiation", description: "Verifies the server returns a capabilities object in its initialize response. An empty object is valid (no optional features declared)." },
|
|
66
|
-
{ id: "lifecycle-jsonrpc", name: "Response is valid JSON-RPC 2.0", category: "lifecycle", required: true, specRef: "basic", description: 'Validates that the initialize response is a proper JSON-RPC 2.0 message with jsonrpc="2.0", an id field, and either a result or error field.' },
|
|
67
|
-
{ id: "lifecycle-ping", name: "Responds to ping", category: "lifecycle", required: true, specRef: "basic/utilities#ping", description: "Tests that the server responds to the ping method with an empty result object. This is a required utility method." },
|
|
68
|
-
{ id: "tools-list", name: "tools/list returns valid response", category: "tools", required: false, specRef: "server/tools#listing-tools", description: "Calls tools/list and validates it returns an array of tool definitions. Required if the server declares tools capability." },
|
|
69
|
-
{ id: "tools-schema", name: "All tools have name and inputSchema", category: "schema", required: false, specRef: "server/tools#data-types", description: 'Validates every tool has a valid name (1-128 chars, alphanumeric/underscore/hyphen/dot) and a required inputSchema of type "object".' },
|
|
70
|
-
{ id: "tools-call", name: "tools/call responds correctly", category: "tools", required: false, specRef: "server/tools#calling-tools", description: "Calls the first tool with empty arguments and verifies the response format. Accepts both successful results and InvalidParams errors." },
|
|
71
|
-
{ id: "tools-call-unknown", name: "Returns error for unknown tool name", category: "errors", required: false, specRef: "server/tools#error-handling", description: "Calls tools/call with a nonexistent tool name and verifies the server returns an error response." },
|
|
72
|
-
{ id: "resources-list", name: "resources/list returns valid response", category: "resources", required: false, specRef: "server/resources#listing-resources", description: "Calls resources/list and validates it returns an array. Required if the server declares resources capability." },
|
|
73
|
-
{ id: "resources-schema", name: "Resources have uri and name", category: "schema", required: false, specRef: "server/resources#data-types", description: "Validates every resource has a valid URI (parseable as a URL) and a name field." },
|
|
74
|
-
{ id: "resources-read", name: "resources/read returns content", category: "resources", required: false, specRef: "server/resources#reading-resources", description: "Reads the first resource and validates the response contains a contents array with proper uri and text/blob fields." },
|
|
75
|
-
{ id: "resources-templates", name: "resources/templates/list returns valid response", category: "resources", required: false, specRef: "server/resources#resource-templates", description: "Tests the resource templates endpoint. Accepts Method not found (-32601) since templates are optional." },
|
|
76
|
-
{ id: "prompts-list", name: "prompts/list returns valid response", category: "prompts", required: false, specRef: "server/prompts#listing-prompts", description: "Calls prompts/list and validates it returns an array. Required if the server declares prompts capability." },
|
|
77
|
-
{ id: "prompts-schema", name: "Prompts have name field", category: "schema", required: false, specRef: "server/prompts#data-types", description: "Validates every prompt has a name and that any arguments array contains items with name fields." },
|
|
78
|
-
{ id: "prompts-get", name: "prompts/get returns valid messages", category: "prompts", required: false, specRef: "server/prompts#getting-a-prompt", description: "Gets the first prompt and validates the response contains a messages array with proper role and content fields." },
|
|
79
|
-
{ id: "error-unknown-method", name: "Returns JSON-RPC error for unknown method", category: "errors", required: true, specRef: "basic", description: "Sends an unknown method and verifies the server returns a JSON-RPC error. The spec requires error code -32601 (Method not found)." },
|
|
80
|
-
{ id: "error-method-code", name: "Uses correct JSON-RPC error code for unknown method", category: "errors", required: false, specRef: "basic", description: "Checks the error code is specifically -32601 (Method not found) for unknown methods, as required by JSON-RPC 2.0." },
|
|
81
|
-
{ id: "error-invalid-jsonrpc", name: "Handles malformed JSON-RPC", category: "errors", required: true, specRef: "basic", description: "Sends a malformed JSON-RPC message (missing required fields) and verifies the server returns an error or 4xx status." },
|
|
82
|
-
{ id: "error-invalid-json", name: "Handles invalid JSON body", category: "errors", required: false, specRef: "basic", description: "Sends invalid JSON and verifies the server returns a parse error (-32700) or 4xx status code." },
|
|
83
|
-
{ id: "error-missing-params", name: "Returns error for tools/call without name", category: "errors", required: false, specRef: "server/tools#error-handling", description: "Calls tools/call with an empty params object (missing required name field) and verifies an error is returned." }
|
|
84
|
-
];
|
|
85
|
-
|
|
86
|
-
// src/runner.ts
|
|
87
|
-
var SPEC_VERSION = "2025-11-25";
|
|
88
|
-
var SPEC_BASE = `https://modelcontextprotocol.io/specification/${SPEC_VERSION}`;
|
|
89
|
-
function createIdCounter() {
|
|
90
|
-
let id = 0;
|
|
91
|
-
return () => ++id;
|
|
92
|
-
}
|
|
93
|
-
function parseSSEResponse(text) {
|
|
94
|
-
const lines = text.split("\n");
|
|
95
|
-
let lastJsonRpcResponse = null;
|
|
96
|
-
for (const line of lines) {
|
|
97
|
-
if (line.startsWith("data: ")) {
|
|
98
|
-
const data = line.slice(6).trim();
|
|
99
|
-
if (!data) continue;
|
|
100
|
-
try {
|
|
101
|
-
const parsed = JSON.parse(data);
|
|
102
|
-
if (parsed.jsonrpc === "2.0" && parsed.id !== void 0) {
|
|
103
|
-
lastJsonRpcResponse = parsed;
|
|
104
|
-
}
|
|
105
|
-
} catch {
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
return lastJsonRpcResponse;
|
|
110
|
-
}
|
|
111
|
-
var _defaultNextId = createIdCounter();
|
|
112
|
-
async function mcpRequest(backendUrl, method, params, nextId = _defaultNextId, extraHeaders) {
|
|
113
|
-
const id = nextId();
|
|
114
|
-
const body = JSON.stringify({
|
|
115
|
-
jsonrpc: "2.0",
|
|
116
|
-
id,
|
|
117
|
-
method,
|
|
118
|
-
params: params || {}
|
|
119
|
-
});
|
|
120
|
-
const headers = {
|
|
121
|
-
"Content-Type": "application/json",
|
|
122
|
-
"Accept": "application/json, text/event-stream",
|
|
123
|
-
...extraHeaders
|
|
124
|
-
};
|
|
125
|
-
const res = await request(backendUrl, {
|
|
126
|
-
method: "POST",
|
|
127
|
-
headers,
|
|
128
|
-
body,
|
|
129
|
-
signal: AbortSignal.timeout(15e3)
|
|
130
|
-
});
|
|
131
|
-
const text = await res.body.text();
|
|
132
|
-
const responseHeaders = {};
|
|
133
|
-
for (const [k, v] of Object.entries(res.headers)) {
|
|
134
|
-
if (typeof v === "string") responseHeaders[k] = v;
|
|
135
|
-
}
|
|
136
|
-
const contentType = (responseHeaders["content-type"] || "").toLowerCase();
|
|
137
|
-
if (contentType.includes("text/event-stream")) {
|
|
138
|
-
const parsed = parseSSEResponse(text);
|
|
139
|
-
if (parsed) {
|
|
140
|
-
return { statusCode: res.statusCode, body: parsed, headers: responseHeaders };
|
|
141
|
-
}
|
|
142
|
-
try {
|
|
143
|
-
return { statusCode: res.statusCode, body: JSON.parse(text), headers: responseHeaders };
|
|
144
|
-
} catch {
|
|
145
|
-
return { statusCode: res.statusCode, body: { _raw: text }, headers: responseHeaders };
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
try {
|
|
149
|
-
return { statusCode: res.statusCode, body: JSON.parse(text), headers: responseHeaders };
|
|
150
|
-
} catch {
|
|
151
|
-
return { statusCode: res.statusCode, body: { _raw: text }, headers: responseHeaders };
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
async function mcpNotification(backendUrl, method, params, extraHeaders) {
|
|
155
|
-
const headers = {
|
|
156
|
-
"Content-Type": "application/json",
|
|
157
|
-
...extraHeaders
|
|
158
|
-
};
|
|
159
|
-
await request(backendUrl, {
|
|
160
|
-
method: "POST",
|
|
161
|
-
headers,
|
|
162
|
-
body: JSON.stringify({ jsonrpc: "2.0", method, ...params ? { params } : {} }),
|
|
163
|
-
signal: AbortSignal.timeout(5e3)
|
|
164
|
-
}).then((r) => r.body.text()).catch(() => {
|
|
165
|
-
});
|
|
166
|
-
}
|
|
167
|
-
async function runComplianceSuite(url, options = {}) {
|
|
168
|
-
let parsed;
|
|
169
|
-
try {
|
|
170
|
-
parsed = new URL(url);
|
|
171
|
-
if (!["http:", "https:"].includes(parsed.protocol)) {
|
|
172
|
-
throw new Error("Only HTTP and HTTPS URLs are supported");
|
|
173
|
-
}
|
|
174
|
-
} catch (e) {
|
|
175
|
-
if (e.message.includes("Only HTTP")) throw e;
|
|
176
|
-
throw new Error(`Invalid URL: ${url}`);
|
|
177
|
-
}
|
|
178
|
-
const backendUrl = url;
|
|
179
|
-
const tests = [];
|
|
180
|
-
const nextId = createIdCounter();
|
|
181
|
-
let sessionId = null;
|
|
182
|
-
let negotiatedProtocolVersion = null;
|
|
183
|
-
const userHeaders = options.headers || {};
|
|
184
|
-
function buildHeaders() {
|
|
185
|
-
const h = { ...userHeaders };
|
|
186
|
-
if (sessionId) h["mcp-session-id"] = sessionId;
|
|
187
|
-
if (negotiatedProtocolVersion) h["mcp-protocol-version"] = negotiatedProtocolVersion;
|
|
188
|
-
return h;
|
|
189
|
-
}
|
|
190
|
-
const rpc = (method, params) => mcpRequest(backendUrl, method, params, nextId, buildHeaders());
|
|
191
|
-
let serverInfo = {
|
|
192
|
-
protocolVersion: null,
|
|
193
|
-
name: null,
|
|
194
|
-
version: null,
|
|
195
|
-
capabilities: {}
|
|
196
|
-
};
|
|
197
|
-
let toolCount = 0;
|
|
198
|
-
let toolNames = [];
|
|
199
|
-
let resourceCount = 0;
|
|
200
|
-
let promptCount = 0;
|
|
201
|
-
async function test(id, name, category, required, specRef, fn) {
|
|
202
|
-
const start = Date.now();
|
|
203
|
-
try {
|
|
204
|
-
const result = await fn();
|
|
205
|
-
tests.push({
|
|
206
|
-
id,
|
|
207
|
-
name,
|
|
208
|
-
category,
|
|
209
|
-
required,
|
|
210
|
-
passed: result.passed,
|
|
211
|
-
details: result.details,
|
|
212
|
-
durationMs: Date.now() - start,
|
|
213
|
-
specRef: `${SPEC_BASE}/${specRef}`
|
|
214
|
-
});
|
|
215
|
-
options.onProgress?.(id, result.passed, result.details);
|
|
216
|
-
} catch (err) {
|
|
217
|
-
tests.push({
|
|
218
|
-
id,
|
|
219
|
-
name,
|
|
220
|
-
category,
|
|
221
|
-
required,
|
|
222
|
-
passed: false,
|
|
223
|
-
details: `Error: ${err.message}`,
|
|
224
|
-
durationMs: Date.now() - start,
|
|
225
|
-
specRef: `${SPEC_BASE}/${specRef}`
|
|
226
|
-
});
|
|
227
|
-
options.onProgress?.(id, false, `Error: ${err.message}`);
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
await test("transport-post", "HTTP POST accepted", "transport", true, "basic/transports#streamable-http", async () => {
|
|
231
|
-
const res = await request(backendUrl, {
|
|
232
|
-
method: "POST",
|
|
233
|
-
headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...userHeaders },
|
|
234
|
-
body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "ping" }),
|
|
235
|
-
signal: AbortSignal.timeout(1e4)
|
|
236
|
-
});
|
|
237
|
-
await res.body.text();
|
|
238
|
-
const passed = res.statusCode >= 200 && res.statusCode < 300;
|
|
239
|
-
const note = res.statusCode === 401 || res.statusCode === 403 ? " (auth required)" : "";
|
|
240
|
-
return { passed, details: `HTTP ${res.statusCode}${note}` };
|
|
241
|
-
});
|
|
242
|
-
await test("transport-content-type", "Responds with JSON or SSE", "transport", true, "basic/transports#streamable-http", async () => {
|
|
243
|
-
const res = await request(backendUrl, {
|
|
244
|
-
method: "POST",
|
|
245
|
-
headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", ...userHeaders },
|
|
246
|
-
body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "ping" }),
|
|
247
|
-
signal: AbortSignal.timeout(1e4)
|
|
248
|
-
});
|
|
249
|
-
await res.body.text();
|
|
250
|
-
const rawCt = res.headers["content-type"];
|
|
251
|
-
const ct = (Array.isArray(rawCt) ? rawCt[0] : rawCt || "").toLowerCase();
|
|
252
|
-
const valid = ct.includes("application/json") || ct.includes("text/event-stream");
|
|
253
|
-
return { passed: valid, details: `Content-Type: ${ct}` };
|
|
254
|
-
});
|
|
255
|
-
let initRes = null;
|
|
256
|
-
await test("lifecycle-init", "Initialize handshake", "lifecycle", true, "basic/lifecycle#initialization", async () => {
|
|
257
|
-
initRes = await rpc("initialize", {
|
|
258
|
-
protocolVersion: SPEC_VERSION,
|
|
259
|
-
capabilities: { roots: { listChanged: true }, sampling: {} },
|
|
260
|
-
clientInfo: { name: "mcp-compliance", version: "1.0.0" }
|
|
261
|
-
});
|
|
262
|
-
const result = initRes.body?.result;
|
|
263
|
-
if (!result) return { passed: false, details: "No result in response" };
|
|
264
|
-
serverInfo.protocolVersion = result.protocolVersion || null;
|
|
265
|
-
serverInfo.name = result.serverInfo?.name || null;
|
|
266
|
-
serverInfo.version = result.serverInfo?.version || null;
|
|
267
|
-
serverInfo.capabilities = result.capabilities || {};
|
|
268
|
-
const sid = initRes.headers["mcp-session-id"];
|
|
269
|
-
if (sid) sessionId = sid;
|
|
270
|
-
if (result.protocolVersion) negotiatedProtocolVersion = result.protocolVersion;
|
|
271
|
-
return { passed: !!result.protocolVersion, details: `Protocol: ${result.protocolVersion || "missing"}` };
|
|
272
|
-
});
|
|
273
|
-
await test("lifecycle-proto-version", "Returns valid protocol version", "lifecycle", true, "basic/lifecycle#version-negotiation", async () => {
|
|
274
|
-
const version = initRes?.body?.result?.protocolVersion;
|
|
275
|
-
if (!version) return { passed: false, details: "No protocolVersion" };
|
|
276
|
-
const valid = /^\d{4}-\d{2}-\d{2}$/.test(version);
|
|
277
|
-
return { passed: valid, details: `Version: ${version}` };
|
|
278
|
-
});
|
|
279
|
-
await test("lifecycle-server-info", "Includes serverInfo", "lifecycle", false, "basic/lifecycle#initialization", async () => {
|
|
280
|
-
const info = initRes?.body?.result?.serverInfo;
|
|
281
|
-
return { passed: !!info?.name, details: info ? `${info.name} v${info.version || "?"}` : "Missing serverInfo" };
|
|
282
|
-
});
|
|
283
|
-
await test("lifecycle-capabilities", "Returns capabilities object", "lifecycle", true, "basic/lifecycle#capability-negotiation", async () => {
|
|
284
|
-
const caps = initRes?.body?.result?.capabilities;
|
|
285
|
-
if (!caps || typeof caps !== "object") return { passed: false, details: "No capabilities object in response" };
|
|
286
|
-
const declared = Object.keys(caps).filter((k) => caps[k] !== void 0);
|
|
287
|
-
return { passed: true, details: declared.length > 0 ? `Capabilities: ${declared.join(", ")}` : "Empty capabilities (valid)" };
|
|
288
|
-
});
|
|
289
|
-
await test("lifecycle-jsonrpc", "Response is valid JSON-RPC 2.0", "lifecycle", true, "basic", async () => {
|
|
290
|
-
const body = initRes?.body;
|
|
291
|
-
const valid = body?.jsonrpc === "2.0" && body?.id !== void 0 && (body?.result !== void 0 || body?.error !== void 0);
|
|
292
|
-
return { passed: valid, details: valid ? "Valid JSON-RPC 2.0 response" : `Missing fields: jsonrpc=${body?.jsonrpc}, id=${body?.id}` };
|
|
293
|
-
});
|
|
294
|
-
await mcpNotification(backendUrl, "notifications/initialized", void 0, buildHeaders());
|
|
295
|
-
await test("lifecycle-ping", "Responds to ping", "lifecycle", true, "basic/utilities#ping", async () => {
|
|
296
|
-
const res = await rpc("ping");
|
|
297
|
-
const body = res.body;
|
|
298
|
-
if (body?.error) return { passed: false, details: `Error: ${body.error.message}` };
|
|
299
|
-
if (body?.result !== void 0) return { passed: true, details: "Ping responded successfully" };
|
|
300
|
-
return { passed: false, details: "No result in ping response" };
|
|
301
|
-
});
|
|
302
|
-
const hasTools = !!serverInfo.capabilities.tools;
|
|
303
|
-
let cachedToolsList = null;
|
|
304
|
-
await test("tools-list", "tools/list returns valid response", "tools", hasTools, "server/tools#listing-tools", async () => {
|
|
305
|
-
const res = await rpc("tools/list");
|
|
306
|
-
const tools = res.body?.result?.tools;
|
|
307
|
-
if (!Array.isArray(tools)) return { passed: false, details: "No tools array in result" };
|
|
308
|
-
cachedToolsList = tools;
|
|
309
|
-
toolCount = tools.length;
|
|
310
|
-
toolNames = tools.map((t) => t.name).filter(Boolean);
|
|
311
|
-
return { passed: true, details: `${toolCount} tool(s): ${toolNames.slice(0, 5).join(", ")}${toolCount > 5 ? "..." : ""}` };
|
|
312
|
-
});
|
|
313
|
-
await test("tools-schema", "All tools have name and inputSchema", "schema", hasTools, "server/tools#data-types", async () => {
|
|
314
|
-
const tools = cachedToolsList ?? (await rpc("tools/list")).body?.result?.tools ?? [];
|
|
315
|
-
const issues = [];
|
|
316
|
-
const warnings = [];
|
|
317
|
-
for (const tool of tools) {
|
|
318
|
-
if (!tool.name) {
|
|
319
|
-
issues.push("Tool missing name");
|
|
320
|
-
continue;
|
|
321
|
-
}
|
|
322
|
-
if (tool.name.length > 128 || !/^[A-Za-z0-9_.\-]+$/.test(tool.name)) {
|
|
323
|
-
issues.push(`${tool.name}: name format invalid`);
|
|
324
|
-
}
|
|
325
|
-
if (!tool.description) warnings.push(`${tool.name}: missing description`);
|
|
326
|
-
if (!tool.inputSchema) {
|
|
327
|
-
issues.push(`${tool.name}: missing inputSchema (required)`);
|
|
328
|
-
} else if (typeof tool.inputSchema !== "object" || tool.inputSchema === null) {
|
|
329
|
-
issues.push(`${tool.name}: inputSchema must be a valid JSON Schema object`);
|
|
330
|
-
} else if (tool.inputSchema.type !== "object") {
|
|
331
|
-
issues.push(`${tool.name}: inputSchema.type must be "object" (got "${tool.inputSchema.type || "undefined"}")`);
|
|
332
|
-
}
|
|
333
|
-
}
|
|
334
|
-
const detail = issues.length === 0 ? warnings.length > 0 ? `Schemas valid. Warnings: ${warnings.join("; ")}` : "All tools have valid schemas" : issues.join("; ");
|
|
335
|
-
return { passed: issues.length === 0, details: detail };
|
|
336
|
-
});
|
|
337
|
-
if (toolNames.length > 0) {
|
|
338
|
-
await test("tools-call", "tools/call responds correctly", "tools", false, "server/tools#calling-tools", async () => {
|
|
339
|
-
const res = await rpc("tools/call", { name: toolNames[0], arguments: {} });
|
|
340
|
-
const result = res.body?.result;
|
|
341
|
-
const error = res.body?.error;
|
|
342
|
-
if (error) {
|
|
343
|
-
const code = error.code;
|
|
344
|
-
if (code === -32602 || code === -32600) {
|
|
345
|
-
return { passed: true, details: `Invalid params error (acceptable): code ${code}` };
|
|
346
|
-
}
|
|
347
|
-
return { passed: true, details: `Protocol error: code ${code} \u2014 ${error.message}` };
|
|
348
|
-
}
|
|
349
|
-
if (result?.content && Array.isArray(result.content)) {
|
|
350
|
-
const badItems = result.content.filter((c) => !c.type);
|
|
351
|
-
if (badItems.length > 0) return { passed: false, details: `${badItems.length} content item(s) missing 'type' field` };
|
|
352
|
-
return { passed: true, details: `Returned ${result.content.length} content item(s)` };
|
|
353
|
-
}
|
|
354
|
-
if (result?.isError && result?.content && Array.isArray(result.content)) {
|
|
355
|
-
return { passed: true, details: "Tool returned execution error with content (valid)" };
|
|
356
|
-
}
|
|
357
|
-
return { passed: false, details: "Response missing content array" };
|
|
358
|
-
});
|
|
359
|
-
await test("tools-call-unknown", "Returns error for unknown tool name", "errors", false, "server/tools#error-handling", async () => {
|
|
360
|
-
const res = await rpc("tools/call", { name: "__nonexistent_tool_compliance_test__", arguments: {} });
|
|
361
|
-
const error = res.body?.error;
|
|
362
|
-
const isError = res.body?.result?.isError;
|
|
363
|
-
if (error) return { passed: true, details: `Error code: ${error.code} \u2014 ${error.message}` };
|
|
364
|
-
if (isError) return { passed: true, details: "Tool execution error with isError=true (valid)" };
|
|
365
|
-
return { passed: false, details: "No error returned for nonexistent tool" };
|
|
366
|
-
});
|
|
367
|
-
}
|
|
368
|
-
const hasResources = !!serverInfo.capabilities.resources;
|
|
369
|
-
if (hasResources) {
|
|
370
|
-
let cachedResourcesList = null;
|
|
371
|
-
await test("resources-list", "resources/list returns valid response", "resources", true, "server/resources#listing-resources", async () => {
|
|
372
|
-
const res = await rpc("resources/list");
|
|
373
|
-
const resources = res.body?.result?.resources;
|
|
374
|
-
if (!Array.isArray(resources)) return { passed: false, details: "No resources array" };
|
|
375
|
-
cachedResourcesList = resources;
|
|
376
|
-
resourceCount = resources.length;
|
|
377
|
-
return { passed: true, details: `${resourceCount} resource(s)` };
|
|
378
|
-
});
|
|
379
|
-
await test("resources-schema", "Resources have uri and name", "schema", true, "server/resources#data-types", async () => {
|
|
380
|
-
const resources = cachedResourcesList ?? (await rpc("resources/list")).body?.result?.resources ?? [];
|
|
381
|
-
const issues = [];
|
|
382
|
-
for (const r of resources) {
|
|
383
|
-
if (!r.uri) issues.push("Resource missing uri");
|
|
384
|
-
else {
|
|
385
|
-
try {
|
|
386
|
-
new URL(r.uri);
|
|
387
|
-
} catch {
|
|
388
|
-
issues.push(`${r.uri}: invalid URI format`);
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
if (!r.name) issues.push(`${r.uri || "?"}: missing name`);
|
|
392
|
-
}
|
|
393
|
-
return { passed: issues.length === 0, details: issues.length === 0 ? "All resources valid" : issues.join("; ") };
|
|
394
|
-
});
|
|
395
|
-
if (resourceCount > 0) {
|
|
396
|
-
await test("resources-read", "resources/read returns content", "resources", false, "server/resources#reading-resources", async () => {
|
|
397
|
-
const listRes = await rpc("resources/list");
|
|
398
|
-
const firstUri = listRes.body?.result?.resources?.[0]?.uri;
|
|
399
|
-
if (!firstUri) return { passed: false, details: "No resource URI to test" };
|
|
400
|
-
const readRes = await rpc("resources/read", { uri: firstUri });
|
|
401
|
-
const contents = readRes.body?.result?.contents;
|
|
402
|
-
if (!Array.isArray(contents)) return { passed: false, details: "No contents array" };
|
|
403
|
-
const issues = [];
|
|
404
|
-
for (const c of contents) {
|
|
405
|
-
if (!c.uri) issues.push("Content item missing uri");
|
|
406
|
-
if (!c.text && !c.blob) issues.push(`Content item for ${c.uri || "?"} missing both text and blob`);
|
|
407
|
-
}
|
|
408
|
-
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
409
|
-
return { passed: true, details: `Read ${contents.length} content item(s) from ${firstUri}` };
|
|
410
|
-
});
|
|
411
|
-
}
|
|
412
|
-
await test("resources-templates", "resources/templates/list returns valid response", "resources", false, "server/resources#resource-templates", async () => {
|
|
413
|
-
const res = await rpc("resources/templates/list");
|
|
414
|
-
const error = res.body?.error;
|
|
415
|
-
if (error) {
|
|
416
|
-
if (error.code === -32601) return { passed: true, details: "Method not supported (acceptable)" };
|
|
417
|
-
return { passed: false, details: `Error: ${error.message}` };
|
|
418
|
-
}
|
|
419
|
-
const templates = res.body?.result?.resourceTemplates;
|
|
420
|
-
if (!Array.isArray(templates)) return { passed: false, details: "No resourceTemplates array" };
|
|
421
|
-
const issues = [];
|
|
422
|
-
for (const t of templates) {
|
|
423
|
-
if (!t.uriTemplate) issues.push("Template missing uriTemplate");
|
|
424
|
-
if (!t.name) issues.push(`${t.uriTemplate || "?"}: missing name`);
|
|
425
|
-
}
|
|
426
|
-
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
427
|
-
return { passed: true, details: `${templates.length} resource template(s)` };
|
|
428
|
-
});
|
|
429
|
-
}
|
|
430
|
-
const hasPrompts = !!serverInfo.capabilities.prompts;
|
|
431
|
-
if (hasPrompts) {
|
|
432
|
-
let promptNames = [];
|
|
433
|
-
let cachedPromptsList = null;
|
|
434
|
-
await test("prompts-list", "prompts/list returns valid response", "prompts", true, "server/prompts#listing-prompts", async () => {
|
|
435
|
-
const res = await rpc("prompts/list");
|
|
436
|
-
const prompts = res.body?.result?.prompts;
|
|
437
|
-
if (!Array.isArray(prompts)) return { passed: false, details: "No prompts array" };
|
|
438
|
-
cachedPromptsList = prompts;
|
|
439
|
-
promptCount = prompts.length;
|
|
440
|
-
promptNames = prompts.map((p) => p.name).filter(Boolean);
|
|
441
|
-
return { passed: true, details: `${promptCount} prompt(s): ${promptNames.slice(0, 5).join(", ")}${promptCount > 5 ? "..." : ""}` };
|
|
442
|
-
});
|
|
443
|
-
await test("prompts-schema", "Prompts have name field", "schema", true, "server/prompts#data-types", async () => {
|
|
444
|
-
const prompts = cachedPromptsList ?? (await rpc("prompts/list")).body?.result?.prompts ?? [];
|
|
445
|
-
const issues = [];
|
|
446
|
-
for (const p of prompts) {
|
|
447
|
-
if (!p.name) issues.push("Prompt missing name");
|
|
448
|
-
if (p.arguments && !Array.isArray(p.arguments)) issues.push(`${p.name || "?"}: arguments must be an array`);
|
|
449
|
-
if (Array.isArray(p.arguments)) {
|
|
450
|
-
for (const arg of p.arguments) {
|
|
451
|
-
if (!arg.name) issues.push(`${p.name}: argument missing name`);
|
|
452
|
-
}
|
|
453
|
-
}
|
|
454
|
-
}
|
|
455
|
-
return { passed: issues.length === 0, details: issues.length === 0 ? "All prompts valid" : issues.join("; ") };
|
|
456
|
-
});
|
|
457
|
-
if (promptNames.length > 0) {
|
|
458
|
-
await test("prompts-get", "prompts/get returns valid messages", "prompts", false, "server/prompts#getting-a-prompt", async () => {
|
|
459
|
-
const res = await rpc("prompts/get", { name: promptNames[0] });
|
|
460
|
-
const error = res.body?.error;
|
|
461
|
-
if (error) return { passed: true, details: `Error (may need arguments): code ${error.code}` };
|
|
462
|
-
const messages = res.body?.result?.messages;
|
|
463
|
-
if (!Array.isArray(messages)) return { passed: false, details: "No messages array in result" };
|
|
464
|
-
const issues = [];
|
|
465
|
-
for (const msg of messages) {
|
|
466
|
-
if (!msg.role || !["user", "assistant"].includes(msg.role)) issues.push(`Invalid role: ${msg.role}`);
|
|
467
|
-
if (!msg.content) issues.push("Message missing content");
|
|
468
|
-
}
|
|
469
|
-
if (issues.length > 0) return { passed: false, details: issues.join("; ") };
|
|
470
|
-
return { passed: true, details: `${messages.length} message(s) from ${promptNames[0]}` };
|
|
471
|
-
});
|
|
472
|
-
}
|
|
473
|
-
}
|
|
474
|
-
await test("error-unknown-method", "Returns JSON-RPC error for unknown method", "errors", true, "basic", async () => {
|
|
475
|
-
const res = await rpc("nonexistent/method");
|
|
476
|
-
const error = res.body?.error;
|
|
477
|
-
if (!error) return { passed: false, details: "No JSON-RPC error returned for unknown method" };
|
|
478
|
-
const correctCode = error.code === -32601;
|
|
479
|
-
return {
|
|
480
|
-
passed: true,
|
|
481
|
-
details: `Error code: ${error.code}${correctCode ? " (correct: Method not found)" : " (expected -32601)"} \u2014 ${error.message}`
|
|
482
|
-
};
|
|
483
|
-
});
|
|
484
|
-
await test("error-method-code", "Uses correct JSON-RPC error code for unknown method", "errors", false, "basic", async () => {
|
|
485
|
-
const res = await rpc("nonexistent/method");
|
|
486
|
-
const error = res.body?.error;
|
|
487
|
-
if (!error) return { passed: false, details: "No error returned" };
|
|
488
|
-
return { passed: error.code === -32601, details: `Expected -32601, got ${error.code}` };
|
|
489
|
-
});
|
|
490
|
-
await test("error-invalid-jsonrpc", "Handles malformed JSON-RPC", "errors", true, "basic", async () => {
|
|
491
|
-
const res = await request(backendUrl, {
|
|
492
|
-
method: "POST",
|
|
493
|
-
headers: { "Content-Type": "application/json", ...buildHeaders() },
|
|
494
|
-
body: JSON.stringify({ not: "a valid jsonrpc message" }),
|
|
495
|
-
signal: AbortSignal.timeout(1e4)
|
|
496
|
-
});
|
|
497
|
-
const text = await res.body.text();
|
|
498
|
-
try {
|
|
499
|
-
const body = JSON.parse(text);
|
|
500
|
-
if (body?.error) {
|
|
501
|
-
const correctCode = body.error.code === -32600;
|
|
502
|
-
return { passed: true, details: `Error code: ${body.error.code}${correctCode ? " (correct: Invalid Request)" : ""} \u2014 ${body.error.message}` };
|
|
503
|
-
}
|
|
504
|
-
} catch {
|
|
505
|
-
}
|
|
506
|
-
if (res.statusCode >= 400 && res.statusCode < 500) return { passed: true, details: `HTTP ${res.statusCode} (acceptable)` };
|
|
507
|
-
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected JSON-RPC error or 4xx status` };
|
|
508
|
-
});
|
|
509
|
-
await test("error-invalid-json", "Handles invalid JSON body", "errors", false, "basic", async () => {
|
|
510
|
-
const res = await request(backendUrl, {
|
|
511
|
-
method: "POST",
|
|
512
|
-
headers: { "Content-Type": "application/json", ...buildHeaders() },
|
|
513
|
-
body: "{this is not valid json!!!",
|
|
514
|
-
signal: AbortSignal.timeout(1e4)
|
|
515
|
-
});
|
|
516
|
-
const text = await res.body.text();
|
|
517
|
-
try {
|
|
518
|
-
const body = JSON.parse(text);
|
|
519
|
-
if (body?.error) return { passed: true, details: `Error code: ${body.error.code} \u2014 ${body.error.message}` };
|
|
520
|
-
} catch {
|
|
521
|
-
}
|
|
522
|
-
if (res.statusCode >= 400 && res.statusCode < 500) return { passed: true, details: `HTTP ${res.statusCode} (acceptable)` };
|
|
523
|
-
return { passed: false, details: `HTTP ${res.statusCode} \u2014 expected parse error or 4xx status` };
|
|
524
|
-
});
|
|
525
|
-
await test("error-missing-params", "Returns error for tools/call without name", "errors", false, "server/tools#error-handling", async () => {
|
|
526
|
-
const res = await rpc("tools/call", {});
|
|
527
|
-
const error = res.body?.error;
|
|
528
|
-
const isError = res.body?.result?.isError;
|
|
529
|
-
if (error) {
|
|
530
|
-
const correctCode = error.code === -32602;
|
|
531
|
-
return { passed: true, details: `Error code: ${error.code}${correctCode ? " (correct: Invalid params)" : ""} \u2014 ${error.message}` };
|
|
532
|
-
}
|
|
533
|
-
if (isError) return { passed: true, details: "Tool execution error (valid)" };
|
|
534
|
-
return { passed: false, details: "No error for tools/call without name" };
|
|
535
|
-
});
|
|
536
|
-
const { score, grade, overall, summary, categories } = computeScore(tests);
|
|
537
|
-
const badge = generateBadge(url);
|
|
538
|
-
return {
|
|
539
|
-
specVersion: SPEC_VERSION,
|
|
540
|
-
url,
|
|
541
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
542
|
-
score,
|
|
543
|
-
grade,
|
|
544
|
-
overall,
|
|
545
|
-
summary,
|
|
546
|
-
categories,
|
|
547
|
-
tests,
|
|
548
|
-
serverInfo,
|
|
549
|
-
toolCount,
|
|
550
|
-
toolNames,
|
|
551
|
-
resourceCount,
|
|
552
|
-
promptCount,
|
|
553
|
-
badge
|
|
554
|
-
};
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
export {
|
|
558
|
-
computeGrade,
|
|
559
|
-
computeScore,
|
|
560
|
-
generateBadge,
|
|
561
|
-
TEST_DEFINITIONS,
|
|
562
|
-
runComplianceSuite
|
|
563
|
-
};
|