cbrowser 18.14.1 → 18.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -6
- package/dist/analysis/accessibility-empathy.d.ts.map +1 -1
- package/dist/analysis/accessibility-empathy.js +408 -1
- package/dist/analysis/accessibility-empathy.js.map +1 -1
- package/dist/analysis/bug-hunter.d.ts +2 -1
- package/dist/analysis/bug-hunter.d.ts.map +1 -1
- package/dist/analysis/bug-hunter.js +137 -2
- package/dist/analysis/bug-hunter.js.map +1 -1
- package/dist/analysis/competitive-benchmark.d.ts.map +1 -1
- package/dist/analysis/competitive-benchmark.js +243 -113
- package/dist/analysis/competitive-benchmark.js.map +1 -1
- package/dist/analysis/index.d.ts +1 -0
- package/dist/analysis/index.d.ts.map +1 -1
- package/dist/analysis/index.js +1 -0
- package/dist/analysis/index.js.map +1 -1
- package/dist/analysis/webmcp-readiness.d.ts +15 -0
- package/dist/analysis/webmcp-readiness.d.ts.map +1 -0
- package/dist/analysis/webmcp-readiness.js +855 -0
- package/dist/analysis/webmcp-readiness.js.map +1 -0
- package/dist/mcp-tools/base/audit-tools.d.ts +1 -1
- package/dist/mcp-tools/base/audit-tools.d.ts.map +1 -1
- package/dist/mcp-tools/base/audit-tools.js +127 -30
- package/dist/mcp-tools/base/audit-tools.js.map +1 -1
- package/dist/mcp-tools/base/index.d.ts +4 -4
- package/dist/mcp-tools/base/index.js +4 -4
- package/dist/mcp-tools/index.d.ts +9 -9
- package/dist/mcp-tools/index.js +10 -10
- package/dist/types.d.ts +154 -9
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +9 -0
- package/dist/types.js.map +1 -1
- package/docs/hunt-bugs-coverage.md +103 -0
- package/package.json +2 -1
|
@@ -0,0 +1,855 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CBrowser - WebMCP Readiness Audit
|
|
3
|
+
*
|
|
4
|
+
* 6-tier evaluation framework for MCP server Claude in Chrome compatibility.
|
|
5
|
+
*
|
|
6
|
+
* @copyright 2026 Alexandria Eden alexandria.shai.eden@gmail.com https://cbrowser.ai
|
|
7
|
+
* @license MIT
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* Tier weights for scoring
|
|
11
|
+
*/
|
|
12
|
+
const TIER_CONFIG = {
|
|
13
|
+
1: { name: "Server Implementation", weight: 0.25 },
|
|
14
|
+
2: { name: "Tool Discoverability", weight: 0.20 },
|
|
15
|
+
3: { name: "Instrumentation", weight: 0.15 },
|
|
16
|
+
4: { name: "Consistency", weight: 0.15 },
|
|
17
|
+
5: { name: "Agent Optimizations", weight: 0.15 },
|
|
18
|
+
6: { name: "Documentation", weight: 0.10 },
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* Run WebMCP readiness audit on an MCP server
|
|
22
|
+
*/
|
|
23
|
+
export async function runWebMCPReadyAudit(url, options = {}) {
|
|
24
|
+
const startTime = Date.now();
|
|
25
|
+
const timeout = options.timeout || 30000;
|
|
26
|
+
// Normalize URL
|
|
27
|
+
const serverUrl = url.endsWith("/mcp") ? url : `${url.replace(/\/$/, "")}/mcp`;
|
|
28
|
+
const baseUrl = serverUrl.replace(/\/mcp$/, "");
|
|
29
|
+
// Initialize result structure
|
|
30
|
+
const tiers = [];
|
|
31
|
+
const issues = [];
|
|
32
|
+
let serverResponded = false;
|
|
33
|
+
let protocolVersion;
|
|
34
|
+
let toolCount;
|
|
35
|
+
let toolList = [];
|
|
36
|
+
// Helper to add issue
|
|
37
|
+
const addIssue = (tier, severity, issue, remediation, effort = "moderate") => {
|
|
38
|
+
issues.push({ tier, severity, issue, remediation, effort });
|
|
39
|
+
};
|
|
40
|
+
// Tier 1: Server Implementation (25%)
|
|
41
|
+
const tier1Checks = await runTier1ServerImplementation(serverUrl, baseUrl, timeout, options, addIssue);
|
|
42
|
+
serverResponded = tier1Checks.some(c => c.id === "server_responds" && c.passed);
|
|
43
|
+
const versionCheck = tier1Checks.find(c => c.id === "protocol_version");
|
|
44
|
+
if (versionCheck?.evidence) {
|
|
45
|
+
protocolVersion = versionCheck.evidence;
|
|
46
|
+
}
|
|
47
|
+
tiers.push(createTierResult(1, tier1Checks));
|
|
48
|
+
// Tier 2: Tool Discoverability (20%)
|
|
49
|
+
const tier2Result = await runTier2ToolDiscoverability(serverUrl, timeout, options, addIssue);
|
|
50
|
+
toolCount = tier2Result.toolCount;
|
|
51
|
+
toolList = tier2Result.tools;
|
|
52
|
+
tiers.push(createTierResult(2, tier2Result.checks));
|
|
53
|
+
// Tier 3: Instrumentation (15%)
|
|
54
|
+
const tier3Checks = await runTier3Instrumentation(baseUrl, timeout, options, addIssue);
|
|
55
|
+
tiers.push(createTierResult(3, tier3Checks));
|
|
56
|
+
// Tier 4: Consistency (15%)
|
|
57
|
+
const tier4Checks = await runTier4Consistency(serverUrl, toolList, timeout, options, addIssue);
|
|
58
|
+
tiers.push(createTierResult(4, tier4Checks));
|
|
59
|
+
// Tier 5: Agent Optimizations (15%)
|
|
60
|
+
const tier5Checks = await runTier5AgentOptimizations(toolList, options, addIssue);
|
|
61
|
+
tiers.push(createTierResult(5, tier5Checks));
|
|
62
|
+
// Tier 6: Documentation (10%)
|
|
63
|
+
const tier6Checks = await runTier6Documentation(baseUrl, timeout, options, addIssue);
|
|
64
|
+
tiers.push(createTierResult(6, tier6Checks));
|
|
65
|
+
// Calculate overall score
|
|
66
|
+
let overallScore = 0;
|
|
67
|
+
for (const tier of tiers) {
|
|
68
|
+
overallScore += tier.score * tier.weight;
|
|
69
|
+
}
|
|
70
|
+
overallScore = Math.round(overallScore);
|
|
71
|
+
// Determine grade
|
|
72
|
+
const grade = scoreToGrade(overallScore);
|
|
73
|
+
// Sort issues by severity
|
|
74
|
+
const severityOrder = {
|
|
75
|
+
critical: 0,
|
|
76
|
+
high: 1,
|
|
77
|
+
medium: 2,
|
|
78
|
+
low: 3,
|
|
79
|
+
};
|
|
80
|
+
issues.sort((a, b) => severityOrder[a.severity] - severityOrder[b.severity]);
|
|
81
|
+
// Generate prioritized recommendations
|
|
82
|
+
const recommendations = generateRecommendations(issues, tiers);
|
|
83
|
+
// Count checks
|
|
84
|
+
const totalChecks = tiers.reduce((sum, t) => sum + t.checks.length, 0);
|
|
85
|
+
const passedChecks = tiers.reduce((sum, t) => sum + t.checks.filter(c => c.passed).length, 0);
|
|
86
|
+
return {
|
|
87
|
+
url: serverUrl,
|
|
88
|
+
timestamp: new Date().toISOString(),
|
|
89
|
+
score: overallScore,
|
|
90
|
+
grade,
|
|
91
|
+
tiers,
|
|
92
|
+
issues,
|
|
93
|
+
recommendations,
|
|
94
|
+
duration: Date.now() - startTime,
|
|
95
|
+
summary: {
|
|
96
|
+
totalChecks,
|
|
97
|
+
passedChecks,
|
|
98
|
+
criticalIssues: issues.filter(i => i.severity === "critical").length,
|
|
99
|
+
highIssues: issues.filter(i => i.severity === "high").length,
|
|
100
|
+
serverResponded,
|
|
101
|
+
protocolVersion,
|
|
102
|
+
toolCount,
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Create a tier result from checks
|
|
108
|
+
*/
|
|
109
|
+
function createTierResult(tier, checks) {
|
|
110
|
+
const totalScore = checks.reduce((sum, c) => sum + c.score, 0);
|
|
111
|
+
const maxScore = checks.reduce((sum, c) => sum + c.maxScore, 0);
|
|
112
|
+
const score = maxScore > 0 ? Math.round((totalScore / maxScore) * 100) : 0;
|
|
113
|
+
return {
|
|
114
|
+
tier,
|
|
115
|
+
name: TIER_CONFIG[tier].name,
|
|
116
|
+
score,
|
|
117
|
+
weight: TIER_CONFIG[tier].weight,
|
|
118
|
+
checks,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Convert score to letter grade
|
|
123
|
+
*/
|
|
124
|
+
function scoreToGrade(score) {
|
|
125
|
+
if (score >= 90)
|
|
126
|
+
return "A";
|
|
127
|
+
if (score >= 80)
|
|
128
|
+
return "B";
|
|
129
|
+
if (score >= 70)
|
|
130
|
+
return "C";
|
|
131
|
+
if (score >= 60)
|
|
132
|
+
return "D";
|
|
133
|
+
return "F";
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Tier 1: Server Implementation (25%)
|
|
137
|
+
* - Protocol version, tool definitions, error handling, timeouts
|
|
138
|
+
*/
|
|
139
|
+
async function runTier1ServerImplementation(serverUrl, baseUrl, timeout, options, addIssue) {
|
|
140
|
+
const checks = [];
|
|
141
|
+
// Check 1.1: Server responds
|
|
142
|
+
let serverResponds = false;
|
|
143
|
+
let responseTime = 0;
|
|
144
|
+
try {
|
|
145
|
+
const controller = new AbortController();
|
|
146
|
+
const timeoutId = setTimeout(() => controller.abort(), timeout);
|
|
147
|
+
const start = Date.now();
|
|
148
|
+
const headers = {
|
|
149
|
+
"Content-Type": "application/json",
|
|
150
|
+
};
|
|
151
|
+
if (options.apiKey) {
|
|
152
|
+
headers["Authorization"] = `Bearer ${options.apiKey}`;
|
|
153
|
+
}
|
|
154
|
+
else if (options.oauthToken) {
|
|
155
|
+
headers["Authorization"] = `Bearer ${options.oauthToken}`;
|
|
156
|
+
}
|
|
157
|
+
// Send MCP initialize request
|
|
158
|
+
const response = await fetch(serverUrl, {
|
|
159
|
+
method: "POST",
|
|
160
|
+
headers,
|
|
161
|
+
body: JSON.stringify({
|
|
162
|
+
jsonrpc: "2.0",
|
|
163
|
+
id: 1,
|
|
164
|
+
method: "initialize",
|
|
165
|
+
params: {
|
|
166
|
+
protocolVersion: "2024-11-05",
|
|
167
|
+
capabilities: {},
|
|
168
|
+
clientInfo: { name: "cbrowser-audit", version: "1.0.0" },
|
|
169
|
+
},
|
|
170
|
+
}),
|
|
171
|
+
signal: controller.signal,
|
|
172
|
+
});
|
|
173
|
+
clearTimeout(timeoutId);
|
|
174
|
+
responseTime = Date.now() - start;
|
|
175
|
+
serverResponds = response.ok || response.status === 401 || response.status === 403;
|
|
176
|
+
if (response.ok) {
|
|
177
|
+
checks.push({
|
|
178
|
+
id: "server_responds",
|
|
179
|
+
name: "Server responds to MCP requests",
|
|
180
|
+
passed: true,
|
|
181
|
+
score: 1,
|
|
182
|
+
maxScore: 1,
|
|
183
|
+
details: `Server responded in ${responseTime}ms`,
|
|
184
|
+
evidence: `HTTP ${response.status}`,
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
else if (response.status === 401 || response.status === 403) {
|
|
188
|
+
checks.push({
|
|
189
|
+
id: "server_responds",
|
|
190
|
+
name: "Server responds to MCP requests",
|
|
191
|
+
passed: true,
|
|
192
|
+
score: 0.8,
|
|
193
|
+
maxScore: 1,
|
|
194
|
+
details: `Server requires authentication (${response.status})`,
|
|
195
|
+
evidence: `HTTP ${response.status}`,
|
|
196
|
+
});
|
|
197
|
+
addIssue(1, "medium", "Server requires authentication", "Provide API key or OAuth token in options", "quick");
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
checks.push({
|
|
201
|
+
id: "server_responds",
|
|
202
|
+
name: "Server responds to MCP requests",
|
|
203
|
+
passed: false,
|
|
204
|
+
score: 0,
|
|
205
|
+
maxScore: 1,
|
|
206
|
+
details: `Server returned error: HTTP ${response.status}`,
|
|
207
|
+
evidence: `HTTP ${response.status}`,
|
|
208
|
+
});
|
|
209
|
+
addIssue(1, "critical", "Server not responding correctly", "Verify MCP endpoint is accessible and returns 200 OK", "significant");
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
catch (error) {
|
|
213
|
+
checks.push({
|
|
214
|
+
id: "server_responds",
|
|
215
|
+
name: "Server responds to MCP requests",
|
|
216
|
+
passed: false,
|
|
217
|
+
score: 0,
|
|
218
|
+
maxScore: 1,
|
|
219
|
+
details: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
220
|
+
});
|
|
221
|
+
addIssue(1, "critical", "Cannot connect to MCP server", "Verify server is running and URL is correct", "significant");
|
|
222
|
+
}
|
|
223
|
+
// Check 1.2: Protocol version
|
|
224
|
+
try {
|
|
225
|
+
const headers = { "Content-Type": "application/json" };
|
|
226
|
+
if (options.apiKey)
|
|
227
|
+
headers["Authorization"] = `Bearer ${options.apiKey}`;
|
|
228
|
+
else if (options.oauthToken)
|
|
229
|
+
headers["Authorization"] = `Bearer ${options.oauthToken}`;
|
|
230
|
+
const response = await fetch(serverUrl, {
|
|
231
|
+
method: "POST",
|
|
232
|
+
headers,
|
|
233
|
+
body: JSON.stringify({
|
|
234
|
+
jsonrpc: "2.0",
|
|
235
|
+
id: 1,
|
|
236
|
+
method: "initialize",
|
|
237
|
+
params: {
|
|
238
|
+
protocolVersion: "2024-11-05",
|
|
239
|
+
capabilities: {},
|
|
240
|
+
clientInfo: { name: "cbrowser-audit", version: "1.0.0" },
|
|
241
|
+
},
|
|
242
|
+
}),
|
|
243
|
+
});
|
|
244
|
+
if (response.ok) {
|
|
245
|
+
const data = await response.json();
|
|
246
|
+
const serverVersion = data?.result?.protocolVersion;
|
|
247
|
+
const isLatest = serverVersion === "2024-11-05";
|
|
248
|
+
checks.push({
|
|
249
|
+
id: "protocol_version",
|
|
250
|
+
name: "Uses latest MCP protocol version",
|
|
251
|
+
passed: isLatest,
|
|
252
|
+
score: isLatest ? 1 : 0.5,
|
|
253
|
+
maxScore: 1,
|
|
254
|
+
details: isLatest
|
|
255
|
+
? "Using latest protocol version 2024-11-05"
|
|
256
|
+
: `Using older protocol version: ${serverVersion || "unknown"}`,
|
|
257
|
+
evidence: serverVersion || "unknown",
|
|
258
|
+
});
|
|
259
|
+
if (!isLatest && serverVersion) {
|
|
260
|
+
addIssue(1, "medium", `Protocol version ${serverVersion} is not the latest`, "Upgrade to protocol version 2024-11-05", "moderate");
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
else {
|
|
264
|
+
checks.push({
|
|
265
|
+
id: "protocol_version",
|
|
266
|
+
name: "Uses latest MCP protocol version",
|
|
267
|
+
passed: false,
|
|
268
|
+
score: 0,
|
|
269
|
+
maxScore: 1,
|
|
270
|
+
details: "Could not determine protocol version",
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
catch {
|
|
275
|
+
checks.push({
|
|
276
|
+
id: "protocol_version",
|
|
277
|
+
name: "Uses latest MCP protocol version",
|
|
278
|
+
passed: false,
|
|
279
|
+
score: 0,
|
|
280
|
+
maxScore: 1,
|
|
281
|
+
details: "Failed to check protocol version",
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
// Check 1.3: Error handling
|
|
285
|
+
try {
|
|
286
|
+
const headers = { "Content-Type": "application/json" };
|
|
287
|
+
if (options.apiKey)
|
|
288
|
+
headers["Authorization"] = `Bearer ${options.apiKey}`;
|
|
289
|
+
else if (options.oauthToken)
|
|
290
|
+
headers["Authorization"] = `Bearer ${options.oauthToken}`;
|
|
291
|
+
// Send invalid request to test error handling
|
|
292
|
+
const response = await fetch(serverUrl, {
|
|
293
|
+
method: "POST",
|
|
294
|
+
headers,
|
|
295
|
+
body: JSON.stringify({
|
|
296
|
+
jsonrpc: "2.0",
|
|
297
|
+
id: 1,
|
|
298
|
+
method: "nonexistent_method_xyz",
|
|
299
|
+
params: {},
|
|
300
|
+
}),
|
|
301
|
+
});
|
|
302
|
+
if (response.ok) {
|
|
303
|
+
const data = await response.json();
|
|
304
|
+
const hasError = data?.error !== undefined;
|
|
305
|
+
const hasErrorCode = data?.error?.code !== undefined;
|
|
306
|
+
const hasErrorMessage = data?.error?.message !== undefined;
|
|
307
|
+
checks.push({
|
|
308
|
+
id: "error_handling",
|
|
309
|
+
name: "Returns proper JSON-RPC errors",
|
|
310
|
+
passed: hasError && hasErrorCode && hasErrorMessage,
|
|
311
|
+
score: hasError ? (hasErrorCode && hasErrorMessage ? 1 : 0.5) : 0,
|
|
312
|
+
maxScore: 1,
|
|
313
|
+
details: hasError
|
|
314
|
+
? `Returns error with code ${data.error.code}: ${data.error.message}`
|
|
315
|
+
: "Does not return proper error for invalid method",
|
|
316
|
+
evidence: hasError ? `Code: ${data.error.code}` : undefined,
|
|
317
|
+
});
|
|
318
|
+
if (!hasError) {
|
|
319
|
+
addIssue(1, "high", "Server does not return proper errors for invalid methods", "Implement JSON-RPC error handling per MCP spec", "moderate");
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
else {
|
|
323
|
+
// HTTP error is also acceptable for invalid requests
|
|
324
|
+
checks.push({
|
|
325
|
+
id: "error_handling",
|
|
326
|
+
name: "Returns proper JSON-RPC errors",
|
|
327
|
+
passed: true,
|
|
328
|
+
score: 0.8,
|
|
329
|
+
maxScore: 1,
|
|
330
|
+
details: `Returns HTTP ${response.status} for invalid method`,
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
catch {
|
|
335
|
+
checks.push({
|
|
336
|
+
id: "error_handling",
|
|
337
|
+
name: "Returns proper JSON-RPC errors",
|
|
338
|
+
passed: false,
|
|
339
|
+
score: 0,
|
|
340
|
+
maxScore: 1,
|
|
341
|
+
details: "Failed to test error handling",
|
|
342
|
+
});
|
|
343
|
+
}
|
|
344
|
+
// Check 1.4: Response timeout behavior
|
|
345
|
+
checks.push({
|
|
346
|
+
id: "response_time",
|
|
347
|
+
name: "Responds within acceptable time",
|
|
348
|
+
passed: responseTime < 5000,
|
|
349
|
+
score: responseTime < 1000 ? 1 : responseTime < 5000 ? 0.7 : 0.3,
|
|
350
|
+
maxScore: 1,
|
|
351
|
+
details: responseTime > 0
|
|
352
|
+
? `Response time: ${responseTime}ms`
|
|
353
|
+
: "Could not measure response time",
|
|
354
|
+
evidence: responseTime > 0 ? `${responseTime}ms` : undefined,
|
|
355
|
+
});
|
|
356
|
+
if (responseTime > 5000) {
|
|
357
|
+
addIssue(1, "medium", `Slow response time: ${responseTime}ms`, "Optimize server startup and response handling", "moderate");
|
|
358
|
+
}
|
|
359
|
+
return checks;
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Tier 2: Tool Discoverability (20%)
|
|
363
|
+
* - Schema completeness, descriptions, input validation
|
|
364
|
+
*/
|
|
365
|
+
async function runTier2ToolDiscoverability(serverUrl, timeout, options, addIssue) {
|
|
366
|
+
const checks = [];
|
|
367
|
+
let tools = [];
|
|
368
|
+
let toolCount;
|
|
369
|
+
try {
|
|
370
|
+
const headers = { "Content-Type": "application/json" };
|
|
371
|
+
if (options.apiKey)
|
|
372
|
+
headers["Authorization"] = `Bearer ${options.apiKey}`;
|
|
373
|
+
else if (options.oauthToken)
|
|
374
|
+
headers["Authorization"] = `Bearer ${options.oauthToken}`;
|
|
375
|
+
// Initialize first
|
|
376
|
+
await fetch(serverUrl, {
|
|
377
|
+
method: "POST",
|
|
378
|
+
headers,
|
|
379
|
+
body: JSON.stringify({
|
|
380
|
+
jsonrpc: "2.0",
|
|
381
|
+
id: 1,
|
|
382
|
+
method: "initialize",
|
|
383
|
+
params: {
|
|
384
|
+
protocolVersion: "2024-11-05",
|
|
385
|
+
capabilities: {},
|
|
386
|
+
clientInfo: { name: "cbrowser-audit", version: "1.0.0" },
|
|
387
|
+
},
|
|
388
|
+
}),
|
|
389
|
+
});
|
|
390
|
+
// List tools
|
|
391
|
+
const response = await fetch(serverUrl, {
|
|
392
|
+
method: "POST",
|
|
393
|
+
headers,
|
|
394
|
+
body: JSON.stringify({
|
|
395
|
+
jsonrpc: "2.0",
|
|
396
|
+
id: 2,
|
|
397
|
+
method: "tools/list",
|
|
398
|
+
params: {},
|
|
399
|
+
}),
|
|
400
|
+
});
|
|
401
|
+
if (response.ok) {
|
|
402
|
+
const data = await response.json();
|
|
403
|
+
tools = data?.result?.tools || [];
|
|
404
|
+
toolCount = tools.length;
|
|
405
|
+
// Check 2.1: Tools are exposed
|
|
406
|
+
checks.push({
|
|
407
|
+
id: "tools_exposed",
|
|
408
|
+
name: "Exposes tools via tools/list",
|
|
409
|
+
passed: toolCount > 0,
|
|
410
|
+
score: toolCount > 0 ? 1 : 0,
|
|
411
|
+
maxScore: 1,
|
|
412
|
+
details: `Found ${toolCount} tools`,
|
|
413
|
+
evidence: `${toolCount} tools`,
|
|
414
|
+
});
|
|
415
|
+
if (toolCount === 0) {
|
|
416
|
+
addIssue(2, "critical", "No tools exposed", "Implement and register at least one MCP tool", "significant");
|
|
417
|
+
}
|
|
418
|
+
// Check 2.2: Tool descriptions
|
|
419
|
+
const toolsWithDesc = tools.filter((t) => t.description && t.description.length > 10);
|
|
420
|
+
const descRatio = toolCount > 0 ? toolsWithDesc.length / toolCount : 0;
|
|
421
|
+
checks.push({
|
|
422
|
+
id: "tool_descriptions",
|
|
423
|
+
name: "Tools have descriptive descriptions",
|
|
424
|
+
passed: descRatio >= 0.9,
|
|
425
|
+
score: descRatio,
|
|
426
|
+
maxScore: 1,
|
|
427
|
+
details: `${toolsWithDesc.length}/${toolCount} tools have descriptions > 10 chars`,
|
|
428
|
+
evidence: `${Math.round(descRatio * 100)}%`,
|
|
429
|
+
});
|
|
430
|
+
if (descRatio < 0.9) {
|
|
431
|
+
addIssue(2, "medium", `${toolCount - toolsWithDesc.length} tools lack good descriptions`, "Add descriptive help text to all tools (>10 chars)", "quick");
|
|
432
|
+
}
|
|
433
|
+
// Check 2.3: Input schemas
|
|
434
|
+
const toolsWithSchema = tools.filter((t) => t.inputSchema && Object.keys(t.inputSchema).length > 0);
|
|
435
|
+
const schemaRatio = toolCount > 0 ? toolsWithSchema.length / toolCount : 0;
|
|
436
|
+
checks.push({
|
|
437
|
+
id: "input_schemas",
|
|
438
|
+
name: "Tools have input schemas",
|
|
439
|
+
passed: schemaRatio >= 0.9,
|
|
440
|
+
score: schemaRatio,
|
|
441
|
+
maxScore: 1,
|
|
442
|
+
details: `${toolsWithSchema.length}/${toolCount} tools have input schemas`,
|
|
443
|
+
evidence: `${Math.round(schemaRatio * 100)}%`,
|
|
444
|
+
});
|
|
445
|
+
if (schemaRatio < 0.9) {
|
|
446
|
+
addIssue(2, "high", `${toolCount - toolsWithSchema.length} tools lack input schemas`, "Define JSON Schema for all tool inputs", "moderate");
|
|
447
|
+
}
|
|
448
|
+
// Check 2.4: Schema property descriptions
|
|
449
|
+
let propsWithDesc = 0;
|
|
450
|
+
let totalProps = 0;
|
|
451
|
+
for (const tool of tools) {
|
|
452
|
+
const props = tool.inputSchema?.properties || {};
|
|
453
|
+
for (const key of Object.keys(props)) {
|
|
454
|
+
totalProps++;
|
|
455
|
+
if (props[key].description)
|
|
456
|
+
propsWithDesc++;
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
const propDescRatio = totalProps > 0 ? propsWithDesc / totalProps : 1;
|
|
460
|
+
checks.push({
|
|
461
|
+
id: "property_descriptions",
|
|
462
|
+
name: "Schema properties have descriptions",
|
|
463
|
+
passed: propDescRatio >= 0.8,
|
|
464
|
+
score: propDescRatio,
|
|
465
|
+
maxScore: 1,
|
|
466
|
+
details: `${propsWithDesc}/${totalProps} properties have descriptions`,
|
|
467
|
+
evidence: `${Math.round(propDescRatio * 100)}%`,
|
|
468
|
+
});
|
|
469
|
+
if (propDescRatio < 0.8) {
|
|
470
|
+
addIssue(2, "low", `${totalProps - propsWithDesc} properties lack descriptions`, "Add descriptions to all schema properties", "quick");
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
else {
|
|
474
|
+
checks.push({
|
|
475
|
+
id: "tools_exposed",
|
|
476
|
+
name: "Exposes tools via tools/list",
|
|
477
|
+
passed: false,
|
|
478
|
+
score: 0,
|
|
479
|
+
maxScore: 1,
|
|
480
|
+
details: `tools/list failed: HTTP ${response.status}`,
|
|
481
|
+
});
|
|
482
|
+
addIssue(2, "critical", "tools/list endpoint not working", "Implement tools/list method per MCP spec", "significant");
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
catch (error) {
|
|
486
|
+
checks.push({
|
|
487
|
+
id: "tools_exposed",
|
|
488
|
+
name: "Exposes tools via tools/list",
|
|
489
|
+
passed: false,
|
|
490
|
+
score: 0,
|
|
491
|
+
maxScore: 1,
|
|
492
|
+
details: `Failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
return { checks, toolCount, tools };
|
|
496
|
+
}
|
|
497
|
+
/**
|
|
498
|
+
* Tier 3: Instrumentation (15%)
|
|
499
|
+
* - Execution timing, error logging, health endpoint
|
|
500
|
+
*/
|
|
501
|
+
async function runTier3Instrumentation(baseUrl, timeout, options, addIssue) {
|
|
502
|
+
const checks = [];
|
|
503
|
+
// Check 3.1: Health endpoint
|
|
504
|
+
try {
|
|
505
|
+
const response = await fetch(`${baseUrl}/health`, { method: "GET" });
|
|
506
|
+
const hasHealth = response.ok;
|
|
507
|
+
checks.push({
|
|
508
|
+
id: "health_endpoint",
|
|
509
|
+
name: "Exposes /health endpoint",
|
|
510
|
+
passed: hasHealth,
|
|
511
|
+
score: hasHealth ? 1 : 0,
|
|
512
|
+
maxScore: 1,
|
|
513
|
+
details: hasHealth
|
|
514
|
+
? "Health endpoint available"
|
|
515
|
+
: `Health endpoint returned ${response.status}`,
|
|
516
|
+
});
|
|
517
|
+
if (!hasHealth) {
|
|
518
|
+
addIssue(3, "medium", "No /health endpoint", "Add GET /health endpoint for monitoring", "quick");
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
catch {
|
|
522
|
+
checks.push({
|
|
523
|
+
id: "health_endpoint",
|
|
524
|
+
name: "Exposes /health endpoint",
|
|
525
|
+
passed: false,
|
|
526
|
+
score: 0,
|
|
527
|
+
maxScore: 1,
|
|
528
|
+
details: "Health endpoint not accessible",
|
|
529
|
+
});
|
|
530
|
+
addIssue(3, "medium", "No /health endpoint", "Add GET /health endpoint for monitoring", "quick");
|
|
531
|
+
}
|
|
532
|
+
// Check 3.2: Info endpoint
|
|
533
|
+
try {
|
|
534
|
+
const response = await fetch(`${baseUrl}/info`, { method: "GET" });
|
|
535
|
+
if (response.ok) {
|
|
536
|
+
const data = await response.json();
|
|
537
|
+
const hasVersion = data.version !== undefined;
|
|
538
|
+
const hasName = data.name !== undefined;
|
|
539
|
+
checks.push({
|
|
540
|
+
id: "info_endpoint",
|
|
541
|
+
name: "Exposes /info endpoint with metadata",
|
|
542
|
+
passed: hasVersion && hasName,
|
|
543
|
+
score: hasVersion && hasName ? 1 : hasVersion || hasName ? 0.5 : 0,
|
|
544
|
+
maxScore: 1,
|
|
545
|
+
details: hasVersion && hasName
|
|
546
|
+
? `Name: ${data.name}, Version: ${data.version}`
|
|
547
|
+
: "Incomplete info response",
|
|
548
|
+
evidence: hasVersion ? data.version : undefined,
|
|
549
|
+
});
|
|
550
|
+
}
|
|
551
|
+
else {
|
|
552
|
+
checks.push({
|
|
553
|
+
id: "info_endpoint",
|
|
554
|
+
name: "Exposes /info endpoint with metadata",
|
|
555
|
+
passed: false,
|
|
556
|
+
score: 0,
|
|
557
|
+
maxScore: 1,
|
|
558
|
+
details: `Info endpoint returned ${response.status}`,
|
|
559
|
+
});
|
|
560
|
+
addIssue(3, "low", "No /info endpoint", "Add GET /info endpoint with name and version", "quick");
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
catch {
|
|
564
|
+
checks.push({
|
|
565
|
+
id: "info_endpoint",
|
|
566
|
+
name: "Exposes /info endpoint with metadata",
|
|
567
|
+
passed: false,
|
|
568
|
+
score: 0,
|
|
569
|
+
maxScore: 1,
|
|
570
|
+
details: "Info endpoint not accessible",
|
|
571
|
+
});
|
|
572
|
+
addIssue(3, "low", "No /info endpoint", "Add GET /info endpoint with name and version", "quick");
|
|
573
|
+
}
|
|
574
|
+
// Check 3.3: OAuth metadata (for Claude.ai compatibility)
|
|
575
|
+
try {
|
|
576
|
+
const response = await fetch(`${baseUrl}/.well-known/oauth-protected-resource`, {
|
|
577
|
+
method: "GET",
|
|
578
|
+
});
|
|
579
|
+
if (response.ok) {
|
|
580
|
+
const data = await response.json();
|
|
581
|
+
const hasResource = data.resource !== undefined;
|
|
582
|
+
const hasAuthServer = data.authorization_servers !== undefined;
|
|
583
|
+
checks.push({
|
|
584
|
+
id: "oauth_metadata",
|
|
585
|
+
name: "OAuth Protected Resource metadata",
|
|
586
|
+
passed: hasResource && hasAuthServer,
|
|
587
|
+
score: hasResource && hasAuthServer ? 1 : 0.5,
|
|
588
|
+
maxScore: 1,
|
|
589
|
+
details: hasResource && hasAuthServer
|
|
590
|
+
? "OAuth metadata properly configured"
|
|
591
|
+
: "Partial OAuth metadata",
|
|
592
|
+
evidence: data.resource,
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
else {
|
|
596
|
+
checks.push({
|
|
597
|
+
id: "oauth_metadata",
|
|
598
|
+
name: "OAuth Protected Resource metadata",
|
|
599
|
+
passed: false,
|
|
600
|
+
score: 0,
|
|
601
|
+
maxScore: 1,
|
|
602
|
+
details: "No OAuth metadata (optional for non-Claude.ai use)",
|
|
603
|
+
});
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
catch {
|
|
607
|
+
checks.push({
|
|
608
|
+
id: "oauth_metadata",
|
|
609
|
+
name: "OAuth Protected Resource metadata",
|
|
610
|
+
passed: false,
|
|
611
|
+
score: 0,
|
|
612
|
+
maxScore: 1,
|
|
613
|
+
details: "No OAuth metadata (optional for non-Claude.ai use)",
|
|
614
|
+
});
|
|
615
|
+
}
|
|
616
|
+
return checks;
|
|
617
|
+
}
|
|
618
|
+
/**
|
|
619
|
+
* Tier 4: Consistency (15%)
|
|
620
|
+
* - Selector healing, idempotency, state management
|
|
621
|
+
*/
|
|
622
|
+
async function runTier4Consistency(serverUrl, tools, timeout, options, addIssue) {
|
|
623
|
+
const checks = [];
|
|
624
|
+
// Check 4.1: Has session management tools
|
|
625
|
+
const sessionTools = tools.filter(t => t.name?.includes("session") ||
|
|
626
|
+
t.name?.includes("state") ||
|
|
627
|
+
t.description?.toLowerCase().includes("session"));
|
|
628
|
+
const hasSessionMgmt = sessionTools.length > 0;
|
|
629
|
+
checks.push({
|
|
630
|
+
id: "session_management",
|
|
631
|
+
name: "Provides session management tools",
|
|
632
|
+
passed: hasSessionMgmt,
|
|
633
|
+
score: hasSessionMgmt ? 1 : 0,
|
|
634
|
+
maxScore: 1,
|
|
635
|
+
details: hasSessionMgmt
|
|
636
|
+
? `Found ${sessionTools.length} session-related tools`
|
|
637
|
+
: "No session management tools found",
|
|
638
|
+
evidence: hasSessionMgmt
|
|
639
|
+
? sessionTools.map((t) => t.name).join(", ")
|
|
640
|
+
: undefined,
|
|
641
|
+
});
|
|
642
|
+
if (!hasSessionMgmt) {
|
|
643
|
+
addIssue(4, "medium", "No session management", "Add session save/load tools for state persistence", "moderate");
|
|
644
|
+
}
|
|
645
|
+
// Check 4.2: Has healing/recovery tools
|
|
646
|
+
const healingTools = tools.filter(t => t.name?.includes("heal") ||
|
|
647
|
+
t.name?.includes("recover") ||
|
|
648
|
+
t.name?.includes("retry") ||
|
|
649
|
+
t.description?.toLowerCase().includes("healing") ||
|
|
650
|
+
t.description?.toLowerCase().includes("self-healing"));
|
|
651
|
+
const hasHealing = healingTools.length > 0;
|
|
652
|
+
checks.push({
|
|
653
|
+
id: "self_healing",
|
|
654
|
+
name: "Provides self-healing/recovery tools",
|
|
655
|
+
passed: hasHealing,
|
|
656
|
+
score: hasHealing ? 1 : 0,
|
|
657
|
+
maxScore: 1,
|
|
658
|
+
details: hasHealing
|
|
659
|
+
? `Found ${healingTools.length} healing-related tools`
|
|
660
|
+
: "No self-healing tools found",
|
|
661
|
+
evidence: hasHealing
|
|
662
|
+
? healingTools.map((t) => t.name).join(", ")
|
|
663
|
+
: undefined,
|
|
664
|
+
});
|
|
665
|
+
if (!hasHealing) {
|
|
666
|
+
addIssue(4, "low", "No self-healing capabilities", "Consider adding selector healing or retry tools", "moderate");
|
|
667
|
+
}
|
|
668
|
+
// Check 4.3: Consistent error structure
|
|
669
|
+
// (Inferred from earlier checks - this is a meta-check)
|
|
670
|
+
checks.push({
|
|
671
|
+
id: "consistent_responses",
|
|
672
|
+
name: "Uses consistent response structure",
|
|
673
|
+
passed: true, // Assume true if server responds at all
|
|
674
|
+
score: 1,
|
|
675
|
+
maxScore: 1,
|
|
676
|
+
details: "JSON-RPC response structure detected",
|
|
677
|
+
});
|
|
678
|
+
return checks;
|
|
679
|
+
}
|
|
680
|
+
/**
|
|
681
|
+
* Tier 5: Agent Optimizations (15%)
|
|
682
|
+
* - Vision mode, NL parsing, persona support
|
|
683
|
+
*/
|
|
684
|
+
async function runTier5AgentOptimizations(tools, options, addIssue) {
|
|
685
|
+
const checks = [];
|
|
686
|
+
const toolNames = tools.map(t => t.name?.toLowerCase() || "");
|
|
687
|
+
const toolDescs = tools.map(t => t.description?.toLowerCase() || "");
|
|
688
|
+
const combined = [...toolNames, ...toolDescs].join(" ");
|
|
689
|
+
// Check 5.1: Vision/screenshot support
|
|
690
|
+
const hasVision = combined.includes("screenshot") ||
|
|
691
|
+
combined.includes("vision") ||
|
|
692
|
+
combined.includes("visual");
|
|
693
|
+
checks.push({
|
|
694
|
+
id: "vision_support",
|
|
695
|
+
name: "Supports visual/screenshot capabilities",
|
|
696
|
+
passed: hasVision,
|
|
697
|
+
score: hasVision ? 1 : 0,
|
|
698
|
+
maxScore: 1,
|
|
699
|
+
details: hasVision
|
|
700
|
+
? "Visual capabilities available"
|
|
701
|
+
: "No visual/screenshot tools found",
|
|
702
|
+
});
|
|
703
|
+
if (!hasVision) {
|
|
704
|
+
addIssue(5, "medium", "No vision/screenshot support", "Add screenshot tool for visual verification", "moderate");
|
|
705
|
+
}
|
|
706
|
+
// Check 5.2: Natural language support
|
|
707
|
+
const hasNL = combined.includes("natural language") ||
|
|
708
|
+
combined.includes("intent") ||
|
|
709
|
+
combined.includes("smart") ||
|
|
710
|
+
combined.includes("ai-");
|
|
711
|
+
checks.push({
|
|
712
|
+
id: "nl_support",
|
|
713
|
+
name: "Supports natural language inputs",
|
|
714
|
+
passed: hasNL,
|
|
715
|
+
score: hasNL ? 1 : 0,
|
|
716
|
+
maxScore: 1,
|
|
717
|
+
details: hasNL
|
|
718
|
+
? "Natural language capabilities available"
|
|
719
|
+
: "No natural language tools found",
|
|
720
|
+
});
|
|
721
|
+
// Check 5.3: Persona support
|
|
722
|
+
const hasPersona = combined.includes("persona") ||
|
|
723
|
+
combined.includes("cognitive") ||
|
|
724
|
+
combined.includes("journey");
|
|
725
|
+
checks.push({
|
|
726
|
+
id: "persona_support",
|
|
727
|
+
name: "Supports persona-based testing",
|
|
728
|
+
passed: hasPersona,
|
|
729
|
+
score: hasPersona ? 1 : 0,
|
|
730
|
+
maxScore: 1,
|
|
731
|
+
details: hasPersona
|
|
732
|
+
? "Persona capabilities available"
|
|
733
|
+
: "No persona tools found",
|
|
734
|
+
});
|
|
735
|
+
// Check 5.4: Accessibility support
|
|
736
|
+
const hasA11y = combined.includes("accessibility") ||
|
|
737
|
+
combined.includes("a11y") ||
|
|
738
|
+
combined.includes("wcag") ||
|
|
739
|
+
combined.includes("empathy");
|
|
740
|
+
checks.push({
|
|
741
|
+
id: "accessibility_support",
|
|
742
|
+
name: "Supports accessibility testing",
|
|
743
|
+
passed: hasA11y,
|
|
744
|
+
score: hasA11y ? 1 : 0,
|
|
745
|
+
maxScore: 1,
|
|
746
|
+
details: hasA11y
|
|
747
|
+
? "Accessibility testing available"
|
|
748
|
+
: "No accessibility tools found",
|
|
749
|
+
});
|
|
750
|
+
return checks;
|
|
751
|
+
}
|
|
752
|
+
/**
|
|
753
|
+
* Tier 6: Documentation (10%)
|
|
754
|
+
* - /webmcp.txt, examples, troubleshooting
|
|
755
|
+
*/
|
|
756
|
+
async function runTier6Documentation(baseUrl, timeout, options, addIssue) {
|
|
757
|
+
const checks = [];
|
|
758
|
+
// Check 6.1: /llms.txt or /webmcp.txt
|
|
759
|
+
let hasLlmsTxt = false;
|
|
760
|
+
let llmsContent = "";
|
|
761
|
+
for (const path of ["/llms.txt", "/webmcp.txt", "/.well-known/llms.txt"]) {
|
|
762
|
+
try {
|
|
763
|
+
const response = await fetch(`${baseUrl}${path}`, { method: "GET" });
|
|
764
|
+
if (response.ok) {
|
|
765
|
+
llmsContent = await response.text();
|
|
766
|
+
hasLlmsTxt = llmsContent.length > 50;
|
|
767
|
+
if (hasLlmsTxt)
|
|
768
|
+
break;
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
catch {
|
|
772
|
+
// Try next path
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
checks.push({
|
|
776
|
+
id: "llms_txt",
|
|
777
|
+
name: "Provides /llms.txt or /webmcp.txt",
|
|
778
|
+
passed: hasLlmsTxt,
|
|
779
|
+
score: hasLlmsTxt ? 1 : 0,
|
|
780
|
+
maxScore: 1,
|
|
781
|
+
details: hasLlmsTxt
|
|
782
|
+
? `Found AI documentation (${llmsContent.length} chars)`
|
|
783
|
+
: "No /llms.txt or /webmcp.txt found",
|
|
784
|
+
});
|
|
785
|
+
if (!hasLlmsTxt) {
|
|
786
|
+
addIssue(6, "medium", "No /llms.txt file", "Create /llms.txt with AI-readable server documentation", "quick");
|
|
787
|
+
}
|
|
788
|
+
// Check 6.2: README or docs endpoint
|
|
789
|
+
let hasReadme = false;
|
|
790
|
+
try {
|
|
791
|
+
const response = await fetch(`${baseUrl}/docs`, { method: "GET" });
|
|
792
|
+
hasReadme = response.ok;
|
|
793
|
+
}
|
|
794
|
+
catch {
|
|
795
|
+
// No docs endpoint
|
|
796
|
+
}
|
|
797
|
+
if (!hasReadme) {
|
|
798
|
+
try {
|
|
799
|
+
const response = await fetch(`${baseUrl}/README`, { method: "GET" });
|
|
800
|
+
hasReadme = response.ok;
|
|
801
|
+
}
|
|
802
|
+
catch {
|
|
803
|
+
// No README endpoint
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
checks.push({
|
|
807
|
+
id: "documentation",
|
|
808
|
+
name: "Provides documentation endpoint",
|
|
809
|
+
passed: hasReadme,
|
|
810
|
+
score: hasReadme ? 1 : 0,
|
|
811
|
+
maxScore: 1,
|
|
812
|
+
details: hasReadme
|
|
813
|
+
? "Documentation endpoint available"
|
|
814
|
+
: "No /docs or /README endpoint",
|
|
815
|
+
});
|
|
816
|
+
if (!hasReadme) {
|
|
817
|
+
addIssue(6, "low", "No documentation endpoint", "Add /docs endpoint with usage examples", "quick");
|
|
818
|
+
}
|
|
819
|
+
return checks;
|
|
820
|
+
}
|
|
821
|
+
/**
|
|
822
|
+
* Generate prioritized recommendations from issues and tier results
|
|
823
|
+
*/
|
|
824
|
+
function generateRecommendations(issues, tiers) {
|
|
825
|
+
const recommendations = [];
|
|
826
|
+
// Add critical issues first
|
|
827
|
+
for (const issue of issues.filter(i => i.severity === "critical")) {
|
|
828
|
+
recommendations.push(`[CRITICAL] ${issue.remediation}`);
|
|
829
|
+
}
|
|
830
|
+
// Add high issues
|
|
831
|
+
for (const issue of issues.filter(i => i.severity === "high")) {
|
|
832
|
+
recommendations.push(`[HIGH] ${issue.remediation}`);
|
|
833
|
+
}
|
|
834
|
+
// Add quick wins (low effort, medium severity)
|
|
835
|
+
for (const issue of issues.filter(i => i.severity === "medium" && i.effort === "quick")) {
|
|
836
|
+
recommendations.push(`[QUICK WIN] ${issue.remediation}`);
|
|
837
|
+
}
|
|
838
|
+
// Add tier-specific recommendations for lowest scoring tiers
|
|
839
|
+
const sortedTiers = [...tiers].sort((a, b) => a.score - b.score);
|
|
840
|
+
for (const tier of sortedTiers.slice(0, 2)) {
|
|
841
|
+
if (tier.score < 70) {
|
|
842
|
+
recommendations.push(`[${tier.name.toUpperCase()}] Focus on improving tier ${tier.tier} (${tier.score}%): ${tier.checks
|
|
843
|
+
.filter(c => !c.passed)
|
|
844
|
+
.map(c => c.name)
|
|
845
|
+
.slice(0, 2)
|
|
846
|
+
.join(", ")}`);
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
// Add remaining medium issues
|
|
850
|
+
for (const issue of issues.filter(i => i.severity === "medium" && i.effort !== "quick")) {
|
|
851
|
+
recommendations.push(issue.remediation);
|
|
852
|
+
}
|
|
853
|
+
return recommendations.slice(0, 10); // Top 10 recommendations
|
|
854
|
+
}
|
|
855
|
+
//# sourceMappingURL=webmcp-readiness.js.map
|