@bryan-thompson/inspector-assessment-client 1.24.2 → 1.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-CoDaMN6l.js → OAuthCallback-CNC5_mEQ.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-DVsgc4Jd.js → OAuthDebugCallback-BstXB61i.js} +1 -1
- package/dist/assets/{index-CauENw8a.js → index-jmzR9VGa.js} +4 -4
- package/dist/index.html +1 -1
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.d.ts +67 -0
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.d.ts.map +1 -0
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.js +586 -0
- package/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts +108 -0
- package/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts.map +1 -0
- package/lib/services/assessment/modules/ProtocolComplianceAssessor.js +782 -0
- package/lib/services/assessment/modules/index.d.ts +57 -11
- package/lib/services/assessment/modules/index.d.ts.map +1 -1
- package/lib/services/assessment/modules/index.js +72 -14
- package/package.json +1 -1
|
@@ -0,0 +1,586 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Developer Experience Assessor Module
|
|
3
|
+
*
|
|
4
|
+
* Unified module for evaluating developer experience aspects of MCP servers.
|
|
5
|
+
* Merges DocumentationAssessor and UsabilityAssessor functionality.
|
|
6
|
+
*
|
|
7
|
+
* Assessment Areas:
|
|
8
|
+
* 1. Documentation Quality - README completeness, examples, guides
|
|
9
|
+
* 2. Usability - Tool naming, parameter clarity, best practices
|
|
10
|
+
*
|
|
11
|
+
* This module is part of Tier 4 (Extended) and is optional for security-focused audits.
|
|
12
|
+
*
|
|
13
|
+
* @module assessment/modules/DeveloperExperienceAssessor
|
|
14
|
+
*/
|
|
15
|
+
import { BaseAssessor } from "./BaseAssessor.js";
|
|
16
|
+
export class DeveloperExperienceAssessor extends BaseAssessor {
|
|
17
|
+
async assess(context) {
|
|
18
|
+
this.log("Starting developer experience assessment");
|
|
19
|
+
// Assess documentation
|
|
20
|
+
const documentationMetrics = this.analyzeDocumentation(context.readmeContent || "", context.tools, "verbose");
|
|
21
|
+
const documentationScore = this.calculateDocumentationScore(documentationMetrics);
|
|
22
|
+
// Assess usability
|
|
23
|
+
const usabilityMetrics = this.analyzeUsability(context.tools);
|
|
24
|
+
const usabilityScore = this.calculateUsabilityScore(usabilityMetrics);
|
|
25
|
+
// Calculate overall score (weighted average)
|
|
26
|
+
const overallScore = Math.round(documentationScore * 0.6 + usabilityScore * 0.4);
|
|
27
|
+
// Determine status
|
|
28
|
+
const status = this.determineOverallStatus(overallScore);
|
|
29
|
+
// Generate explanation and recommendations
|
|
30
|
+
const explanation = this.generateExplanation(documentationMetrics, usabilityMetrics, context.tools);
|
|
31
|
+
const recommendations = this.generateRecommendations(documentationMetrics, usabilityMetrics);
|
|
32
|
+
this.testCount = 9; // Documentation (5) + Usability (4) checks
|
|
33
|
+
return {
|
|
34
|
+
documentation: documentationMetrics,
|
|
35
|
+
usability: usabilityMetrics,
|
|
36
|
+
status,
|
|
37
|
+
explanation,
|
|
38
|
+
recommendations,
|
|
39
|
+
scores: {
|
|
40
|
+
documentation: documentationScore,
|
|
41
|
+
usability: usabilityScore,
|
|
42
|
+
overall: overallScore,
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
// ============================================================================
|
|
47
|
+
// Documentation Analysis (from DocumentationAssessor)
|
|
48
|
+
// ============================================================================
|
|
49
|
+
analyzeDocumentation(content, tools, verbosity = "standard") {
|
|
50
|
+
const hasReadme = content.length > 0;
|
|
51
|
+
const functionalExamples = this.extractFunctionalExamples(content);
|
|
52
|
+
const allCodeExamples = this.extractCodeExamples(content);
|
|
53
|
+
const hasInstallInstructions = this.checkInstallInstructions(content);
|
|
54
|
+
const hasUsageGuide = this.checkUsageGuide(content);
|
|
55
|
+
const hasAPIReference = this.checkAPIReference(content);
|
|
56
|
+
const missingExamples = [];
|
|
57
|
+
let documentedToolsCount = 0;
|
|
58
|
+
const toolDocumentation = [];
|
|
59
|
+
const ADEQUATE_DESCRIPTION_LENGTH = 50;
|
|
60
|
+
let toolsWithDescriptions = 0;
|
|
61
|
+
const toolDocGaps = [];
|
|
62
|
+
if (tools && tools.length > 0) {
|
|
63
|
+
for (const tool of tools) {
|
|
64
|
+
const toolName = tool.name;
|
|
65
|
+
const description = tool.description?.trim() || "";
|
|
66
|
+
const descriptionLength = description.length;
|
|
67
|
+
const hasDescription = descriptionLength > 0;
|
|
68
|
+
const hasAdequateDescription = descriptionLength >= ADEQUATE_DESCRIPTION_LENGTH;
|
|
69
|
+
const headingRegex = new RegExp(`^#{1,6}\\s+${toolName}`, "mi");
|
|
70
|
+
const mentionRegex = new RegExp(`\\b${toolName}\\b`, "i");
|
|
71
|
+
const hasHeading = headingRegex.test(content);
|
|
72
|
+
const hasMention = mentionRegex.test(content);
|
|
73
|
+
const documentedInReadme = hasHeading || hasMention;
|
|
74
|
+
if (documentedInReadme) {
|
|
75
|
+
documentedToolsCount++;
|
|
76
|
+
}
|
|
77
|
+
if (!hasDescription && !documentedInReadme) {
|
|
78
|
+
missingExamples.push(toolName);
|
|
79
|
+
}
|
|
80
|
+
if (hasAdequateDescription) {
|
|
81
|
+
toolsWithDescriptions++;
|
|
82
|
+
}
|
|
83
|
+
else {
|
|
84
|
+
toolDocGaps.push({
|
|
85
|
+
toolName,
|
|
86
|
+
issue: descriptionLength === 0 ? "missing" : "too_short",
|
|
87
|
+
descriptionLength,
|
|
88
|
+
documentedInReadme,
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
if (verbosity !== "minimal") {
|
|
92
|
+
toolDocumentation.push({
|
|
93
|
+
name: toolName,
|
|
94
|
+
hasDescription,
|
|
95
|
+
descriptionLength,
|
|
96
|
+
documentedInReadme,
|
|
97
|
+
description: hasDescription ? description.slice(0, 200) : undefined,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
else {
|
|
103
|
+
if (functionalExamples.length < 1)
|
|
104
|
+
missingExamples.push("Basic usage example");
|
|
105
|
+
}
|
|
106
|
+
const requiredExamples = 3;
|
|
107
|
+
const functionalExampleCount = functionalExamples.length +
|
|
108
|
+
(tools && tools.length > 0 ? documentedToolsCount : 0);
|
|
109
|
+
const sectionHeadings = verbosity !== "minimal" ? this.extractSectionHeadings(content) : [];
|
|
110
|
+
const baseMetrics = {
|
|
111
|
+
hasReadme,
|
|
112
|
+
exampleCount: functionalExampleCount,
|
|
113
|
+
requiredExamples,
|
|
114
|
+
missingExamples,
|
|
115
|
+
hasInstallInstructions,
|
|
116
|
+
hasUsageGuide,
|
|
117
|
+
hasAPIReference,
|
|
118
|
+
extractedExamples: allCodeExamples,
|
|
119
|
+
installInstructions: hasInstallInstructions
|
|
120
|
+
? this.extractSection(content, "install")
|
|
121
|
+
: undefined,
|
|
122
|
+
usageInstructions: hasUsageGuide
|
|
123
|
+
? this.extractSection(content, "usage")
|
|
124
|
+
: undefined,
|
|
125
|
+
toolsWithDescriptions,
|
|
126
|
+
toolsTotal: tools?.length || 0,
|
|
127
|
+
toolDocGaps,
|
|
128
|
+
};
|
|
129
|
+
if (verbosity !== "minimal") {
|
|
130
|
+
baseMetrics.readmeLength = content.length;
|
|
131
|
+
baseMetrics.readmeWordCount = content
|
|
132
|
+
.split(/\s+/)
|
|
133
|
+
.filter((w) => w.length > 0).length;
|
|
134
|
+
baseMetrics.sectionHeadings = sectionHeadings;
|
|
135
|
+
if (toolDocumentation.length > 0) {
|
|
136
|
+
baseMetrics.toolDocumentation = toolDocumentation;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (verbosity === "verbose" && content.length > 0) {
|
|
140
|
+
baseMetrics.readmeContent = content.substring(0, 5000);
|
|
141
|
+
}
|
|
142
|
+
return baseMetrics;
|
|
143
|
+
}
|
|
144
|
+
extractFunctionalExamples(content) {
|
|
145
|
+
const functionalExamples = [];
|
|
146
|
+
const standalonePromptRegex = /^[ \t]*([A-Z][^\n]{10,300}?(?:use (?:context7|library|@?[\w-]+\/[\w-]+)|with \S+)[^\n]*?)[ \t]*$/gim;
|
|
147
|
+
let standaloneMatch;
|
|
148
|
+
while ((standaloneMatch = standalonePromptRegex.exec(content)) !== null) {
|
|
149
|
+
const prompt = standaloneMatch[1].trim();
|
|
150
|
+
if (!this.isNonFunctionalCodeBlock(prompt) &&
|
|
151
|
+
this.scoreFunctionalExample(prompt)) {
|
|
152
|
+
functionalExamples.push({
|
|
153
|
+
code: prompt,
|
|
154
|
+
language: "prompt",
|
|
155
|
+
description: "Functional example prompt",
|
|
156
|
+
lineNumber: this.getLineNumber(content, standaloneMatch.index),
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
const bulletedExampleRegex = /^[ \t]*[-*][ \t]+([A-Z][^\n]{10,300}?)[ \t]*$/gim;
|
|
161
|
+
let bulletMatch;
|
|
162
|
+
while ((bulletMatch = bulletedExampleRegex.exec(content)) !== null) {
|
|
163
|
+
const prompt = bulletMatch[1].trim();
|
|
164
|
+
if (!this.isNonFunctionalCodeBlock(prompt) &&
|
|
165
|
+
this.scoreFunctionalExample(prompt)) {
|
|
166
|
+
functionalExamples.push({
|
|
167
|
+
code: prompt,
|
|
168
|
+
language: "prompt",
|
|
169
|
+
description: "Bulleted example",
|
|
170
|
+
lineNumber: this.getLineNumber(content, bulletMatch.index),
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
return this.deduplicateExamples(functionalExamples);
|
|
175
|
+
}
|
|
176
|
+
isNonFunctionalCodeBlock(text) {
|
|
177
|
+
const excludePatterns = [
|
|
178
|
+
/^\s*{\s*["']mcpServers["']/i,
|
|
179
|
+
/^\s*{\s*["']command["']/i,
|
|
180
|
+
/^\s*(npx|npm|yarn|pnpm|docker|git)\s+/i,
|
|
181
|
+
/^\s*FROM\s+\w+:/i,
|
|
182
|
+
/^\s*import\s+.*\s+from/i,
|
|
183
|
+
/^\s*\[.*\]\s*=\s*["']/i,
|
|
184
|
+
/^\s*<\w+>/i,
|
|
185
|
+
/^\s*(export|const|let|var|function)\s+/i,
|
|
186
|
+
/^\s*\/\//i,
|
|
187
|
+
/^\s*#\s*\w+/i,
|
|
188
|
+
];
|
|
189
|
+
return excludePatterns.some((pattern) => pattern.test(text));
|
|
190
|
+
}
|
|
191
|
+
scoreFunctionalExample(text) {
|
|
192
|
+
let score = 0;
|
|
193
|
+
if (/\b(create|configure|implement|show|generate|build|write|add|get|set|use|run|start)\b/i.test(text)) {
|
|
194
|
+
score += 2;
|
|
195
|
+
}
|
|
196
|
+
if (/\b(?:use|with)\s+(?:context7|library|@?\w+\/\w+)\b/i.test(text)) {
|
|
197
|
+
score += 2;
|
|
198
|
+
}
|
|
199
|
+
if (/^[A-Z].{10,}/.test(text)) {
|
|
200
|
+
score += 1;
|
|
201
|
+
}
|
|
202
|
+
if (/\b(Next\.js|React|Vue|Angular|PostgreSQL|MySQL|MongoDB|Redis|AWS|Cloudflare|API|HTTP|REST|GraphQL|TypeScript|JavaScript|Python|Docker|Kubernetes|Supabase|Firebase|Auth|JWT|OAuth)\b/i.test(text)) {
|
|
203
|
+
score += 1;
|
|
204
|
+
}
|
|
205
|
+
return score >= 4;
|
|
206
|
+
}
|
|
207
|
+
getLineNumber(content, position) {
|
|
208
|
+
const beforeMatch = content.substring(0, position);
|
|
209
|
+
return beforeMatch.split("\n").length;
|
|
210
|
+
}
|
|
211
|
+
deduplicateExamples(examples) {
|
|
212
|
+
const seen = new Set();
|
|
213
|
+
const unique = [];
|
|
214
|
+
for (const example of examples) {
|
|
215
|
+
const normalized = example.code
|
|
216
|
+
.toLowerCase()
|
|
217
|
+
.replace(/[^\w\s]/g, "")
|
|
218
|
+
.replace(/\s+/g, " ")
|
|
219
|
+
.trim();
|
|
220
|
+
if (!seen.has(normalized)) {
|
|
221
|
+
seen.add(normalized);
|
|
222
|
+
unique.push(example);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return unique;
|
|
226
|
+
}
|
|
227
|
+
extractCodeExamples(content) {
|
|
228
|
+
const examples = [];
|
|
229
|
+
const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
|
|
230
|
+
let match;
|
|
231
|
+
const lines = content.split("\n");
|
|
232
|
+
while ((match = codeBlockRegex.exec(content)) !== null) {
|
|
233
|
+
const language = match[1] || "plaintext";
|
|
234
|
+
const code = match[2].trim();
|
|
235
|
+
const position = match.index;
|
|
236
|
+
const beforeMatch = content.substring(0, position);
|
|
237
|
+
const lineNumber = beforeMatch.split("\n").length;
|
|
238
|
+
let description = "";
|
|
239
|
+
const lineIndex = lines.findIndex((_, i) => lines.slice(0, i + 1).join("\n").length >= position);
|
|
240
|
+
if (lineIndex > 0) {
|
|
241
|
+
const prevLine = lines[lineIndex - 1].trim();
|
|
242
|
+
if (prevLine && !prevLine.startsWith("#")) {
|
|
243
|
+
description = prevLine;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
examples.push({
|
|
247
|
+
code,
|
|
248
|
+
language,
|
|
249
|
+
description,
|
|
250
|
+
lineNumber,
|
|
251
|
+
lineCount: code.split("\n").length,
|
|
252
|
+
exampleType: this.classifyCodeExample(code, language),
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
return examples;
|
|
256
|
+
}
|
|
257
|
+
checkInstallInstructions(content) {
|
|
258
|
+
const installKeywords = [
|
|
259
|
+
"install",
|
|
260
|
+
"npm install",
|
|
261
|
+
"yarn add",
|
|
262
|
+
"pip install",
|
|
263
|
+
"setup",
|
|
264
|
+
"getting started",
|
|
265
|
+
];
|
|
266
|
+
const contentLower = content.toLowerCase();
|
|
267
|
+
return installKeywords.some((keyword) => contentLower.includes(keyword));
|
|
268
|
+
}
|
|
269
|
+
checkUsageGuide(content) {
|
|
270
|
+
const usageKeywords = [
|
|
271
|
+
"usage",
|
|
272
|
+
"how to use",
|
|
273
|
+
"example",
|
|
274
|
+
"quick start",
|
|
275
|
+
"tutorial",
|
|
276
|
+
];
|
|
277
|
+
const contentLower = content.toLowerCase();
|
|
278
|
+
return usageKeywords.some((keyword) => contentLower.includes(keyword));
|
|
279
|
+
}
|
|
280
|
+
checkAPIReference(content) {
|
|
281
|
+
const apiKeywords = [
|
|
282
|
+
"api",
|
|
283
|
+
"reference",
|
|
284
|
+
"methods",
|
|
285
|
+
"functions",
|
|
286
|
+
"parameters",
|
|
287
|
+
"returns",
|
|
288
|
+
"endpoints",
|
|
289
|
+
];
|
|
290
|
+
const contentLower = content.toLowerCase();
|
|
291
|
+
return apiKeywords.some((keyword) => contentLower.includes(keyword));
|
|
292
|
+
}
|
|
293
|
+
extractSection(content, section) {
|
|
294
|
+
const sectionRegex = new RegExp(`#+\\s*${section}[\\s\\S]*?(?=\\n#|$)`, "gi");
|
|
295
|
+
const match = content.match(sectionRegex);
|
|
296
|
+
return match ? match[0].trim() : "";
|
|
297
|
+
}
|
|
298
|
+
extractSectionHeadings(content) {
|
|
299
|
+
const headingRegex = /^(#{1,6})\s+(.+)$/gm;
|
|
300
|
+
const headings = [];
|
|
301
|
+
let match;
|
|
302
|
+
while ((match = headingRegex.exec(content)) !== null) {
|
|
303
|
+
headings.push(match[2].trim());
|
|
304
|
+
}
|
|
305
|
+
return headings;
|
|
306
|
+
}
|
|
307
|
+
classifyCodeExample(code, language) {
|
|
308
|
+
if (/^\s*(npx|npm|yarn|pnpm|pip|docker|git)\s+/i.test(code)) {
|
|
309
|
+
return "install";
|
|
310
|
+
}
|
|
311
|
+
if (/^\s*{\s*["']mcpServers["']/i.test(code) ||
|
|
312
|
+
/^\s*{\s*["']command["']/i.test(code) ||
|
|
313
|
+
language === "json" ||
|
|
314
|
+
language === "toml" ||
|
|
315
|
+
language === "yaml") {
|
|
316
|
+
return "config";
|
|
317
|
+
}
|
|
318
|
+
if (/^\s*(import|export|const|let|var|function|class)\s+/i.test(code) ||
|
|
319
|
+
/^\s*from\s+/i.test(code)) {
|
|
320
|
+
return "implementation";
|
|
321
|
+
}
|
|
322
|
+
return "functional";
|
|
323
|
+
}
|
|
324
|
+
calculateDocumentationScore(metrics) {
|
|
325
|
+
let score = 0;
|
|
326
|
+
const maxScore = 5;
|
|
327
|
+
if (metrics.hasReadme)
|
|
328
|
+
score++;
|
|
329
|
+
if (metrics.hasInstallInstructions)
|
|
330
|
+
score++;
|
|
331
|
+
if (metrics.hasUsageGuide)
|
|
332
|
+
score++;
|
|
333
|
+
if (metrics.hasAPIReference)
|
|
334
|
+
score++;
|
|
335
|
+
if (metrics.exampleCount >= metrics.requiredExamples)
|
|
336
|
+
score++;
|
|
337
|
+
return Math.round((score / maxScore) * 100);
|
|
338
|
+
}
|
|
339
|
+
// ============================================================================
|
|
340
|
+
// Usability Analysis (from UsabilityAssessor)
|
|
341
|
+
// ============================================================================
|
|
342
|
+
analyzeUsability(tools) {
|
|
343
|
+
const toolNamingConvention = this.analyzeNamingConvention(tools);
|
|
344
|
+
const parameterClarity = this.analyzeParameterClarity(tools);
|
|
345
|
+
const hasHelpfulDescriptions = this.checkDescriptions(tools);
|
|
346
|
+
const followsBestPractices = this.checkBestPractices(tools);
|
|
347
|
+
return {
|
|
348
|
+
toolNamingConvention,
|
|
349
|
+
parameterClarity,
|
|
350
|
+
hasHelpfulDescriptions,
|
|
351
|
+
followsBestPractices,
|
|
352
|
+
};
|
|
353
|
+
}
|
|
354
|
+
analyzeNamingConvention(tools) {
|
|
355
|
+
if (tools.length === 0)
|
|
356
|
+
return "consistent";
|
|
357
|
+
const namingPatterns = {
|
|
358
|
+
camelCase: 0,
|
|
359
|
+
snake_case: 0,
|
|
360
|
+
kebab_case: 0,
|
|
361
|
+
PascalCase: 0,
|
|
362
|
+
};
|
|
363
|
+
for (const tool of tools) {
|
|
364
|
+
const name = tool.name;
|
|
365
|
+
if (/^[a-z][a-zA-Z0-9]*$/.test(name)) {
|
|
366
|
+
namingPatterns.camelCase++;
|
|
367
|
+
}
|
|
368
|
+
else if (/^[a-z]+(_[a-z]+)*$/.test(name)) {
|
|
369
|
+
namingPatterns.snake_case++;
|
|
370
|
+
}
|
|
371
|
+
else if (/^[a-z]+(-[a-z]+)*$/.test(name)) {
|
|
372
|
+
namingPatterns.kebab_case++;
|
|
373
|
+
}
|
|
374
|
+
else if (/^[A-Z][a-zA-Z0-9]*$/.test(name)) {
|
|
375
|
+
namingPatterns.PascalCase++;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
const total = tools.length;
|
|
379
|
+
const threshold = total * 0.7;
|
|
380
|
+
for (const count of Object.values(namingPatterns)) {
|
|
381
|
+
if (count >= threshold) {
|
|
382
|
+
return "consistent";
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
return "inconsistent";
|
|
386
|
+
}
|
|
387
|
+
analyzeParameterClarity(tools) {
|
|
388
|
+
if (tools.length === 0)
|
|
389
|
+
return "clear";
|
|
390
|
+
let clearCount = 0;
|
|
391
|
+
let unclearCount = 0;
|
|
392
|
+
for (const tool of tools) {
|
|
393
|
+
const schema = this.getToolSchema(tool);
|
|
394
|
+
if (!schema?.properties)
|
|
395
|
+
continue;
|
|
396
|
+
for (const [paramName, paramDef] of Object.entries(schema.properties)) {
|
|
397
|
+
if (this.isDescriptiveName(paramName)) {
|
|
398
|
+
clearCount++;
|
|
399
|
+
}
|
|
400
|
+
else {
|
|
401
|
+
unclearCount++;
|
|
402
|
+
}
|
|
403
|
+
if (paramDef.description) {
|
|
404
|
+
clearCount++;
|
|
405
|
+
}
|
|
406
|
+
else {
|
|
407
|
+
unclearCount++;
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
const total = clearCount + unclearCount;
|
|
412
|
+
if (total === 0)
|
|
413
|
+
return "clear";
|
|
414
|
+
const clarityRatio = clearCount / total;
|
|
415
|
+
if (clarityRatio >= 0.8)
|
|
416
|
+
return "clear";
|
|
417
|
+
if (clarityRatio <= 0.3)
|
|
418
|
+
return "unclear";
|
|
419
|
+
return "mixed";
|
|
420
|
+
}
|
|
421
|
+
checkDescriptions(tools) {
|
|
422
|
+
if (tools.length === 0)
|
|
423
|
+
return false;
|
|
424
|
+
let toolsWithDescriptions = 0;
|
|
425
|
+
for (const tool of tools) {
|
|
426
|
+
if (tool.description && tool.description.length > 10) {
|
|
427
|
+
toolsWithDescriptions++;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
return toolsWithDescriptions / tools.length >= 0.7;
|
|
431
|
+
}
|
|
432
|
+
checkBestPractices(tools) {
|
|
433
|
+
const practices = {
|
|
434
|
+
hasVersioning: false,
|
|
435
|
+
hasErrorHandling: false,
|
|
436
|
+
hasValidation: false,
|
|
437
|
+
hasDocumentation: false,
|
|
438
|
+
};
|
|
439
|
+
for (const tool of tools) {
|
|
440
|
+
const schema = this.getToolSchema(tool);
|
|
441
|
+
if (schema?.required && schema.required.length > 0) {
|
|
442
|
+
practices.hasValidation = true;
|
|
443
|
+
}
|
|
444
|
+
if (schema?.properties) {
|
|
445
|
+
for (const prop of Object.values(schema.properties)) {
|
|
446
|
+
if (prop.enum ||
|
|
447
|
+
prop.minimum !== undefined ||
|
|
448
|
+
prop.maximum !== undefined) {
|
|
449
|
+
practices.hasValidation = true;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
if (tool.description) {
|
|
454
|
+
practices.hasDocumentation = true;
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
const followedPractices = Object.values(practices).filter((v) => v).length;
|
|
458
|
+
return followedPractices >= 2;
|
|
459
|
+
}
|
|
460
|
+
isDescriptiveName(name) {
|
|
461
|
+
const goodNames = [
|
|
462
|
+
"query",
|
|
463
|
+
"search",
|
|
464
|
+
"input",
|
|
465
|
+
"output",
|
|
466
|
+
"data",
|
|
467
|
+
"content",
|
|
468
|
+
"message",
|
|
469
|
+
"text",
|
|
470
|
+
"file",
|
|
471
|
+
"path",
|
|
472
|
+
"url",
|
|
473
|
+
"name",
|
|
474
|
+
"id",
|
|
475
|
+
"value",
|
|
476
|
+
"result",
|
|
477
|
+
"response",
|
|
478
|
+
"request",
|
|
479
|
+
"params",
|
|
480
|
+
];
|
|
481
|
+
const nameLower = name.toLowerCase();
|
|
482
|
+
for (const goodName of goodNames) {
|
|
483
|
+
if (nameLower.includes(goodName)) {
|
|
484
|
+
return true;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
return name.length > 3 && !/^[a-z]$/.test(name);
|
|
488
|
+
}
|
|
489
|
+
getToolSchema(tool) {
|
|
490
|
+
if (!tool.inputSchema)
|
|
491
|
+
return null;
|
|
492
|
+
return typeof tool.inputSchema === "string"
|
|
493
|
+
? this.safeJsonParse(tool.inputSchema)
|
|
494
|
+
: tool.inputSchema;
|
|
495
|
+
}
|
|
496
|
+
calculateUsabilityScore(metrics) {
|
|
497
|
+
let score = 0;
|
|
498
|
+
const maxScore = 4;
|
|
499
|
+
if (metrics.toolNamingConvention === "consistent")
|
|
500
|
+
score++;
|
|
501
|
+
if (metrics.parameterClarity === "clear")
|
|
502
|
+
score++;
|
|
503
|
+
if (metrics.hasHelpfulDescriptions)
|
|
504
|
+
score++;
|
|
505
|
+
if (metrics.followsBestPractices)
|
|
506
|
+
score++;
|
|
507
|
+
return Math.round((score / maxScore) * 100);
|
|
508
|
+
}
|
|
509
|
+
// ============================================================================
|
|
510
|
+
// Combined Status, Explanation, and Recommendations
|
|
511
|
+
// ============================================================================
|
|
512
|
+
determineOverallStatus(overallScore) {
|
|
513
|
+
if (overallScore >= 80)
|
|
514
|
+
return "PASS";
|
|
515
|
+
if (overallScore >= 50)
|
|
516
|
+
return "NEED_MORE_INFO";
|
|
517
|
+
return "FAIL";
|
|
518
|
+
}
|
|
519
|
+
generateExplanation(docMetrics, usabilityMetrics, tools) {
|
|
520
|
+
const parts = [];
|
|
521
|
+
// Documentation summary
|
|
522
|
+
if (!docMetrics.hasReadme) {
|
|
523
|
+
parts.push("No README documentation found.");
|
|
524
|
+
}
|
|
525
|
+
else {
|
|
526
|
+
parts.push(`README contains ${docMetrics.exampleCount} code examples.`);
|
|
527
|
+
const features = [];
|
|
528
|
+
if (docMetrics.hasInstallInstructions)
|
|
529
|
+
features.push("installation");
|
|
530
|
+
if (docMetrics.hasUsageGuide)
|
|
531
|
+
features.push("usage guide");
|
|
532
|
+
if (docMetrics.hasAPIReference)
|
|
533
|
+
features.push("API reference");
|
|
534
|
+
if (features.length > 0) {
|
|
535
|
+
parts.push(`Documentation includes: ${features.join(", ")}.`);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
// Usability summary
|
|
539
|
+
parts.push(`Analyzed ${tools.length} tools for usability.`);
|
|
540
|
+
parts.push(`Naming convention: ${usabilityMetrics.toolNamingConvention}.`);
|
|
541
|
+
parts.push(`Parameter clarity: ${usabilityMetrics.parameterClarity}.`);
|
|
542
|
+
const usabilityFeatures = [];
|
|
543
|
+
if (usabilityMetrics.hasHelpfulDescriptions)
|
|
544
|
+
usabilityFeatures.push("helpful descriptions");
|
|
545
|
+
if (usabilityMetrics.followsBestPractices)
|
|
546
|
+
usabilityFeatures.push("follows best practices");
|
|
547
|
+
if (usabilityFeatures.length > 0) {
|
|
548
|
+
parts.push(`Usability: ${usabilityFeatures.join(", ")}.`);
|
|
549
|
+
}
|
|
550
|
+
return parts.join(" ");
|
|
551
|
+
}
|
|
552
|
+
generateRecommendations(docMetrics, usabilityMetrics) {
|
|
553
|
+
const recommendations = [];
|
|
554
|
+
// Documentation recommendations
|
|
555
|
+
if (!docMetrics.hasReadme) {
|
|
556
|
+
recommendations.push("Create a comprehensive README.md file");
|
|
557
|
+
}
|
|
558
|
+
if (!docMetrics.hasInstallInstructions) {
|
|
559
|
+
recommendations.push("Add clear installation instructions");
|
|
560
|
+
}
|
|
561
|
+
if (!docMetrics.hasUsageGuide) {
|
|
562
|
+
recommendations.push("Include a usage guide with examples");
|
|
563
|
+
}
|
|
564
|
+
if (!docMetrics.hasAPIReference) {
|
|
565
|
+
recommendations.push("Document all available tools and parameters");
|
|
566
|
+
}
|
|
567
|
+
if (docMetrics.exampleCount < docMetrics.requiredExamples) {
|
|
568
|
+
recommendations.push(`Add ${docMetrics.requiredExamples - docMetrics.exampleCount} more code examples`);
|
|
569
|
+
}
|
|
570
|
+
// Usability recommendations
|
|
571
|
+
if (usabilityMetrics.toolNamingConvention === "inconsistent") {
|
|
572
|
+
recommendations.push("Adopt a consistent naming convention for all tools");
|
|
573
|
+
}
|
|
574
|
+
if (usabilityMetrics.parameterClarity !== "clear") {
|
|
575
|
+
recommendations.push("Use descriptive parameter names");
|
|
576
|
+
recommendations.push("Add descriptions for all parameters");
|
|
577
|
+
}
|
|
578
|
+
if (!usabilityMetrics.hasHelpfulDescriptions) {
|
|
579
|
+
recommendations.push("Provide detailed descriptions for each tool");
|
|
580
|
+
}
|
|
581
|
+
if (!usabilityMetrics.followsBestPractices) {
|
|
582
|
+
recommendations.push("Implement input validation with constraints");
|
|
583
|
+
}
|
|
584
|
+
return recommendations;
|
|
585
|
+
}
|
|
586
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Protocol Compliance Assessor Module
|
|
3
|
+
*
|
|
4
|
+
* Unified module for MCP protocol compliance validation.
|
|
5
|
+
* Merges MCPSpecComplianceAssessor and ProtocolConformanceAssessor functionality.
|
|
6
|
+
*
|
|
7
|
+
* Protocol Checks:
|
|
8
|
+
* 1. JSON-RPC 2.0 Compliance - Validates request/response structure
|
|
9
|
+
* 2. Server Info Validity - Validates initialization handshake
|
|
10
|
+
* 3. Schema Compliance - Validates tool input schemas
|
|
11
|
+
* 4. Error Response Format - Validates isError flag, content array structure
|
|
12
|
+
* 5. Content Type Support - Validates valid content types (text, image, audio, resource)
|
|
13
|
+
* 6. Structured Output Support - Checks for outputSchema usage
|
|
14
|
+
* 7. Capabilities Compliance - Validates declared vs actual capabilities
|
|
15
|
+
*
|
|
16
|
+
* @module assessment/modules/ProtocolComplianceAssessor
|
|
17
|
+
*/
|
|
18
|
+
import { MCPSpecComplianceAssessment, AssessmentConfiguration } from "../../../lib/assessmentTypes.js";
|
|
19
|
+
import type { ProtocolCheck } from "../../../lib/assessment/extendedTypes.js";
|
|
20
|
+
import { BaseAssessor } from "./BaseAssessor.js";
|
|
21
|
+
import { AssessmentContext } from "../AssessmentOrchestrator.js";
|
|
22
|
+
/**
|
|
23
|
+
* Protocol Compliance Assessment Result
|
|
24
|
+
* Unified output type for protocol compliance checks
|
|
25
|
+
*/
|
|
26
|
+
export interface ProtocolComplianceAssessment extends MCPSpecComplianceAssessment {
|
|
27
|
+
/** Additional conformance-style checks from ProtocolConformanceAssessor */
|
|
28
|
+
conformanceChecks?: {
|
|
29
|
+
errorResponseFormat: ProtocolCheck;
|
|
30
|
+
contentTypeSupport: ProtocolCheck;
|
|
31
|
+
initializationHandshake: ProtocolCheck;
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
export declare class ProtocolComplianceAssessor extends BaseAssessor<ProtocolComplianceAssessment> {
|
|
35
|
+
private ajv;
|
|
36
|
+
constructor(config: AssessmentConfiguration);
|
|
37
|
+
/**
|
|
38
|
+
* Get MCP spec version from config or use default
|
|
39
|
+
*/
|
|
40
|
+
private getSpecVersion;
|
|
41
|
+
/**
|
|
42
|
+
* Get base URL for MCP specification
|
|
43
|
+
*/
|
|
44
|
+
private getSpecBaseUrl;
|
|
45
|
+
/**
|
|
46
|
+
* Assess MCP Protocol Compliance - Unified Approach
|
|
47
|
+
* Combines MCPSpecComplianceAssessor and ProtocolConformanceAssessor functionality
|
|
48
|
+
*/
|
|
49
|
+
assess(context: AssessmentContext): Promise<ProtocolComplianceAssessment>;
|
|
50
|
+
/**
|
|
51
|
+
* Extract protocol version from context
|
|
52
|
+
*/
|
|
53
|
+
private extractProtocolVersion;
|
|
54
|
+
/**
|
|
55
|
+
* Check JSON-RPC 2.0 compliance
|
|
56
|
+
*/
|
|
57
|
+
private checkJsonRpcCompliance;
|
|
58
|
+
/**
|
|
59
|
+
* Check if server info is valid and properly formatted
|
|
60
|
+
*/
|
|
61
|
+
private checkServerInfoValidity;
|
|
62
|
+
/**
|
|
63
|
+
* Check schema compliance for all tools
|
|
64
|
+
*/
|
|
65
|
+
private checkSchemaCompliance;
|
|
66
|
+
/**
|
|
67
|
+
* Check error response compliance (basic check from MCPSpec)
|
|
68
|
+
*/
|
|
69
|
+
private checkErrorResponses;
|
|
70
|
+
/**
|
|
71
|
+
* Check structured output support (2025-06-18 feature)
|
|
72
|
+
*/
|
|
73
|
+
private checkStructuredOutputSupport;
|
|
74
|
+
/**
|
|
75
|
+
* Check capabilities compliance
|
|
76
|
+
*/
|
|
77
|
+
private checkCapabilitiesCompliance;
|
|
78
|
+
/**
|
|
79
|
+
* Select representative tools for testing (first, middle, last for diversity)
|
|
80
|
+
*/
|
|
81
|
+
private selectToolsForTesting;
|
|
82
|
+
/**
|
|
83
|
+
* Check Error Response Format (conformance-style with multi-tool testing)
|
|
84
|
+
*/
|
|
85
|
+
private checkErrorResponseFormat;
|
|
86
|
+
/**
|
|
87
|
+
* Check Content Type Support
|
|
88
|
+
*/
|
|
89
|
+
private checkContentTypeSupport;
|
|
90
|
+
/**
|
|
91
|
+
* Check Initialization Handshake
|
|
92
|
+
*/
|
|
93
|
+
private checkInitializationHandshake;
|
|
94
|
+
private assessTransportCompliance;
|
|
95
|
+
private assessAnnotationSupport;
|
|
96
|
+
private assessStreamingSupport;
|
|
97
|
+
private assessOAuthCompliance;
|
|
98
|
+
private extractMetadataHints;
|
|
99
|
+
/**
|
|
100
|
+
* Generate explanation based on all protocol checks
|
|
101
|
+
*/
|
|
102
|
+
private generateExplanation;
|
|
103
|
+
/**
|
|
104
|
+
* Generate recommendations based on all checks
|
|
105
|
+
*/
|
|
106
|
+
private generateRecommendations;
|
|
107
|
+
}
|
|
108
|
+
//# sourceMappingURL=ProtocolComplianceAssessor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ProtocolComplianceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ProtocolComplianceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EACL,2BAA2B,EAM3B,uBAAuB,EAGxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAOpE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAmB9D;;;GAGG;AACH,MAAM,WAAW,4BAA6B,SAAQ,2BAA2B;IAC/E,2EAA2E;IAC3E,iBAAiB,CAAC,EAAE;QAClB,mBAAmB,EAAE,aAAa,CAAC;QACnC,kBAAkB,EAAE,aAAa,CAAC;QAClC,uBAAuB,EAAE,aAAa,CAAC;KACxC,CAAC;CACH;AAED,qBAAa,0BAA2B,SAAQ,YAAY,CAAC,4BAA4B,CAAC;IACxF,OAAO,CAAC,GAAG,CAAc;gBAEb,MAAM,EAAE,uBAAuB;IAK3C;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;;OAGG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,4BAA4B,CAAC;IAmIxC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAqB9B;;OAEG;YACW,sBAAsB;IAuBpC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAsB/B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAuC7B;;OAEG;YACW,mBAAmB;IAiCjC;;OAEG;IACH,OAAO,CAAC,4BAA4B;IAYpC;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAS7B;;OAEG;YACW,wBAAwB;IA4GtC;;OAEG;YACW,uBAAuB;IA2FrC;;OAEG;YACW,4BAA4B;IAoD1C,OAAO,CAAC,yBAAyB;IAkEjC,OAAO,CAAC,uBAAuB;IAqB/B,OAAO,CAAC,sBAAsB;IA0B9B,OAAO,CAAC,qBAAqB;IAgC7B,OAAO,CAAC,oBAAoB;IA8E5B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAoC3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CAqEhC"}
|