@bryan-thompson/inspector-assessment 1.43.2 → 1.43.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +1062 -224
  2. package/cli/build/assess-full.js +532 -106
  3. package/cli/build/assess-security.js +54 -90
  4. package/cli/build/lib/cli-parser.js +14 -1
  5. package/cli/build/lib/cli-parserSchemas.js +1 -0
  6. package/cli/build/lib/result-output.js +21 -0
  7. package/cli/build/profiles.js +20 -0
  8. package/cli/build/validate-testbed.js +0 -0
  9. package/cli/package.json +1 -1
  10. package/client/dist/assets/{OAuthCallback-BS8-A1sU.js → OAuthCallback-Chi58kRc.js} +1 -1
  11. package/client/dist/assets/{OAuthDebugCallback-025_TM2i.js → OAuthDebugCallback-BluD_Wxg.js} +1 -1
  12. package/client/dist/assets/{index-DEhlIjy-.js → index-KW2LwGdp.js} +4 -4
  13. package/client/dist/index.html +1 -1
  14. package/client/lib/lib/assessment/configSchemas.d.ts +64 -64
  15. package/client/lib/lib/assessment/jsonlEventSchemas.d.ts +286 -286
  16. package/client/lib/lib/assessment/resultTypes.d.ts +10 -0
  17. package/client/lib/lib/assessment/resultTypes.d.ts.map +1 -1
  18. package/client/lib/lib/assessmentTypes.d.ts +1 -20
  19. package/client/lib/lib/assessmentTypes.d.ts.map +1 -1
  20. package/client/lib/lib/assessmentTypes.js +1 -20
  21. package/client/lib/services/assessment/AssessmentOrchestrator.d.ts +57 -104
  22. package/client/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
  23. package/client/lib/services/assessment/AssessmentOrchestrator.js +298 -133
  24. package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts +25 -0
  25. package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts.map +1 -0
  26. package/client/lib/services/assessment/modules/ErrorHandlingAssessor.deprecated.js +1 -1
  27. package/client/lib/services/assessment/modules/ErrorHandlingAssessor.js +564 -0
  28. package/client/lib/services/assessment/modules/SecurityAssessor.d.ts +5 -0
  29. package/client/lib/services/assessment/modules/SecurityAssessor.d.ts.map +1 -1
  30. package/client/lib/services/assessment/modules/SecurityAssessor.js +62 -0
  31. package/client/lib/services/assessment/modules/index.d.ts +1 -1
  32. package/client/lib/services/assessment/modules/index.js +1 -1
  33. package/client/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts +15 -0
  34. package/client/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts.map +1 -1
  35. package/client/lib/services/assessment/modules/securityTests/SecurityPayloadTester.js +72 -0
  36. package/client/lib/services/assessment/modules/securityTests/factory.d.ts +2 -0
  37. package/client/lib/services/assessment/modules/securityTests/factory.d.ts.map +1 -1
  38. package/client/lib/services/assessment/modules/securityTests/factory.js +1 -0
  39. package/client/lib/services/assessment/registry/AssessorDefinitions.js +1 -1
  40. package/client/lib/services/assessment/responseValidatorSchemas.d.ts +12 -12
  41. package/client/package.json +3 -3
  42. package/package.json +4 -2
  43. package/server/package.json +1 -1
  44. package/cli/build/lib/__tests__/zodErrorFormatter.test.js +0 -282
  45. package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts +0 -109
  46. package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts.map +0 -1
  47. package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.deprecated.d.ts +0 -109
  48. package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.deprecated.d.ts.map +0 -1
  49. package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.deprecated.js +0 -852
  50. package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.js +0 -852
@@ -1,852 +0,0 @@
1
- /**
2
- * Protocol Compliance Assessor Module
3
- *
4
- * Unified module for MCP protocol compliance validation.
5
- * Merges MCPSpecComplianceAssessor and ProtocolConformanceAssessor functionality.
6
- *
7
- * Protocol Checks:
8
- * 1. JSON-RPC 2.0 Compliance - Validates request/response structure
9
- * 2. Server Info Validity - Validates initialization handshake
10
- * 3. Schema Compliance - Validates tool input schemas
11
- * 4. Error Response Format - Validates isError flag, content array structure
12
- * 5. Content Type Support - Validates valid content types (text, image, audio, resource)
13
- * 6. Structured Output Support - Checks for outputSchema usage
14
- * 7. Capabilities Compliance - Validates declared vs actual capabilities
15
- *
16
- * @module assessment/modules/ProtocolComplianceAssessor
17
- */
18
- import Ajv from "ajv";
19
- import { BaseAssessor } from "./BaseAssessor.js";
20
- // Valid MCP content types
21
- const VALID_CONTENT_TYPES = [
22
- "text",
23
- "image",
24
- "audio",
25
- "resource",
26
- "resource_link",
27
- ];
28
- export class ProtocolComplianceAssessor extends BaseAssessor {
29
- ajv;
30
- constructor(config) {
31
- super(config);
32
- this.ajv = new Ajv({ allErrors: true });
33
- }
34
- /**
35
- * Get MCP spec version from config or use default
36
- */
37
- getSpecVersion() {
38
- return this.config.mcpProtocolVersion || "2025-06";
39
- }
40
- /**
41
- * Get base URL for MCP specification
42
- */
43
- getSpecBaseUrl() {
44
- return `https://modelcontextprotocol.io/specification/${this.getSpecVersion()}`;
45
- }
46
- /**
47
- * Assess MCP Protocol Compliance - Unified Approach
48
- * Combines MCPSpecComplianceAssessor and ProtocolConformanceAssessor functionality
49
- */
50
- async assess(context) {
51
- const protocolVersion = this.extractProtocolVersion(context);
52
- const tools = context.tools;
53
- const callTool = context.callTool;
54
- // SECTION 1: Protocol Checks (from MCPSpecComplianceAssessor)
55
- const schemaCheck = this.checkSchemaCompliance(tools);
56
- const jsonRpcCheck = await this.checkJsonRpcCompliance(callTool);
57
- const errorCheck = await this.checkErrorResponses(tools, callTool);
58
- const capabilitiesCheck = this.checkCapabilitiesCompliance(context);
59
- const protocolChecks = {
60
- jsonRpcCompliance: {
61
- passed: jsonRpcCheck.passed,
62
- confidence: "high",
63
- evidence: "Verified via actual tool call",
64
- rawResponse: jsonRpcCheck.rawResponse,
65
- },
66
- serverInfoValidity: {
67
- passed: this.checkServerInfoValidity(context.serverInfo),
68
- confidence: "high",
69
- evidence: "Validated server info structure",
70
- rawResponse: context.serverInfo,
71
- },
72
- schemaCompliance: {
73
- passed: schemaCheck.passed,
74
- confidence: schemaCheck.confidence,
75
- warnings: schemaCheck.details ? [schemaCheck.details] : undefined,
76
- rawResponse: tools.map((t) => ({
77
- name: t.name,
78
- inputSchema: t.inputSchema,
79
- })),
80
- },
81
- errorResponseCompliance: {
82
- passed: errorCheck.passed,
83
- confidence: "high",
84
- evidence: "Tested error handling with invalid parameters",
85
- rawResponse: errorCheck.rawResponse,
86
- },
87
- structuredOutputSupport: (() => {
88
- const { coverage, toolResults } = this.analyzeOutputSchemaCoverage(tools);
89
- return {
90
- passed: coverage.withOutputSchema > 0,
91
- confidence: "high",
92
- evidence: `${coverage.withOutputSchema}/${coverage.totalTools} tools have outputSchema (${coverage.coveragePercent}%)`,
93
- coverage, // Issue #64: Detailed coverage metrics
94
- toolResults, // Issue #64: Per-tool analysis
95
- rawResponse: tools.map((t) => ({
96
- name: t.name,
97
- hasOutputSchema: !!t.outputSchema,
98
- outputSchema: t.outputSchema,
99
- })),
100
- };
101
- })(),
102
- capabilitiesCompliance: {
103
- passed: capabilitiesCheck.passed,
104
- confidence: capabilitiesCheck.confidence,
105
- evidence: capabilitiesCheck.evidence,
106
- warnings: capabilitiesCheck.warnings,
107
- rawResponse: capabilitiesCheck.rawResponse,
108
- },
109
- };
110
- // SECTION 2: Conformance Checks (from ProtocolConformanceAssessor)
111
- const conformanceChecks = {
112
- errorResponseFormat: await this.checkErrorResponseFormat(context),
113
- contentTypeSupport: await this.checkContentTypeSupport(context),
114
- initializationHandshake: await this.checkInitializationHandshake(context),
115
- };
116
- // SECTION 3: Metadata Hints (LOW CONFIDENCE - not tested, just parsed)
117
- const metadataHints = this.extractMetadataHints(context);
118
- // Calculate score based on all protocol checks (reliable)
119
- const allChecks = [
120
- ...Object.values(protocolChecks),
121
- ...Object.values(conformanceChecks),
122
- ];
123
- const passedCount = allChecks.filter((c) => c.passed).length;
124
- const totalChecks = allChecks.length;
125
- const complianceScore = (passedCount / totalChecks) * 100;
126
- // Track test count
127
- this.testCount = totalChecks;
128
- // Log score/check consistency for debugging
129
- this.logger.info(`Protocol Compliance: ${passedCount}/${totalChecks} checks passed (${complianceScore.toFixed(1)}%)`);
130
- // Determine status based on protocol checks only
131
- let status;
132
- if (!protocolChecks.serverInfoValidity.passed) {
133
- status = "FAIL";
134
- }
135
- else if (complianceScore >= 90) {
136
- status = "PASS";
137
- }
138
- else if (complianceScore >= 70) {
139
- status = "NEED_MORE_INFO";
140
- }
141
- else {
142
- status = "FAIL";
143
- }
144
- const explanation = this.generateExplanation(complianceScore, protocolChecks, conformanceChecks);
145
- const recommendations = this.generateRecommendations(protocolChecks, conformanceChecks, metadataHints);
146
- // Legacy fields for backward compatibility
147
- const transportCompliance = this.assessTransportCompliance(context);
148
- const oauthImplementation = this.assessOAuthCompliance(context);
149
- const annotationSupport = this.assessAnnotationSupport(context);
150
- const streamingSupport = this.assessStreamingSupport(context);
151
- return {
152
- protocolVersion,
153
- protocolChecks,
154
- conformanceChecks,
155
- metadataHints,
156
- status,
157
- complianceScore,
158
- explanation,
159
- recommendations,
160
- // Legacy fields (deprecated but maintained for backward compatibility)
161
- transportCompliance,
162
- oauthImplementation,
163
- annotationSupport,
164
- streamingSupport,
165
- };
166
- }
167
- /**
168
- * Extract protocol version from context
169
- */
170
- extractProtocolVersion(context) {
171
- const metadata = context.serverInfo?.metadata;
172
- const protocolVersion = metadata?.protocolVersion;
173
- if (protocolVersion) {
174
- this.logger.info(`Using protocol version from metadata: ${protocolVersion}`);
175
- return protocolVersion;
176
- }
177
- if (context.serverInfo?.version) {
178
- this.logger.info(`Using server version as protocol version: ${context.serverInfo.version}`);
179
- return context.serverInfo.version;
180
- }
181
- this.logger.info("No protocol version information available, using default");
182
- return "2025-06-18";
183
- }
184
- /**
185
- * Check JSON-RPC 2.0 compliance
186
- */
187
- async checkJsonRpcCompliance(callTool) {
188
- try {
189
- const result = await callTool("list", {});
190
- const hasValidStructure = result !== null &&
191
- (Array.isArray(result.content) || result.isError !== undefined);
192
- return { passed: hasValidStructure, rawResponse: result };
193
- }
194
- catch (error) {
195
- const errorMessage = error instanceof Error ? error.message : String(error);
196
- const isStructuredError = errorMessage.includes("MCP error") ||
197
- errorMessage.includes("jsonrpc") ||
198
- errorMessage.includes("-32");
199
- return { passed: isStructuredError, rawResponse: error };
200
- }
201
- }
202
- /**
203
- * Check if server info is valid and properly formatted
204
- */
205
- checkServerInfoValidity(serverInfo) {
206
- if (!serverInfo) {
207
- return true; // No server info is acceptable (optional)
208
- }
209
- if (serverInfo.name !== undefined && serverInfo.name !== null) {
210
- if (typeof serverInfo.name !== "string") {
211
- this.logger.info("Server info name is not a string");
212
- return false;
213
- }
214
- }
215
- if (serverInfo.metadata !== undefined && serverInfo.metadata !== null) {
216
- if (typeof serverInfo.metadata !== "object") {
217
- this.logger.info("Server info metadata is not an object");
218
- return false;
219
- }
220
- }
221
- return true;
222
- }
223
- /**
224
- * Check schema compliance for all tools
225
- */
226
- checkSchemaCompliance(tools) {
227
- try {
228
- let hasErrors = false;
229
- const errors = [];
230
- for (const tool of tools) {
231
- if (tool.inputSchema) {
232
- const isValid = this.ajv.validateSchema(tool.inputSchema);
233
- if (!isValid) {
234
- hasErrors = true;
235
- const errorMsg = `${tool.name}: ${JSON.stringify(this.ajv.errors)}`;
236
- errors.push(errorMsg);
237
- this.logger.warn(`Invalid schema for tool ${tool.name}`, {
238
- errors: this.ajv.errors,
239
- });
240
- }
241
- }
242
- }
243
- return {
244
- passed: !hasErrors,
245
- confidence: hasErrors ? "low" : "high",
246
- details: hasErrors ? errors.join("; ") : undefined,
247
- };
248
- }
249
- catch (error) {
250
- this.logger.error("Schema compliance check failed", {
251
- error: String(error),
252
- });
253
- return {
254
- passed: false,
255
- confidence: "low",
256
- details: String(error),
257
- };
258
- }
259
- }
260
- /**
261
- * Check error response compliance (basic check from MCPSpec)
262
- */
263
- async checkErrorResponses(tools, callTool) {
264
- try {
265
- if (tools.length === 0)
266
- return { passed: true, rawResponse: "No tools to test" };
267
- const testTool = tools[0];
268
- try {
269
- const result = await callTool(testTool.name, { invalid_param: "test" });
270
- const isErrorResponse = result.isError === true;
271
- const hasContent = Array.isArray(result.content);
272
- const passed = (isErrorResponse && hasContent) || (!isErrorResponse && hasContent);
273
- return { passed, rawResponse: result };
274
- }
275
- catch (error) {
276
- const errorMessage = error instanceof Error ? error.message : String(error);
277
- const isStructuredError = errorMessage.includes("MCP error") ||
278
- errorMessage.includes("-32") ||
279
- errorMessage.includes("jsonrpc");
280
- return { passed: isStructuredError, rawResponse: error };
281
- }
282
- }
283
- catch (error) {
284
- return { passed: false, rawResponse: error };
285
- }
286
- }
287
- /**
288
- * Analyze outputSchema coverage across all tools (Issue #64)
289
- * Returns detailed coverage metrics instead of just a boolean
290
- */
291
- analyzeOutputSchemaCoverage(tools) {
292
- const toolResults = [];
293
- const toolsWithoutSchema = [];
294
- let withOutputSchema = 0;
295
- let withoutOutputSchema = 0;
296
- for (const tool of tools) {
297
- const hasOutputSchema = !!tool.outputSchema;
298
- if (hasOutputSchema) {
299
- withOutputSchema++;
300
- }
301
- else {
302
- withoutOutputSchema++;
303
- toolsWithoutSchema.push(tool.name);
304
- }
305
- toolResults.push({
306
- toolName: tool.name,
307
- hasOutputSchema,
308
- outputSchema: tool.outputSchema,
309
- });
310
- }
311
- const totalTools = tools.length;
312
- const coveragePercent = totalTools > 0 ? Math.round((withOutputSchema / totalTools) * 100) : 0;
313
- this.logger.info(`Structured output support: ${withOutputSchema}/${totalTools} tools (${coveragePercent}%)`);
314
- const coverage = {
315
- totalTools,
316
- withOutputSchema,
317
- withoutOutputSchema,
318
- coveragePercent,
319
- toolsWithoutSchema,
320
- status: coveragePercent === 100 ? "PASS" : "INFO",
321
- recommendation: coveragePercent < 100
322
- ? "Add outputSchema to tools for client-side response validation"
323
- : undefined,
324
- };
325
- return { coverage, toolResults };
326
- }
327
- /**
328
- * Check capabilities compliance
329
- */
330
- checkCapabilitiesCompliance(context) {
331
- const warnings = [];
332
- const capabilities = context.serverCapabilities;
333
- if (!capabilities) {
334
- return {
335
- passed: true,
336
- confidence: "medium",
337
- evidence: "No server capabilities declared (optional)",
338
- rawResponse: undefined,
339
- };
340
- }
341
- if (capabilities.tools) {
342
- if (context.tools.length === 0) {
343
- warnings.push("Declared tools capability but no tools registered");
344
- }
345
- this.testCount++;
346
- }
347
- if (capabilities.resources) {
348
- if (!context.resources || context.resources.length === 0) {
349
- if (!context.readResource) {
350
- warnings.push("Declared resources capability but no resources data provided for validation");
351
- }
352
- }
353
- this.testCount++;
354
- }
355
- if (capabilities.prompts) {
356
- if (!context.prompts || context.prompts.length === 0) {
357
- if (!context.getPrompt) {
358
- warnings.push("Declared prompts capability but no prompts data provided for validation");
359
- }
360
- }
361
- this.testCount++;
362
- }
363
- const passed = warnings.length === 0;
364
- const confidence = warnings.length === 0 ? "high" : "medium";
365
- return {
366
- passed,
367
- confidence,
368
- evidence: passed
369
- ? "All declared capabilities have corresponding implementations"
370
- : `Capability validation issues: ${warnings.join("; ")}`,
371
- warnings: warnings.length > 0 ? warnings : undefined,
372
- rawResponse: capabilities,
373
- };
374
- }
375
- // ============================================================================
376
- // Conformance-style checks (from ProtocolConformanceAssessor)
377
- // ============================================================================
378
- /**
379
- * Select representative tools for testing (first, middle, last for diversity)
380
- */
381
- selectToolsForTesting(tools, maxTools = 3) {
382
- if (tools.length <= maxTools)
383
- return tools;
384
- const indices = [0, Math.floor(tools.length / 2), tools.length - 1];
385
- return [...new Set(indices)].slice(0, maxTools).map((i) => tools[i]);
386
- }
387
- /**
388
- * Check Error Response Format (conformance-style with multi-tool testing)
389
- */
390
- async checkErrorResponseFormat(context) {
391
- const testTools = this.selectToolsForTesting(context.tools, 3);
392
- if (testTools.length === 0) {
393
- return {
394
- passed: false,
395
- confidence: "low",
396
- evidence: "No tools available to test error response format",
397
- specReference: `${this.getSpecBaseUrl()}/basic/lifecycle`,
398
- warnings: ["Cannot validate error format without tools"],
399
- };
400
- }
401
- const results = [];
402
- for (const testTool of testTools) {
403
- try {
404
- const result = await this.executeWithTimeout(context.callTool(testTool.name, {
405
- __test_invalid_param__: "should_cause_error",
406
- }), this.config.testTimeout);
407
- const contentArray = Array.isArray(result.content)
408
- ? result.content
409
- : [];
410
- const validations = {
411
- hasIsErrorFlag: result.isError === true,
412
- hasContentArray: Array.isArray(result.content),
413
- contentNotEmpty: contentArray.length > 0,
414
- firstContentHasType: contentArray[0]?.type !== undefined,
415
- firstContentIsTextOrResource: contentArray[0]?.type === "text" ||
416
- contentArray[0]?.type === "resource",
417
- hasErrorMessage: typeof contentArray[0]?.text === "string" &&
418
- contentArray[0].text.length > 0,
419
- };
420
- if (!result.isError && contentArray.length > 0) {
421
- results.push({
422
- toolName: testTool.name,
423
- passed: true,
424
- isErrorResponse: false,
425
- validations,
426
- });
427
- }
428
- else {
429
- const passedValidations = Object.values(validations).filter((v) => v);
430
- const allPassed = passedValidations.length === Object.keys(validations).length;
431
- results.push({
432
- toolName: testTool.name,
433
- passed: allPassed,
434
- isErrorResponse: true,
435
- validations,
436
- });
437
- }
438
- }
439
- catch (error) {
440
- results.push({
441
- toolName: testTool.name,
442
- passed: false,
443
- isErrorResponse: false,
444
- error: error instanceof Error ? error.message : String(error),
445
- });
446
- }
447
- }
448
- const errorResponseResults = results.filter((r) => r.isErrorResponse);
449
- const passedCount = results.filter((r) => r.passed).length;
450
- const allPassed = passedCount === results.length;
451
- let confidence;
452
- if (errorResponseResults.length === 0) {
453
- confidence = "medium";
454
- }
455
- else if (allPassed) {
456
- confidence = "high";
457
- }
458
- else {
459
- confidence = "medium";
460
- }
461
- return {
462
- passed: allPassed,
463
- confidence,
464
- evidence: `Tested ${results.length} tool(s): ${passedCount}/${results.length} passed error format validation`,
465
- specReference: `${this.getSpecBaseUrl()}/basic/lifecycle`,
466
- details: {
467
- toolResults: results,
468
- testedToolCount: results.length,
469
- errorResponseCount: errorResponseResults.length,
470
- },
471
- warnings: allPassed
472
- ? undefined
473
- : [
474
- "Error response format issues detected in some tools",
475
- "Ensure all errors have isError: true and content array with text type",
476
- ],
477
- };
478
- }
479
- /**
480
- * Check Content Type Support
481
- */
482
- async checkContentTypeSupport(context) {
483
- try {
484
- const testTool = context.tools[0];
485
- if (!testTool) {
486
- return {
487
- passed: false,
488
- confidence: "low",
489
- evidence: "No tools available to test content types",
490
- specReference: `${this.getSpecBaseUrl()}/server/tools`,
491
- };
492
- }
493
- const schema = testTool.inputSchema;
494
- const hasRequiredParams = schema?.required &&
495
- Array.isArray(schema.required) &&
496
- schema.required.length > 0;
497
- if (hasRequiredParams) {
498
- return {
499
- passed: true,
500
- confidence: "low",
501
- evidence: "Cannot test content types without knowing valid parameters - tool has required params",
502
- specReference: `${this.getSpecBaseUrl()}/server/tools`,
503
- warnings: ["Content type validation requires valid tool parameters"],
504
- };
505
- }
506
- const result = await this.executeWithTimeout(context.callTool(testTool.name, {}), this.config.testTimeout);
507
- const contentArray = Array.isArray(result.content) ? result.content : [];
508
- const validations = {
509
- hasContentArray: Array.isArray(result.content),
510
- contentNotEmpty: contentArray.length > 0,
511
- allContentHasType: contentArray.every((c) => c.type !== undefined),
512
- validContentTypes: contentArray.every((c) => VALID_CONTENT_TYPES.includes(c.type)),
513
- };
514
- const passedValidations = Object.values(validations).filter((v) => v);
515
- const allPassed = passedValidations.length === Object.keys(validations).length;
516
- const detectedTypes = contentArray.map((c) => c.type);
517
- const invalidTypes = detectedTypes.filter((t) => !VALID_CONTENT_TYPES.includes(t));
518
- return {
519
- passed: allPassed,
520
- confidence: allPassed ? "high" : "medium",
521
- evidence: `${passedValidations.length}/${Object.keys(validations).length} content type checks passed`,
522
- specReference: `${this.getSpecBaseUrl()}/server/tools`,
523
- details: {
524
- validations,
525
- detectedContentTypes: detectedTypes,
526
- invalidContentTypes: invalidTypes.length > 0 ? invalidTypes : undefined,
527
- },
528
- warnings: invalidTypes.length > 0
529
- ? [`Invalid content types found: ${invalidTypes.join(", ")}`]
530
- : undefined,
531
- };
532
- }
533
- catch (error) {
534
- return {
535
- passed: false,
536
- confidence: "medium",
537
- evidence: "Could not test content types due to error",
538
- specReference: `${this.getSpecBaseUrl()}/server/tools`,
539
- details: {
540
- error: error instanceof Error ? error.message : String(error),
541
- },
542
- };
543
- }
544
- }
545
- /**
546
- * Check Initialization Handshake
547
- */
548
- async checkInitializationHandshake(context) {
549
- const serverInfo = context.serverInfo;
550
- const serverCapabilities = context.serverCapabilities;
551
- const validations = {
552
- hasServerInfo: serverInfo !== undefined && serverInfo !== null,
553
- hasServerName: typeof serverInfo?.name === "string" && serverInfo.name.length > 0,
554
- hasServerVersion: typeof serverInfo?.version === "string" &&
555
- serverInfo.version.length > 0,
556
- hasCapabilities: serverCapabilities !== undefined,
557
- };
558
- const passedValidations = Object.values(validations).filter((v) => v);
559
- const allPassed = passedValidations.length === Object.keys(validations).length;
560
- const hasMinimumInfo = validations.hasServerInfo && validations.hasServerName;
561
- return {
562
- passed: hasMinimumInfo,
563
- confidence: allPassed ? "high" : "medium",
564
- evidence: `${passedValidations.length}/${Object.keys(validations).length} initialization checks passed`,
565
- specReference: `${this.getSpecBaseUrl()}/basic/lifecycle`,
566
- details: {
567
- validations,
568
- serverInfo: {
569
- name: serverInfo?.name,
570
- version: serverInfo?.version,
571
- hasCapabilities: !!serverCapabilities,
572
- },
573
- },
574
- warnings: !allPassed
575
- ? [
576
- !validations.hasServerVersion
577
- ? "Server should provide version for better compatibility tracking"
578
- : undefined,
579
- !validations.hasCapabilities
580
- ? "Server should declare capabilities for feature negotiation"
581
- : undefined,
582
- ].filter(Boolean)
583
- : undefined,
584
- };
585
- }
586
- // ============================================================================
587
- // Legacy compatibility methods (from MCPSpecComplianceAssessor)
588
- // ============================================================================
589
- assessTransportCompliance(context) {
590
- // Issue #172: Check source-based transport detection first
591
- // This fixes incorrect FAIL for valid stdio servers without serverInfo metadata
592
- if (context.transportDetection?.supportsStdio) {
593
- return {
594
- supportsStreamableHTTP: context.transportDetection.supportsHTTP,
595
- deprecatedSSE: context.transportDetection.supportsSSE,
596
- transportValidation: "passed",
597
- supportsStdio: true,
598
- supportsSSE: context.transportDetection.supportsSSE,
599
- confidence: context.transportDetection.confidence,
600
- detectionMethod: "source-code-analysis",
601
- requiresManualCheck: false,
602
- transportEvidence: context.transportDetection.evidence.map((e) => `${e.source}: ${e.detail}`),
603
- };
604
- }
605
- // Also check HTTP-only detection (no serverInfo but HTTP transport detected)
606
- if (context.transportDetection?.supportsHTTP &&
607
- !context.transportDetection?.supportsStdio) {
608
- return {
609
- supportsStreamableHTTP: true,
610
- deprecatedSSE: context.transportDetection.supportsSSE,
611
- transportValidation: "passed",
612
- supportsStdio: false,
613
- supportsSSE: context.transportDetection.supportsSSE,
614
- confidence: context.transportDetection.confidence,
615
- detectionMethod: "source-code-analysis",
616
- requiresManualCheck: false,
617
- transportEvidence: context.transportDetection.evidence.map((e) => `${e.source}: ${e.detail}`),
618
- };
619
- }
620
- if (!context.serverInfo) {
621
- return {
622
- supportsStreamableHTTP: false,
623
- deprecatedSSE: false,
624
- transportValidation: "failed",
625
- supportsStdio: false,
626
- supportsSSE: false,
627
- confidence: "low",
628
- detectionMethod: "manual-required",
629
- requiresManualCheck: true,
630
- manualVerificationSteps: [
631
- "Test STDIO: Run `npm start`, send JSON-RPC initialize request",
632
- "Test HTTP: Set HTTP_STREAMABLE_SERVER=true, run `npm start`, test /health endpoint",
633
- "Check if framework handles transports internally",
634
- ],
635
- };
636
- }
637
- const metadata = context.serverInfo?.metadata;
638
- const transport = metadata?.transport;
639
- const hasTransportMetadata = !!transport;
640
- const supportsStreamableHTTP = transport === "streamable-http" ||
641
- transport === "http" ||
642
- (!transport && !!context.serverInfo);
643
- const deprecatedSSE = transport === "sse";
644
- let transportValidation = "passed";
645
- if (deprecatedSSE) {
646
- transportValidation = "partial";
647
- }
648
- else if (transport &&
649
- transport !== "streamable-http" &&
650
- transport !== "http" &&
651
- transport !== "stdio") {
652
- transportValidation = "failed";
653
- }
654
- const confidence = hasTransportMetadata ? "medium" : "low";
655
- const requiresManualCheck = !hasTransportMetadata;
656
- return {
657
- supportsStreamableHTTP,
658
- deprecatedSSE,
659
- transportValidation,
660
- supportsStdio: transport === "stdio" || !transport,
661
- supportsSSE: deprecatedSSE,
662
- confidence,
663
- detectionMethod: hasTransportMetadata ? "automated" : "manual-required",
664
- requiresManualCheck,
665
- manualVerificationSteps: requiresManualCheck
666
- ? [
667
- "Test STDIO: Run `npm start`, send JSON-RPC initialize request via stdin",
668
- "Test HTTP: Set HTTP_STREAMABLE_SERVER=true, run `npm start`, curl http://localhost:3000/health",
669
- ]
670
- : undefined,
671
- };
672
- }
673
- assessAnnotationSupport(context) {
674
- const metadata = context.serverInfo?.metadata;
675
- const annotations = metadata?.annotations;
676
- const supportsAnnotations = annotations?.supported || false;
677
- const customAnnotations = annotations?.types || [];
678
- return {
679
- supportsReadOnlyHint: supportsAnnotations,
680
- supportsDestructiveHint: supportsAnnotations,
681
- supportsTitleAnnotation: supportsAnnotations,
682
- customAnnotations: customAnnotations.length > 0 ? customAnnotations : undefined,
683
- };
684
- }
685
- assessStreamingSupport(context) {
686
- const metadata = context.serverInfo?.metadata;
687
- const streaming = metadata?.streaming;
688
- const supportsStreaming = streaming?.supported || false;
689
- const protocol = streaming?.protocol;
690
- const validProtocols = ["http-streaming", "sse", "websocket"];
691
- const streamingProtocol = protocol && validProtocols.includes(protocol)
692
- ? protocol
693
- : supportsStreaming
694
- ? "http-streaming"
695
- : undefined;
696
- return {
697
- supportsStreaming,
698
- streamingProtocol,
699
- };
700
- }
701
- assessOAuthCompliance(context) {
702
- const metadata = context.serverInfo?.metadata;
703
- const oauthConfig = metadata?.oauth;
704
- if (!oauthConfig || !oauthConfig.enabled) {
705
- return undefined;
706
- }
707
- const resourceIndicators = [];
708
- if (oauthConfig.resourceIndicators) {
709
- const indicators = oauthConfig.resourceIndicators;
710
- resourceIndicators.push(...indicators);
711
- }
712
- if (oauthConfig.resourceServer) {
713
- resourceIndicators.push(oauthConfig.resourceServer);
714
- }
715
- return {
716
- implementsResourceServer: oauthConfig.enabled === true,
717
- supportsRFC8707: oauthConfig.supportsRFC8707 || false,
718
- resourceIndicators,
719
- tokenValidation: oauthConfig.tokenValidation !== false,
720
- scopeEnforcement: oauthConfig.scopeEnforcement !== false,
721
- supportsOAuth: oauthConfig.enabled === true,
722
- supportsPKCE: oauthConfig.supportsPKCE || false,
723
- };
724
- }
725
- extractMetadataHints(context) {
726
- const metadata = context.serverInfo?.metadata;
727
- if (!metadata && !context.serverInfo) {
728
- return undefined;
729
- }
730
- const transport = metadata?.transport;
731
- const transportHints = {
732
- detectedTransport: transport,
733
- supportsStdio: transport === "stdio" || !transport,
734
- supportsHTTP: transport === "http" ||
735
- transport === "streamable-http" ||
736
- (!transport && !!context.serverInfo),
737
- supportsSSE: transport === "sse",
738
- detectionMethod: (transport ? "metadata" : "assumed"),
739
- };
740
- const oauthConfig = metadata?.oauth;
741
- const oauthHints = oauthConfig
742
- ? {
743
- hasOAuthConfig: true,
744
- supportsOAuth: oauthConfig.enabled === true,
745
- supportsPKCE: oauthConfig.supportsPKCE || false,
746
- resourceIndicators: oauthConfig.resourceIndicators
747
- ? oauthConfig.resourceIndicators
748
- : undefined,
749
- }
750
- : undefined;
751
- const annotations = metadata?.annotations;
752
- const annotationHints = {
753
- supportsReadOnlyHint: annotations?.supported || false,
754
- supportsDestructiveHint: annotations?.supported || false,
755
- supportsTitleAnnotation: annotations?.supported || false,
756
- customAnnotations: annotations?.types
757
- ? annotations.types
758
- : undefined,
759
- };
760
- const streaming = metadata?.streaming;
761
- const streamingHints = {
762
- supportsStreaming: streaming?.supported || false,
763
- streamingProtocol: streaming?.protocol &&
764
- ["http-streaming", "sse", "websocket"].includes(streaming.protocol)
765
- ? streaming.protocol
766
- : undefined,
767
- };
768
- return {
769
- confidence: "low",
770
- requiresManualVerification: true,
771
- transportHints,
772
- oauthHints,
773
- annotationHints,
774
- streamingHints,
775
- manualVerificationSteps: [
776
- "Test STDIO transport: Run `npm start`, send JSON-RPC initialize request via stdin",
777
- "Test HTTP transport: Set HTTP_STREAMABLE_SERVER=true, run `npm start`, curl http://localhost:3000/health",
778
- "Verify OAuth endpoints if configured",
779
- ],
780
- };
781
- }
782
- /**
783
- * Generate explanation based on all protocol checks
784
- */
785
- generateExplanation(complianceScore, protocolChecks, conformanceChecks) {
786
- const failedChecks = [];
787
- if (!protocolChecks.jsonRpcCompliance.passed)
788
- failedChecks.push("JSON-RPC compliance");
789
- if (!protocolChecks.serverInfoValidity.passed)
790
- failedChecks.push("server info validity");
791
- if (!protocolChecks.schemaCompliance.passed)
792
- failedChecks.push("schema compliance");
793
- if (!protocolChecks.errorResponseCompliance.passed)
794
- failedChecks.push("error response compliance");
795
- Object.entries(conformanceChecks).forEach(([name, check]) => {
796
- if (!check.passed) {
797
- failedChecks.push(name
798
- .replace(/([A-Z])/g, " $1")
799
- .toLowerCase()
800
- .trim());
801
- }
802
- });
803
- if (complianceScore >= 90) {
804
- return "Excellent MCP protocol compliance. Server meets all critical requirements verified through protocol testing.";
805
- }
806
- else if (complianceScore >= 70) {
807
- return `Good MCP compliance with minor issues: ${failedChecks.join(", ")}. Review recommended before directory submission.`;
808
- }
809
- else {
810
- return `Poor MCP compliance detected. Critical issues: ${failedChecks.join(", ")}. Must fix before directory approval.`;
811
- }
812
- }
813
- /**
814
- * Generate recommendations based on all checks
815
- */
816
- generateRecommendations(protocolChecks, conformanceChecks, metadataHints) {
817
- const recommendations = [];
818
- if (!protocolChecks.jsonRpcCompliance.passed) {
819
- recommendations.push("Ensure all requests/responses follow JSON-RPC 2.0 format with proper jsonrpc, id, method/result fields.");
820
- }
821
- if (!protocolChecks.serverInfoValidity.passed) {
822
- recommendations.push("Fix serverInfo structure to include valid name and metadata fields.");
823
- }
824
- if (!protocolChecks.schemaCompliance.passed) {
825
- if (protocolChecks.schemaCompliance.confidence === "low") {
826
- recommendations.push("Schema validation warnings detected (may be false positives from Zod/TypeBox conversion).");
827
- }
828
- else {
829
- recommendations.push("Review tool schemas and ensure they follow JSON Schema specification.");
830
- }
831
- }
832
- if (!conformanceChecks.errorResponseFormat.passed) {
833
- recommendations.push("Ensure error responses include 'isError: true' flag and properly formatted content array.");
834
- }
835
- if (!conformanceChecks.contentTypeSupport.passed) {
836
- recommendations.push("Use only valid content types: text, image, audio, resource, resource_link.");
837
- }
838
- if (!conformanceChecks.initializationHandshake.passed) {
839
- recommendations.push("Ensure server provides name and version during initialization.");
840
- }
841
- if (!protocolChecks.structuredOutputSupport.passed) {
842
- recommendations.push("Consider adding outputSchema to tools for type-safe responses (optional MCP 2025-06-18 feature).");
843
- }
844
- if (metadataHints?.requiresManualVerification) {
845
- recommendations.push("Transport/OAuth/Streaming features require manual verification (metadata-based detection only).");
846
- }
847
- if (recommendations.length === 0) {
848
- recommendations.push("Excellent MCP compliance! All protocol checks passed. Server is ready for directory submission.");
849
- }
850
- return recommendations;
851
- }
852
- }