@bryan-thompson/inspector-assessment-client 1.31.0 → 1.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-CXcl26vR.js → OAuthCallback-Cl2ANLYP.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-J9s4SF_c.js → OAuthDebugCallback-Ws62T4Ns.js} +1 -1
- package/dist/assets/{index-_HAw2b2G.js → index-DaPIdOcS.js} +4 -4
- package/dist/index.html +1 -1
- package/lib/lib/assessment/extendedTypes.d.ts +21 -0
- package/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
- package/lib/lib/assessment/resultTypes.d.ts +12 -2
- package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/lib/lib/moduleScoring.d.ts.map +1 -1
- package/lib/lib/moduleScoring.js +5 -0
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +20 -19
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +118 -152
- package/lib/services/assessment/modules/ConformanceAssessor.d.ts +4 -0
- package/lib/services/assessment/modules/ConformanceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ConformanceAssessor.js +21 -0
- package/lib/services/assessment/modules/ResourceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ResourceAssessor.js +6 -1
- package/lib/services/assessment/registry/AssessorDefinitions.d.ts +38 -0
- package/lib/services/assessment/registry/AssessorDefinitions.d.ts.map +1 -0
- package/lib/services/assessment/registry/AssessorDefinitions.js +370 -0
- package/lib/services/assessment/registry/AssessorRegistry.d.ts +124 -0
- package/lib/services/assessment/registry/AssessorRegistry.d.ts.map +1 -0
- package/lib/services/assessment/registry/AssessorRegistry.js +321 -0
- package/lib/services/assessment/registry/estimators.d.ts +93 -0
- package/lib/services/assessment/registry/estimators.d.ts.map +1 -0
- package/lib/services/assessment/registry/estimators.js +176 -0
- package/lib/services/assessment/registry/index.d.ts +13 -0
- package/lib/services/assessment/registry/index.d.ts.map +1 -0
- package/lib/services/assessment/registry/index.js +16 -0
- package/lib/services/assessment/registry/types.d.ts +180 -0
- package/lib/services/assessment/registry/types.d.ts.map +1 -0
- package/lib/services/assessment/registry/types.js +35 -0
- package/package.json +1 -1
|
@@ -6,34 +6,9 @@
|
|
|
6
6
|
* @module AssessmentOrchestrator
|
|
7
7
|
*/
|
|
8
8
|
import { DEFAULT_ASSESSMENT_CONFIG, } from "../../lib/assessmentTypes.js";
|
|
9
|
-
// Core assessment modules
|
|
10
|
-
import { FunctionalityAssessor } from "./modules/FunctionalityAssessor.js";
|
|
11
|
-
import { SecurityAssessor } from "./modules/SecurityAssessor.js";
|
|
12
|
-
import { DocumentationAssessor } from "./modules/DocumentationAssessor.js";
|
|
13
|
-
import { ErrorHandlingAssessor } from "./modules/ErrorHandlingAssessor.js";
|
|
14
|
-
import { UsabilityAssessor } from "./modules/UsabilityAssessor.js";
|
|
15
|
-
// Extended assessment modules - unified protocol compliance
|
|
16
|
-
import { ProtocolComplianceAssessor } from "./modules/ProtocolComplianceAssessor.js";
|
|
17
|
-
// New MCP Directory Compliance Gap assessors
|
|
18
|
-
import { AUPComplianceAssessor } from "./modules/AUPComplianceAssessor.js";
|
|
19
|
-
import { ToolAnnotationAssessor } from "./modules/ToolAnnotationAssessor.js";
|
|
20
|
-
import { ProhibitedLibrariesAssessor } from "./modules/ProhibitedLibrariesAssessor.js";
|
|
21
|
-
import { ManifestValidationAssessor } from "./modules/ManifestValidationAssessor.js";
|
|
22
|
-
import { PortabilityAssessor } from "./modules/PortabilityAssessor.js";
|
|
23
|
-
import { ExternalAPIScannerAssessor } from "./modules/ExternalAPIScannerAssessor.js";
|
|
24
|
-
import { TemporalAssessor } from "./modules/TemporalAssessor.js";
|
|
25
|
-
import { AuthenticationAssessor } from "./modules/AuthenticationAssessor.js";
|
|
26
|
-
// New capability assessors
|
|
27
|
-
import { ResourceAssessor } from "./modules/ResourceAssessor.js";
|
|
28
|
-
import { PromptAssessor } from "./modules/PromptAssessor.js";
|
|
29
|
-
import { CrossCapabilitySecurityAssessor } from "./modules/CrossCapabilitySecurityAssessor.js";
|
|
30
|
-
// Code quality assessors
|
|
31
|
-
import { FileModularizationAssessor } from "./modules/FileModularizationAssessor.js";
|
|
32
|
-
// Official MCP conformance testing
|
|
33
|
-
import { ConformanceAssessor } from "./modules/ConformanceAssessor.js";
|
|
34
9
|
// Note: ProtocolConformanceAssessor merged into ProtocolComplianceAssessor (v1.25.2)
|
|
35
|
-
// Pattern configuration for tool annotation assessment
|
|
36
|
-
|
|
10
|
+
// Pattern configuration for tool annotation assessment now handled by registry (Issue #91)
|
|
11
|
+
// See AssessorDefinitions.ts customSetup for ToolAnnotationAssessor
|
|
37
12
|
// Claude Code integration for intelligent analysis
|
|
38
13
|
import { ClaudeCodeBridge, FULL_CLAUDE_CODE_CONFIG, } from "./lib/claudeCodeBridge.js";
|
|
39
14
|
import { TestDataGenerator } from "./TestDataGenerator.js";
|
|
@@ -41,6 +16,10 @@ import { TestDataGenerator } from "./TestDataGenerator.js";
|
|
|
41
16
|
import { createLogger, DEFAULT_LOGGING_CONFIG } from "./lib/logger.js";
|
|
42
17
|
// Extracted helpers for testability
|
|
43
18
|
import { emitModuleStartedEvent, emitModuleProgress, determineOverallStatus, generateSummary, generateRecommendations, } from "./orchestratorHelpers.js";
|
|
19
|
+
// Registry pattern for assessor management (Issue #91)
|
|
20
|
+
import { AssessorRegistry, ASSESSOR_DEFINITIONS, } from "./registry/index.js";
|
|
21
|
+
// Module scoring for dual-key output (Issue #124)
|
|
22
|
+
import { calculateModuleScore } from "../../lib/moduleScoring.js";
|
|
44
23
|
/**
|
|
45
24
|
* Main orchestrator class for running MCP server assessments
|
|
46
25
|
*
|
|
@@ -61,31 +40,71 @@ export class AssessmentOrchestrator {
|
|
|
61
40
|
// Claude Code Bridge for intelligent analysis
|
|
62
41
|
claudeBridge;
|
|
63
42
|
claudeEnabled = false;
|
|
64
|
-
//
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
//
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
43
|
+
// Registry for assessor management (Issue #91)
|
|
44
|
+
// Delegates construction, test count aggregation, and Claude bridge wiring
|
|
45
|
+
registry;
|
|
46
|
+
// ============================================================================
|
|
47
|
+
// Private getters for backward compatibility with tests
|
|
48
|
+
// Tests access these via type assertions: (orchestrator as any).assessorName
|
|
49
|
+
// These delegate to the registry to maintain a single source of truth
|
|
50
|
+
// ============================================================================
|
|
51
|
+
get functionalityAssessor() {
|
|
52
|
+
return this.registry.getAssessor("functionality");
|
|
53
|
+
}
|
|
54
|
+
get securityAssessor() {
|
|
55
|
+
return this.registry.getAssessor("security");
|
|
56
|
+
}
|
|
57
|
+
get documentationAssessor() {
|
|
58
|
+
return this.registry.getAssessor("documentation");
|
|
59
|
+
}
|
|
60
|
+
get errorHandlingAssessor() {
|
|
61
|
+
return this.registry.getAssessor("errorHandling");
|
|
62
|
+
}
|
|
63
|
+
get usabilityAssessor() {
|
|
64
|
+
return this.registry.getAssessor("usability");
|
|
65
|
+
}
|
|
66
|
+
get protocolComplianceAssessor() {
|
|
67
|
+
return this.registry.getAssessor("protocolCompliance");
|
|
68
|
+
}
|
|
69
|
+
get aupComplianceAssessor() {
|
|
70
|
+
return this.registry.getAssessor("aupCompliance");
|
|
71
|
+
}
|
|
72
|
+
get toolAnnotationAssessor() {
|
|
73
|
+
return this.registry.getAssessor("toolAnnotations");
|
|
74
|
+
}
|
|
75
|
+
get prohibitedLibrariesAssessor() {
|
|
76
|
+
return this.registry.getAssessor("prohibitedLibraries");
|
|
77
|
+
}
|
|
78
|
+
get manifestValidationAssessor() {
|
|
79
|
+
return this.registry.getAssessor("manifestValidation");
|
|
80
|
+
}
|
|
81
|
+
get portabilityAssessor() {
|
|
82
|
+
return this.registry.getAssessor("portability");
|
|
83
|
+
}
|
|
84
|
+
get externalAPIScannerAssessor() {
|
|
85
|
+
return this.registry.getAssessor("externalAPIScanner");
|
|
86
|
+
}
|
|
87
|
+
get temporalAssessor() {
|
|
88
|
+
return this.registry.getAssessor("temporal");
|
|
89
|
+
}
|
|
90
|
+
get authenticationAssessor() {
|
|
91
|
+
return this.registry.getAssessor("authentication");
|
|
92
|
+
}
|
|
93
|
+
get resourceAssessor() {
|
|
94
|
+
return this.registry.getAssessor("resources");
|
|
95
|
+
}
|
|
96
|
+
get promptAssessor() {
|
|
97
|
+
return this.registry.getAssessor("prompts");
|
|
98
|
+
}
|
|
99
|
+
get crossCapabilityAssessor() {
|
|
100
|
+
return this.registry.getAssessor("crossCapability");
|
|
101
|
+
}
|
|
102
|
+
get fileModularizationAssessor() {
|
|
103
|
+
return this.registry.getAssessor("fileModularization");
|
|
104
|
+
}
|
|
105
|
+
get conformanceAssessor() {
|
|
106
|
+
return this.registry.getAssessor("conformance");
|
|
107
|
+
}
|
|
89
108
|
// Note: protocolConformanceAssessor merged into protocolComplianceAssessor (v1.25.2)
|
|
90
109
|
constructor(config = {}) {
|
|
91
110
|
this.config = { ...DEFAULT_ASSESSMENT_CONFIG, ...config };
|
|
@@ -104,97 +123,17 @@ export class AssessmentOrchestrator {
|
|
|
104
123
|
if (this.config.claudeCode?.enabled) {
|
|
105
124
|
this.initializeClaudeBridge(this.config.claudeCode);
|
|
106
125
|
}
|
|
107
|
-
// Initialize
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
if (this.config.assessmentCategories?.documentation !== false) {
|
|
119
|
-
this.documentationAssessor = new DocumentationAssessor(this.config);
|
|
120
|
-
}
|
|
121
|
-
if (this.config.assessmentCategories?.errorHandling !== false) {
|
|
122
|
-
this.errorHandlingAssessor = new ErrorHandlingAssessor(this.config);
|
|
123
|
-
}
|
|
124
|
-
if (this.config.assessmentCategories?.usability !== false) {
|
|
125
|
-
this.usabilityAssessor = new UsabilityAssessor(this.config);
|
|
126
|
-
}
|
|
127
|
-
// Initialize extended assessors if enabled
|
|
128
|
-
if (this.config.enableExtendedAssessment) {
|
|
129
|
-
// Initialize unified protocol compliance assessor
|
|
130
|
-
// Supports new protocolCompliance flag and deprecated mcpSpecCompliance/protocolConformance
|
|
131
|
-
if (this.config.assessmentCategories?.protocolCompliance ||
|
|
132
|
-
this.config.assessmentCategories?.mcpSpecCompliance ||
|
|
133
|
-
this.config.assessmentCategories?.protocolConformance) {
|
|
134
|
-
this.protocolComplianceAssessor = new ProtocolComplianceAssessor(this.config);
|
|
135
|
-
}
|
|
136
|
-
// Initialize new MCP Directory Compliance Gap assessors
|
|
137
|
-
if (this.config.assessmentCategories?.aupCompliance) {
|
|
138
|
-
this.aupComplianceAssessor = new AUPComplianceAssessor(this.config);
|
|
139
|
-
// Wire up Claude bridge for semantic analysis
|
|
140
|
-
if (this.claudeBridge) {
|
|
141
|
-
this.aupComplianceAssessor.setClaudeBridge(this.claudeBridge);
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
if (this.config.assessmentCategories?.toolAnnotations) {
|
|
145
|
-
this.toolAnnotationAssessor = new ToolAnnotationAssessor(this.config);
|
|
146
|
-
// Wire up Claude bridge for behavior inference
|
|
147
|
-
if (this.claudeBridge) {
|
|
148
|
-
this.toolAnnotationAssessor.setClaudeBridge(this.claudeBridge);
|
|
149
|
-
}
|
|
150
|
-
// Load custom pattern configuration if provided
|
|
151
|
-
if (this.config.patternConfigPath) {
|
|
152
|
-
const patternConfig = loadPatternConfig(this.config.patternConfigPath, this.logger);
|
|
153
|
-
const compiledPatterns = compilePatterns(patternConfig);
|
|
154
|
-
this.toolAnnotationAssessor.setPatterns(compiledPatterns);
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
if (this.config.assessmentCategories?.prohibitedLibraries) {
|
|
158
|
-
this.prohibitedLibrariesAssessor = new ProhibitedLibrariesAssessor(this.config);
|
|
159
|
-
}
|
|
160
|
-
if (this.config.assessmentCategories?.manifestValidation) {
|
|
161
|
-
this.manifestValidationAssessor = new ManifestValidationAssessor(this.config);
|
|
162
|
-
}
|
|
163
|
-
if (this.config.assessmentCategories?.portability) {
|
|
164
|
-
this.portabilityAssessor = new PortabilityAssessor(this.config);
|
|
165
|
-
}
|
|
166
|
-
if (this.config.assessmentCategories?.externalAPIScanner) {
|
|
167
|
-
this.externalAPIScannerAssessor = new ExternalAPIScannerAssessor(this.config);
|
|
168
|
-
}
|
|
169
|
-
if (this.config.assessmentCategories?.temporal) {
|
|
170
|
-
this.temporalAssessor = new TemporalAssessor(this.config);
|
|
171
|
-
}
|
|
172
|
-
if (this.config.assessmentCategories?.authentication) {
|
|
173
|
-
this.authenticationAssessor = new AuthenticationAssessor(this.config);
|
|
174
|
-
}
|
|
175
|
-
// Initialize new capability assessors
|
|
176
|
-
if (this.config.assessmentCategories?.resources) {
|
|
177
|
-
this.resourceAssessor = new ResourceAssessor(this.config);
|
|
178
|
-
}
|
|
179
|
-
if (this.config.assessmentCategories?.prompts) {
|
|
180
|
-
this.promptAssessor = new PromptAssessor(this.config);
|
|
181
|
-
}
|
|
182
|
-
if (this.config.assessmentCategories?.crossCapability) {
|
|
183
|
-
this.crossCapabilityAssessor = new CrossCapabilitySecurityAssessor(this.config);
|
|
184
|
-
}
|
|
185
|
-
// Initialize code quality assessors
|
|
186
|
-
if (this.config.assessmentCategories?.fileModularization) {
|
|
187
|
-
this.fileModularizationAssessor = new FileModularizationAssessor(this.config);
|
|
188
|
-
}
|
|
189
|
-
// Initialize official MCP conformance testing (opt-in via --conformance flag)
|
|
190
|
-
// Requires HTTP/SSE transport with serverUrl available
|
|
191
|
-
if (this.config.assessmentCategories?.conformance) {
|
|
192
|
-
this.conformanceAssessor = new ConformanceAssessor(this.config);
|
|
193
|
-
}
|
|
194
|
-
// Note: Protocol conformance now handled by unified ProtocolComplianceAssessor above
|
|
195
|
-
}
|
|
196
|
-
// Wire up Claude bridge to TestDataGenerator for intelligent test generation
|
|
126
|
+
// Initialize registry and register all enabled assessors (Issue #91)
|
|
127
|
+
// The registry handles:
|
|
128
|
+
// - Conditional instantiation based on config flags
|
|
129
|
+
// - Deprecated flag OR logic (e.g., protocolCompliance supports 3 flags)
|
|
130
|
+
// - Custom setup (e.g., ToolAnnotationAssessor pattern config)
|
|
131
|
+
// - Claude bridge wiring for supporting assessors
|
|
132
|
+
this.registry = new AssessorRegistry(this.config);
|
|
133
|
+
this.registry.registerAll(ASSESSOR_DEFINITIONS);
|
|
134
|
+
// Wire up Claude bridge to registry (handles all supporting assessors)
|
|
197
135
|
if (this.claudeBridge) {
|
|
136
|
+
this.registry.setClaudeBridge(this.claudeBridge);
|
|
198
137
|
TestDataGenerator.setClaudeBridge(this.claudeBridge);
|
|
199
138
|
}
|
|
200
139
|
// Set logger for TestDataGenerator diagnostic output
|
|
@@ -242,17 +181,9 @@ export class AssessmentOrchestrator {
|
|
|
242
181
|
enabled: true,
|
|
243
182
|
};
|
|
244
183
|
this.initializeClaudeBridge(bridgeConfig);
|
|
245
|
-
// Wire up to
|
|
184
|
+
// Wire up to all supporting assessors via registry
|
|
246
185
|
if (this.claudeBridge) {
|
|
247
|
-
|
|
248
|
-
this.aupComplianceAssessor.setClaudeBridge(this.claudeBridge);
|
|
249
|
-
}
|
|
250
|
-
if (this.toolAnnotationAssessor) {
|
|
251
|
-
this.toolAnnotationAssessor.setClaudeBridge(this.claudeBridge);
|
|
252
|
-
}
|
|
253
|
-
if (this.securityAssessor) {
|
|
254
|
-
this.securityAssessor.setClaudeBridge(this.claudeBridge);
|
|
255
|
-
}
|
|
186
|
+
this.registry.setClaudeBridge(this.claudeBridge);
|
|
256
187
|
TestDataGenerator.setClaudeBridge(this.claudeBridge);
|
|
257
188
|
}
|
|
258
189
|
}
|
|
@@ -626,6 +557,41 @@ export class AssessmentOrchestrator {
|
|
|
626
557
|
assessmentResults.security.vulnerabilities.push(`RUG_PULL_TEMPORAL: ${detail.tool} - Tool behavior changed after invocation ${detail.firstDeviationAt}. Requires immediate manual review.`);
|
|
627
558
|
}
|
|
628
559
|
}
|
|
560
|
+
// Issue #124: Dual-key output for v2.0.0 transition
|
|
561
|
+
// Output BOTH old and new keys to maintain backward compatibility
|
|
562
|
+
// Old keys (documentation, usability, mcpSpecCompliance) will be removed in v2.0.0
|
|
563
|
+
// developerExperience (new) = documentation + usability (deprecated)
|
|
564
|
+
if (assessmentResults.documentation && assessmentResults.usability) {
|
|
565
|
+
const docScore = calculateModuleScore(assessmentResults.documentation) ?? 50;
|
|
566
|
+
const usabilityScore = calculateModuleScore(assessmentResults.usability) ?? 50;
|
|
567
|
+
const combinedStatus = determineOverallStatus({
|
|
568
|
+
documentation: assessmentResults.documentation,
|
|
569
|
+
usability: assessmentResults.usability,
|
|
570
|
+
});
|
|
571
|
+
assessmentResults.developerExperience = {
|
|
572
|
+
documentation: assessmentResults.documentation,
|
|
573
|
+
usability: assessmentResults.usability,
|
|
574
|
+
status: combinedStatus,
|
|
575
|
+
score: Math.round((docScore + usabilityScore) / 2),
|
|
576
|
+
};
|
|
577
|
+
// Emit deprecation warning for old keys
|
|
578
|
+
this.logger.warn("Output keys 'documentation' and 'usability' are deprecated. " +
|
|
579
|
+
"Use 'developerExperience' instead. These keys will be removed in v2.0.0.", {
|
|
580
|
+
deprecated: ["documentation", "usability"],
|
|
581
|
+
replacement: "developerExperience",
|
|
582
|
+
});
|
|
583
|
+
}
|
|
584
|
+
// protocolCompliance (new) = mcpSpecCompliance (deprecated)
|
|
585
|
+
if (assessmentResults.mcpSpecCompliance) {
|
|
586
|
+
assessmentResults.protocolCompliance =
|
|
587
|
+
assessmentResults.mcpSpecCompliance;
|
|
588
|
+
// Emit deprecation warning for old key
|
|
589
|
+
this.logger.warn("Output key 'mcpSpecCompliance' is deprecated. " +
|
|
590
|
+
"Use 'protocolCompliance' instead. This key will be removed in v2.0.0.", {
|
|
591
|
+
deprecated: ["mcpSpecCompliance"],
|
|
592
|
+
replacement: "protocolCompliance",
|
|
593
|
+
});
|
|
594
|
+
}
|
|
629
595
|
// Collect test counts from all assessors
|
|
630
596
|
this.totalTestsRun = this.collectTotalTestCount();
|
|
631
597
|
// Determine overall status
|
|
@@ -36,6 +36,10 @@ export declare class ConformanceAssessor extends BaseAssessor<ConformanceAssessm
|
|
|
36
36
|
* Parse checks.json file from conformance results
|
|
37
37
|
*/
|
|
38
38
|
private parseChecksFile;
|
|
39
|
+
/**
|
|
40
|
+
* Check if the MCP conformance package is available
|
|
41
|
+
*/
|
|
42
|
+
private isConformancePackageAvailable;
|
|
39
43
|
/**
|
|
40
44
|
* Cleanup temporary directory
|
|
41
45
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ConformanceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ConformanceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAOH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EACV,qBAAqB,EAGtB,MAAM,gCAAgC,CAAC;AAgCxC;;;;;GAKG;AACH,qBAAa,mBAAoB,SAAQ,YAAY,CAAC,qBAAqB,CAAC;IAC1E;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"ConformanceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ConformanceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAOH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EACV,qBAAqB,EAGtB,MAAM,gCAAgC,CAAC;AAgCxC;;;;;GAKG;AACH,qBAAa,mBAAoB,SAAQ,YAAY,CAAC,qBAAqB,CAAC;IAC1E;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAkGxE;;OAEG;YACW,WAAW;IAwEzB;;OAEG;IACH,OAAO,CAAC,cAAc;IAwBtB;;OAEG;IACH,OAAO,CAAC,eAAe;IAmBvB;;OAEG;IACH,OAAO,CAAC,6BAA6B;IAarC;;OAEG;IACH,OAAO,CAAC,cAAc;IAQtB;;OAEG;IACH,OAAO,CAAC,0BAA0B;IA+BlC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAyB3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA+C/B;;OAEG;IACH,OAAO,CAAC,mBAAmB;CAmB5B"}
|
|
@@ -49,6 +49,11 @@ export class ConformanceAssessor extends BaseAssessor {
|
|
|
49
49
|
this.logger.info("Conformance tests skipped: serverUrl not available (requires HTTP/SSE transport)");
|
|
50
50
|
return this.createSkippedResult("Server URL not available. Conformance tests require HTTP or SSE transport.");
|
|
51
51
|
}
|
|
52
|
+
// Check if conformance package is available
|
|
53
|
+
if (!this.isConformancePackageAvailable()) {
|
|
54
|
+
this.logger.warn("MCP conformance package not available. Install with: npm install -g @modelcontextprotocol/conformance");
|
|
55
|
+
return this.createSkippedResult("MCP conformance package not installed. Run: npm install -g @modelcontextprotocol/conformance");
|
|
56
|
+
}
|
|
52
57
|
this.logger.info(`Running conformance tests against: ${serverUrl}`);
|
|
53
58
|
const scenarios = [];
|
|
54
59
|
const allChecks = [];
|
|
@@ -199,6 +204,22 @@ export class ConformanceAssessor extends BaseAssessor {
|
|
|
199
204
|
return [];
|
|
200
205
|
}
|
|
201
206
|
}
|
|
207
|
+
/**
|
|
208
|
+
* Check if the MCP conformance package is available
|
|
209
|
+
*/
|
|
210
|
+
isConformancePackageAvailable() {
|
|
211
|
+
try {
|
|
212
|
+
execFileSync("npx", ["@modelcontextprotocol/conformance", "--version"], {
|
|
213
|
+
timeout: 30000,
|
|
214
|
+
stdio: "pipe",
|
|
215
|
+
encoding: "utf-8",
|
|
216
|
+
});
|
|
217
|
+
return true;
|
|
218
|
+
}
|
|
219
|
+
catch {
|
|
220
|
+
return false;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
202
223
|
/**
|
|
203
224
|
* Cleanup temporary directory
|
|
204
225
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ResourceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ResourceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,kBAAkB,EAGnB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAuQ9D,qBAAa,gBAAiB,SAAQ,YAAY;IAC1C,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA8FrE,OAAO,CAAC,yBAAyB;YAiBnB,YAAY;IAoG1B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAY/B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA4B3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;YAsBjB,oBAAoB;IAkGlC;;;OAGG;YACW,6BAA6B;IA4F3C;;;OAGG;YACW,2BAA2B;
|
|
1
|
+
{"version":3,"file":"ResourceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ResourceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,kBAAkB,EAGnB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAuQ9D,qBAAa,gBAAiB,SAAQ,YAAY;IAC1C,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA8FrE,OAAO,CAAC,yBAAyB;YAiBnB,YAAY;IAoG1B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAY/B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA4B3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;YAsBjB,oBAAoB;IAkGlC;;;OAGG;YACW,6BAA6B;IA4F3C;;;OAGG;YACW,2BAA2B;IAgEzC;;OAEG;YACW,mBAAmB;IAmEjC,OAAO,CAAC,UAAU;IAsBlB,OAAO,CAAC,kBAAkB;IAM1B,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,wBAAwB;IAIhC;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAc7B,OAAO,CAAC,yBAAyB;IAYjC,OAAO,CAAC,uBAAuB;IAqB/B,OAAO,CAAC,mBAAmB;IAoC3B,OAAO,CAAC,uBAAuB;CA+DhC"}
|
|
@@ -605,7 +605,8 @@ export class ResourceAssessor extends BaseAssessor {
|
|
|
605
605
|
baseSchemes.add("file");
|
|
606
606
|
baseSchemes.add("resource");
|
|
607
607
|
}
|
|
608
|
-
// Test hidden resource patterns
|
|
608
|
+
// Test hidden resource patterns with rate limiting to avoid overwhelming target
|
|
609
|
+
const PROBE_DELAY_MS = 50; // 50ms delay between probes
|
|
609
610
|
for (const pattern of HIDDEN_RESOURCE_PATTERNS) {
|
|
610
611
|
// For patterns with their own scheme, test directly
|
|
611
612
|
if (pattern.includes("://")) {
|
|
@@ -614,6 +615,8 @@ export class ResourceAssessor extends BaseAssessor {
|
|
|
614
615
|
if (probeResult) {
|
|
615
616
|
results.push(probeResult);
|
|
616
617
|
}
|
|
618
|
+
// Rate limit between probes
|
|
619
|
+
await new Promise((resolve) => setTimeout(resolve, PROBE_DELAY_MS));
|
|
617
620
|
}
|
|
618
621
|
else {
|
|
619
622
|
// For file paths, combine with discovered schemes
|
|
@@ -624,6 +627,8 @@ export class ResourceAssessor extends BaseAssessor {
|
|
|
624
627
|
if (probeResult) {
|
|
625
628
|
results.push(probeResult);
|
|
626
629
|
}
|
|
630
|
+
// Rate limit between probes
|
|
631
|
+
await new Promise((resolve) => setTimeout(resolve, PROBE_DELAY_MS));
|
|
627
632
|
}
|
|
628
633
|
}
|
|
629
634
|
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assessor Definitions
|
|
3
|
+
*
|
|
4
|
+
* Declarative configuration for all 19 assessors.
|
|
5
|
+
* Single source of truth for assessor registration, phase ordering,
|
|
6
|
+
* config flags, and Claude bridge support.
|
|
7
|
+
*
|
|
8
|
+
* @module assessment/registry/AssessorDefinitions
|
|
9
|
+
* @see GitHub Issue #91
|
|
10
|
+
*/
|
|
11
|
+
import type { AssessorDefinition } from "./types.js";
|
|
12
|
+
import { AssessmentPhase } from "./types.js";
|
|
13
|
+
/**
|
|
14
|
+
* All assessor definitions in phase order.
|
|
15
|
+
* This is the single source of truth for assessor registration.
|
|
16
|
+
*
|
|
17
|
+
* Order within phases:
|
|
18
|
+
* - Phase 0 (PRE): Temporal only
|
|
19
|
+
* - Phase 1 (CORE): Functionality, Security, Documentation, ErrorHandling, Usability
|
|
20
|
+
* - Phase 2 (PROTOCOL): ProtocolCompliance
|
|
21
|
+
* - Phase 3 (COMPLIANCE): AUP, Annotations, Libraries, Manifest, Portability, APIs, Auth
|
|
22
|
+
* - Phase 4 (CAPABILITY): Resources, Prompts, CrossCapability
|
|
23
|
+
* - Phase 5 (QUALITY): FileModularization, Conformance
|
|
24
|
+
*/
|
|
25
|
+
export declare const ASSESSOR_DEFINITIONS: AssessorDefinition[];
|
|
26
|
+
/**
|
|
27
|
+
* Map of assessor ID to definition for fast lookup.
|
|
28
|
+
*/
|
|
29
|
+
export declare const ASSESSOR_DEFINITION_MAP: Map<string, AssessorDefinition>;
|
|
30
|
+
/**
|
|
31
|
+
* Get assessor definitions by phase.
|
|
32
|
+
*/
|
|
33
|
+
export declare function getDefinitionsByPhase(phase: AssessmentPhase): AssessorDefinition[];
|
|
34
|
+
/**
|
|
35
|
+
* Get all phases in execution order.
|
|
36
|
+
*/
|
|
37
|
+
export declare function getOrderedPhases(): AssessmentPhase[];
|
|
38
|
+
//# sourceMappingURL=AssessorDefinitions.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AssessorDefinitions.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/registry/AssessorDefinitions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AA4D1C;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,oBAAoB,EAAE,kBAAkB,EA4SpD,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,uBAAuB,EAAE,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAEnE,CAAC;AAEF;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,eAAe,GACrB,kBAAkB,EAAE,CAEtB;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,eAAe,EAAE,CASpD"}
|