@bryan-thompson/inspector-assessment-client 1.31.0 → 1.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-CXcl26vR.js → OAuthCallback-DQqdKmh-.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-J9s4SF_c.js → OAuthDebugCallback-CZ79tNPn.js} +1 -1
- package/dist/assets/{index-_HAw2b2G.js → index-7dGKjpGX.js} +4 -4
- package/dist/index.html +1 -1
- package/lib/lib/assessment/extendedTypes.d.ts +21 -0
- package/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
- package/lib/lib/assessment/resultTypes.d.ts +12 -2
- package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/lib/lib/moduleScoring.d.ts.map +1 -1
- package/lib/lib/moduleScoring.js +5 -0
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +1 -28
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +66 -576
- package/lib/services/assessment/modules/ConformanceAssessor.d.ts +4 -0
- package/lib/services/assessment/modules/ConformanceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ConformanceAssessor.js +21 -0
- package/lib/services/assessment/modules/ResourceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ResourceAssessor.js +6 -1
- package/lib/services/assessment/registry/AssessorDefinitions.d.ts +38 -0
- package/lib/services/assessment/registry/AssessorDefinitions.d.ts.map +1 -0
- package/lib/services/assessment/registry/AssessorDefinitions.js +370 -0
- package/lib/services/assessment/registry/AssessorRegistry.d.ts +157 -0
- package/lib/services/assessment/registry/AssessorRegistry.d.ts.map +1 -0
- package/lib/services/assessment/registry/AssessorRegistry.js +364 -0
- package/lib/services/assessment/registry/estimators.d.ts +93 -0
- package/lib/services/assessment/registry/estimators.d.ts.map +1 -0
- package/lib/services/assessment/registry/estimators.js +176 -0
- package/lib/services/assessment/registry/index.d.ts +13 -0
- package/lib/services/assessment/registry/index.d.ts.map +1 -0
- package/lib/services/assessment/registry/index.js +16 -0
- package/lib/services/assessment/registry/types.d.ts +180 -0
- package/lib/services/assessment/registry/types.d.ts.map +1 -0
- package/lib/services/assessment/registry/types.js +35 -0
- package/lib/services/assessment/responseValidatorSchemas.d.ts +4 -4
- package/package.json +1 -1
|
@@ -6,41 +6,18 @@
|
|
|
6
6
|
* @module AssessmentOrchestrator
|
|
7
7
|
*/
|
|
8
8
|
import { DEFAULT_ASSESSMENT_CONFIG, } from "../../lib/assessmentTypes.js";
|
|
9
|
-
//
|
|
10
|
-
import { FunctionalityAssessor } from "./modules/FunctionalityAssessor.js";
|
|
11
|
-
import { SecurityAssessor } from "./modules/SecurityAssessor.js";
|
|
12
|
-
import { DocumentationAssessor } from "./modules/DocumentationAssessor.js";
|
|
13
|
-
import { ErrorHandlingAssessor } from "./modules/ErrorHandlingAssessor.js";
|
|
14
|
-
import { UsabilityAssessor } from "./modules/UsabilityAssessor.js";
|
|
15
|
-
// Extended assessment modules - unified protocol compliance
|
|
16
|
-
import { ProtocolComplianceAssessor } from "./modules/ProtocolComplianceAssessor.js";
|
|
17
|
-
// New MCP Directory Compliance Gap assessors
|
|
18
|
-
import { AUPComplianceAssessor } from "./modules/AUPComplianceAssessor.js";
|
|
19
|
-
import { ToolAnnotationAssessor } from "./modules/ToolAnnotationAssessor.js";
|
|
20
|
-
import { ProhibitedLibrariesAssessor } from "./modules/ProhibitedLibrariesAssessor.js";
|
|
21
|
-
import { ManifestValidationAssessor } from "./modules/ManifestValidationAssessor.js";
|
|
22
|
-
import { PortabilityAssessor } from "./modules/PortabilityAssessor.js";
|
|
23
|
-
import { ExternalAPIScannerAssessor } from "./modules/ExternalAPIScannerAssessor.js";
|
|
24
|
-
import { TemporalAssessor } from "./modules/TemporalAssessor.js";
|
|
25
|
-
import { AuthenticationAssessor } from "./modules/AuthenticationAssessor.js";
|
|
26
|
-
// New capability assessors
|
|
27
|
-
import { ResourceAssessor } from "./modules/ResourceAssessor.js";
|
|
28
|
-
import { PromptAssessor } from "./modules/PromptAssessor.js";
|
|
29
|
-
import { CrossCapabilitySecurityAssessor } from "./modules/CrossCapabilitySecurityAssessor.js";
|
|
30
|
-
// Code quality assessors
|
|
31
|
-
import { FileModularizationAssessor } from "./modules/FileModularizationAssessor.js";
|
|
32
|
-
// Official MCP conformance testing
|
|
33
|
-
import { ConformanceAssessor } from "./modules/ConformanceAssessor.js";
|
|
34
|
-
// Note: ProtocolConformanceAssessor merged into ProtocolComplianceAssessor (v1.25.2)
|
|
35
|
-
// Pattern configuration for tool annotation assessment
|
|
36
|
-
import { loadPatternConfig, compilePatterns, } from "./config/annotationPatterns.js";
|
|
9
|
+
// Note: All assessor module imports now handled by registry/AssessorDefinitions.ts (Issue #91)
|
|
37
10
|
// Claude Code integration for intelligent analysis
|
|
38
11
|
import { ClaudeCodeBridge, FULL_CLAUDE_CODE_CONFIG, } from "./lib/claudeCodeBridge.js";
|
|
39
12
|
import { TestDataGenerator } from "./TestDataGenerator.js";
|
|
40
13
|
// Structured logging
|
|
41
14
|
import { createLogger, DEFAULT_LOGGING_CONFIG } from "./lib/logger.js";
|
|
42
15
|
// Extracted helpers for testability
|
|
43
|
-
import {
|
|
16
|
+
import { determineOverallStatus, generateSummary, generateRecommendations, } from "./orchestratorHelpers.js";
|
|
17
|
+
// Registry pattern for assessor management (Issue #91)
|
|
18
|
+
import { AssessorRegistry, ASSESSOR_DEFINITIONS } from "./registry/index.js";
|
|
19
|
+
// Module scoring for dual-key output (Issue #124)
|
|
20
|
+
import { calculateModuleScore } from "../../lib/moduleScoring.js";
|
|
44
21
|
/**
|
|
45
22
|
* Main orchestrator class for running MCP server assessments
|
|
46
23
|
*
|
|
@@ -61,32 +38,9 @@ export class AssessmentOrchestrator {
|
|
|
61
38
|
// Claude Code Bridge for intelligent analysis
|
|
62
39
|
claudeBridge;
|
|
63
40
|
claudeEnabled = false;
|
|
64
|
-
//
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
documentationAssessor;
|
|
68
|
-
errorHandlingAssessor;
|
|
69
|
-
usabilityAssessor;
|
|
70
|
-
// Extended assessors - unified protocol compliance
|
|
71
|
-
protocolComplianceAssessor;
|
|
72
|
-
// New MCP Directory Compliance Gap assessors
|
|
73
|
-
aupComplianceAssessor;
|
|
74
|
-
toolAnnotationAssessor;
|
|
75
|
-
prohibitedLibrariesAssessor;
|
|
76
|
-
manifestValidationAssessor;
|
|
77
|
-
portabilityAssessor;
|
|
78
|
-
externalAPIScannerAssessor;
|
|
79
|
-
temporalAssessor;
|
|
80
|
-
authenticationAssessor;
|
|
81
|
-
// New capability assessors
|
|
82
|
-
resourceAssessor;
|
|
83
|
-
promptAssessor;
|
|
84
|
-
crossCapabilityAssessor;
|
|
85
|
-
// Code quality assessors
|
|
86
|
-
fileModularizationAssessor;
|
|
87
|
-
// Official MCP conformance testing (opt-in via --conformance flag)
|
|
88
|
-
conformanceAssessor;
|
|
89
|
-
// Note: protocolConformanceAssessor merged into protocolComplianceAssessor (v1.25.2)
|
|
41
|
+
// Registry for assessor management (Issue #91)
|
|
42
|
+
// Delegates construction, test count aggregation, and Claude bridge wiring
|
|
43
|
+
registry;
|
|
90
44
|
constructor(config = {}) {
|
|
91
45
|
this.config = { ...DEFAULT_ASSESSMENT_CONFIG, ...config };
|
|
92
46
|
// Initialize logger
|
|
@@ -104,113 +58,22 @@ export class AssessmentOrchestrator {
|
|
|
104
58
|
if (this.config.claudeCode?.enabled) {
|
|
105
59
|
this.initializeClaudeBridge(this.config.claudeCode);
|
|
106
60
|
}
|
|
107
|
-
// Initialize
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
if (this.config.assessmentCategories?.documentation !== false) {
|
|
119
|
-
this.documentationAssessor = new DocumentationAssessor(this.config);
|
|
120
|
-
}
|
|
121
|
-
if (this.config.assessmentCategories?.errorHandling !== false) {
|
|
122
|
-
this.errorHandlingAssessor = new ErrorHandlingAssessor(this.config);
|
|
123
|
-
}
|
|
124
|
-
if (this.config.assessmentCategories?.usability !== false) {
|
|
125
|
-
this.usabilityAssessor = new UsabilityAssessor(this.config);
|
|
126
|
-
}
|
|
127
|
-
// Initialize extended assessors if enabled
|
|
128
|
-
if (this.config.enableExtendedAssessment) {
|
|
129
|
-
// Initialize unified protocol compliance assessor
|
|
130
|
-
// Supports new protocolCompliance flag and deprecated mcpSpecCompliance/protocolConformance
|
|
131
|
-
if (this.config.assessmentCategories?.protocolCompliance ||
|
|
132
|
-
this.config.assessmentCategories?.mcpSpecCompliance ||
|
|
133
|
-
this.config.assessmentCategories?.protocolConformance) {
|
|
134
|
-
this.protocolComplianceAssessor = new ProtocolComplianceAssessor(this.config);
|
|
135
|
-
}
|
|
136
|
-
// Initialize new MCP Directory Compliance Gap assessors
|
|
137
|
-
if (this.config.assessmentCategories?.aupCompliance) {
|
|
138
|
-
this.aupComplianceAssessor = new AUPComplianceAssessor(this.config);
|
|
139
|
-
// Wire up Claude bridge for semantic analysis
|
|
140
|
-
if (this.claudeBridge) {
|
|
141
|
-
this.aupComplianceAssessor.setClaudeBridge(this.claudeBridge);
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
if (this.config.assessmentCategories?.toolAnnotations) {
|
|
145
|
-
this.toolAnnotationAssessor = new ToolAnnotationAssessor(this.config);
|
|
146
|
-
// Wire up Claude bridge for behavior inference
|
|
147
|
-
if (this.claudeBridge) {
|
|
148
|
-
this.toolAnnotationAssessor.setClaudeBridge(this.claudeBridge);
|
|
149
|
-
}
|
|
150
|
-
// Load custom pattern configuration if provided
|
|
151
|
-
if (this.config.patternConfigPath) {
|
|
152
|
-
const patternConfig = loadPatternConfig(this.config.patternConfigPath, this.logger);
|
|
153
|
-
const compiledPatterns = compilePatterns(patternConfig);
|
|
154
|
-
this.toolAnnotationAssessor.setPatterns(compiledPatterns);
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
if (this.config.assessmentCategories?.prohibitedLibraries) {
|
|
158
|
-
this.prohibitedLibrariesAssessor = new ProhibitedLibrariesAssessor(this.config);
|
|
159
|
-
}
|
|
160
|
-
if (this.config.assessmentCategories?.manifestValidation) {
|
|
161
|
-
this.manifestValidationAssessor = new ManifestValidationAssessor(this.config);
|
|
162
|
-
}
|
|
163
|
-
if (this.config.assessmentCategories?.portability) {
|
|
164
|
-
this.portabilityAssessor = new PortabilityAssessor(this.config);
|
|
165
|
-
}
|
|
166
|
-
if (this.config.assessmentCategories?.externalAPIScanner) {
|
|
167
|
-
this.externalAPIScannerAssessor = new ExternalAPIScannerAssessor(this.config);
|
|
168
|
-
}
|
|
169
|
-
if (this.config.assessmentCategories?.temporal) {
|
|
170
|
-
this.temporalAssessor = new TemporalAssessor(this.config);
|
|
171
|
-
}
|
|
172
|
-
if (this.config.assessmentCategories?.authentication) {
|
|
173
|
-
this.authenticationAssessor = new AuthenticationAssessor(this.config);
|
|
174
|
-
}
|
|
175
|
-
// Initialize new capability assessors
|
|
176
|
-
if (this.config.assessmentCategories?.resources) {
|
|
177
|
-
this.resourceAssessor = new ResourceAssessor(this.config);
|
|
178
|
-
}
|
|
179
|
-
if (this.config.assessmentCategories?.prompts) {
|
|
180
|
-
this.promptAssessor = new PromptAssessor(this.config);
|
|
181
|
-
}
|
|
182
|
-
if (this.config.assessmentCategories?.crossCapability) {
|
|
183
|
-
this.crossCapabilityAssessor = new CrossCapabilitySecurityAssessor(this.config);
|
|
184
|
-
}
|
|
185
|
-
// Initialize code quality assessors
|
|
186
|
-
if (this.config.assessmentCategories?.fileModularization) {
|
|
187
|
-
this.fileModularizationAssessor = new FileModularizationAssessor(this.config);
|
|
188
|
-
}
|
|
189
|
-
// Initialize official MCP conformance testing (opt-in via --conformance flag)
|
|
190
|
-
// Requires HTTP/SSE transport with serverUrl available
|
|
191
|
-
if (this.config.assessmentCategories?.conformance) {
|
|
192
|
-
this.conformanceAssessor = new ConformanceAssessor(this.config);
|
|
193
|
-
}
|
|
194
|
-
// Note: Protocol conformance now handled by unified ProtocolComplianceAssessor above
|
|
195
|
-
}
|
|
196
|
-
// Wire up Claude bridge to TestDataGenerator for intelligent test generation
|
|
61
|
+
// Initialize registry and register all enabled assessors (Issue #91)
|
|
62
|
+
// The registry handles:
|
|
63
|
+
// - Conditional instantiation based on config flags
|
|
64
|
+
// - Deprecated flag OR logic (e.g., protocolCompliance supports 3 flags)
|
|
65
|
+
// - Custom setup (e.g., ToolAnnotationAssessor pattern config)
|
|
66
|
+
// - Claude bridge wiring for supporting assessors
|
|
67
|
+
this.registry = new AssessorRegistry(this.config);
|
|
68
|
+
this.registry.registerAll(ASSESSOR_DEFINITIONS);
|
|
69
|
+
// Wire up Claude bridge to registry (handles all supporting assessors)
|
|
197
70
|
if (this.claudeBridge) {
|
|
71
|
+
this.registry.setClaudeBridge(this.claudeBridge);
|
|
198
72
|
TestDataGenerator.setClaudeBridge(this.claudeBridge);
|
|
199
73
|
}
|
|
200
74
|
// Set logger for TestDataGenerator diagnostic output
|
|
201
75
|
TestDataGenerator.setLogger(this.logger);
|
|
202
76
|
}
|
|
203
|
-
/**
|
|
204
|
-
* Get the count of tools that will actually be tested based on selectedToolsForTesting config.
|
|
205
|
-
* Used for accurate progress estimation in emitModuleStartedEvent calls.
|
|
206
|
-
*/
|
|
207
|
-
getToolCountForTesting(tools) {
|
|
208
|
-
if (this.config.selectedToolsForTesting !== undefined) {
|
|
209
|
-
const selectedNames = new Set(this.config.selectedToolsForTesting);
|
|
210
|
-
return tools.filter((tool) => selectedNames.has(tool.name)).length;
|
|
211
|
-
}
|
|
212
|
-
return tools.length;
|
|
213
|
-
}
|
|
214
77
|
/**
|
|
215
78
|
* Initialize Claude Code Bridge for intelligent analysis
|
|
216
79
|
* This enables semantic AUP violation analysis, behavior inference, and intelligent test generation
|
|
@@ -242,17 +105,9 @@ export class AssessmentOrchestrator {
|
|
|
242
105
|
enabled: true,
|
|
243
106
|
};
|
|
244
107
|
this.initializeClaudeBridge(bridgeConfig);
|
|
245
|
-
// Wire up to
|
|
108
|
+
// Wire up to all supporting assessors via registry
|
|
246
109
|
if (this.claudeBridge) {
|
|
247
|
-
|
|
248
|
-
this.aupComplianceAssessor.setClaudeBridge(this.claudeBridge);
|
|
249
|
-
}
|
|
250
|
-
if (this.toolAnnotationAssessor) {
|
|
251
|
-
this.toolAnnotationAssessor.setClaudeBridge(this.claudeBridge);
|
|
252
|
-
}
|
|
253
|
-
if (this.securityAssessor) {
|
|
254
|
-
this.securityAssessor.setClaudeBridge(this.claudeBridge);
|
|
255
|
-
}
|
|
110
|
+
this.registry.setClaudeBridge(this.claudeBridge);
|
|
256
111
|
TestDataGenerator.setClaudeBridge(this.claudeBridge);
|
|
257
112
|
}
|
|
258
113
|
}
|
|
@@ -270,55 +125,6 @@ export class AssessmentOrchestrator {
|
|
|
270
125
|
getClaudeBridge() {
|
|
271
126
|
return this.claudeBridge;
|
|
272
127
|
}
|
|
273
|
-
/**
|
|
274
|
-
* Reset test counts for all assessors
|
|
275
|
-
*/
|
|
276
|
-
resetAllTestCounts() {
|
|
277
|
-
this.functionalityAssessor?.resetTestCount();
|
|
278
|
-
this.securityAssessor?.resetTestCount();
|
|
279
|
-
this.documentationAssessor?.resetTestCount();
|
|
280
|
-
this.errorHandlingAssessor?.resetTestCount();
|
|
281
|
-
this.usabilityAssessor?.resetTestCount();
|
|
282
|
-
if (this.protocolComplianceAssessor) {
|
|
283
|
-
this.protocolComplianceAssessor.resetTestCount();
|
|
284
|
-
}
|
|
285
|
-
// Reset new assessors
|
|
286
|
-
if (this.aupComplianceAssessor) {
|
|
287
|
-
this.aupComplianceAssessor.resetTestCount();
|
|
288
|
-
}
|
|
289
|
-
if (this.toolAnnotationAssessor) {
|
|
290
|
-
this.toolAnnotationAssessor.resetTestCount();
|
|
291
|
-
}
|
|
292
|
-
if (this.prohibitedLibrariesAssessor) {
|
|
293
|
-
this.prohibitedLibrariesAssessor.resetTestCount();
|
|
294
|
-
}
|
|
295
|
-
if (this.manifestValidationAssessor) {
|
|
296
|
-
this.manifestValidationAssessor.resetTestCount();
|
|
297
|
-
}
|
|
298
|
-
if (this.portabilityAssessor) {
|
|
299
|
-
this.portabilityAssessor.resetTestCount();
|
|
300
|
-
}
|
|
301
|
-
if (this.authenticationAssessor) {
|
|
302
|
-
this.authenticationAssessor.resetTestCount();
|
|
303
|
-
}
|
|
304
|
-
// Reset new capability assessors
|
|
305
|
-
if (this.resourceAssessor) {
|
|
306
|
-
this.resourceAssessor.resetTestCount();
|
|
307
|
-
}
|
|
308
|
-
if (this.promptAssessor) {
|
|
309
|
-
this.promptAssessor.resetTestCount();
|
|
310
|
-
}
|
|
311
|
-
if (this.crossCapabilityAssessor) {
|
|
312
|
-
this.crossCapabilityAssessor.resetTestCount();
|
|
313
|
-
}
|
|
314
|
-
if (this.fileModularizationAssessor) {
|
|
315
|
-
this.fileModularizationAssessor.resetTestCount();
|
|
316
|
-
}
|
|
317
|
-
// Reset official conformance assessor
|
|
318
|
-
if (this.conformanceAssessor) {
|
|
319
|
-
this.conformanceAssessor.resetTestCount();
|
|
320
|
-
}
|
|
321
|
-
}
|
|
322
128
|
/**
|
|
323
129
|
* Run a complete assessment on an MCP server
|
|
324
130
|
* @public
|
|
@@ -326,298 +132,14 @@ export class AssessmentOrchestrator {
|
|
|
326
132
|
async runFullAssessment(context) {
|
|
327
133
|
this.startTime = Date.now();
|
|
328
134
|
this.totalTestsRun = 0;
|
|
329
|
-
this.resetAllTestCounts();
|
|
330
|
-
//
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
//
|
|
334
|
-
//
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
const invocationsPerTool = this.config.temporalInvocations ?? 25;
|
|
338
|
-
emitModuleStartedEvent("Temporal", toolCount * invocationsPerTool, toolCount);
|
|
339
|
-
assessmentResults.temporal = await this.temporalAssessor.assess(context);
|
|
340
|
-
emitModuleProgress("Temporal", assessmentResults.temporal.status, assessmentResults.temporal, this.temporalAssessor.getTestCount());
|
|
341
|
-
}
|
|
342
|
-
if (this.config.parallelTesting) {
|
|
343
|
-
// Calculate estimates for module_started events
|
|
344
|
-
const toolCount = this.getToolCountForTesting(context.tools);
|
|
345
|
-
const securityPatterns = this.config.securityPatternsToTest || 17;
|
|
346
|
-
// Core assessments - only emit and run if not skipped
|
|
347
|
-
if (this.functionalityAssessor) {
|
|
348
|
-
emitModuleStartedEvent("Functionality", toolCount * 10, toolCount);
|
|
349
|
-
assessmentPromises.push(this.functionalityAssessor.assess(context).then((r) => {
|
|
350
|
-
emitModuleProgress("Functionality", r.status, r, this.functionalityAssessor.getTestCount());
|
|
351
|
-
return (assessmentResults.functionality = r);
|
|
352
|
-
}));
|
|
353
|
-
}
|
|
354
|
-
if (this.securityAssessor) {
|
|
355
|
-
emitModuleStartedEvent("Security", securityPatterns * toolCount, toolCount);
|
|
356
|
-
assessmentPromises.push(this.securityAssessor.assess(context).then((r) => {
|
|
357
|
-
emitModuleProgress("Security", r.status, r, this.securityAssessor.getTestCount());
|
|
358
|
-
return (assessmentResults.security = r);
|
|
359
|
-
}));
|
|
360
|
-
}
|
|
361
|
-
if (this.documentationAssessor) {
|
|
362
|
-
emitModuleStartedEvent("Documentation", 5, toolCount);
|
|
363
|
-
assessmentPromises.push(this.documentationAssessor.assess(context).then((r) => {
|
|
364
|
-
emitModuleProgress("Documentation", r.status, r, this.documentationAssessor.getTestCount());
|
|
365
|
-
return (assessmentResults.documentation = r);
|
|
366
|
-
}));
|
|
367
|
-
}
|
|
368
|
-
if (this.errorHandlingAssessor) {
|
|
369
|
-
emitModuleStartedEvent("Error Handling", toolCount * 5, toolCount);
|
|
370
|
-
assessmentPromises.push(this.errorHandlingAssessor.assess(context).then((r) => {
|
|
371
|
-
emitModuleProgress("Error Handling", r.status, r, this.errorHandlingAssessor.getTestCount());
|
|
372
|
-
return (assessmentResults.errorHandling = r);
|
|
373
|
-
}));
|
|
374
|
-
}
|
|
375
|
-
if (this.usabilityAssessor) {
|
|
376
|
-
emitModuleStartedEvent("Usability", 10, toolCount);
|
|
377
|
-
assessmentPromises.push(this.usabilityAssessor.assess(context).then((r) => {
|
|
378
|
-
emitModuleProgress("Usability", r.status, r, this.usabilityAssessor.getTestCount());
|
|
379
|
-
return (assessmentResults.usability = r);
|
|
380
|
-
}));
|
|
381
|
-
}
|
|
382
|
-
// Extended assessments - unified protocol compliance
|
|
383
|
-
if (this.protocolComplianceAssessor) {
|
|
384
|
-
emitModuleStartedEvent("Protocol Compliance", 10, toolCount);
|
|
385
|
-
assessmentPromises.push(this.protocolComplianceAssessor.assess(context).then((r) => {
|
|
386
|
-
emitModuleProgress("Protocol Compliance", r.status, r, this.protocolComplianceAssessor.getTestCount());
|
|
387
|
-
return (assessmentResults.mcpSpecCompliance = r);
|
|
388
|
-
}));
|
|
389
|
-
}
|
|
390
|
-
// New MCP Directory Compliance Gap assessments
|
|
391
|
-
if (this.aupComplianceAssessor) {
|
|
392
|
-
emitModuleStartedEvent("AUP", 20, toolCount);
|
|
393
|
-
assessmentPromises.push(this.aupComplianceAssessor.assess(context).then((r) => {
|
|
394
|
-
emitModuleProgress("AUP", r.status, r, this.aupComplianceAssessor.getTestCount());
|
|
395
|
-
return (assessmentResults.aupCompliance = r);
|
|
396
|
-
}));
|
|
397
|
-
}
|
|
398
|
-
if (this.toolAnnotationAssessor) {
|
|
399
|
-
emitModuleStartedEvent("Annotations", toolCount, toolCount);
|
|
400
|
-
assessmentPromises.push(this.toolAnnotationAssessor.assess(context).then((r) => {
|
|
401
|
-
emitModuleProgress("Annotations", r.status, r, this.toolAnnotationAssessor.getTestCount());
|
|
402
|
-
return (assessmentResults.toolAnnotations = r);
|
|
403
|
-
}));
|
|
404
|
-
}
|
|
405
|
-
if (this.prohibitedLibrariesAssessor) {
|
|
406
|
-
emitModuleStartedEvent("Libraries", 5, toolCount);
|
|
407
|
-
assessmentPromises.push(this.prohibitedLibrariesAssessor.assess(context).then((r) => {
|
|
408
|
-
emitModuleProgress("Libraries", r.status, r, this.prohibitedLibrariesAssessor.getTestCount());
|
|
409
|
-
return (assessmentResults.prohibitedLibraries = r);
|
|
410
|
-
}));
|
|
411
|
-
}
|
|
412
|
-
if (this.manifestValidationAssessor) {
|
|
413
|
-
emitModuleStartedEvent("Manifest", 10, toolCount);
|
|
414
|
-
assessmentPromises.push(this.manifestValidationAssessor.assess(context).then((r) => {
|
|
415
|
-
emitModuleProgress("Manifest", r.status, r, this.manifestValidationAssessor.getTestCount());
|
|
416
|
-
return (assessmentResults.manifestValidation = r);
|
|
417
|
-
}));
|
|
418
|
-
}
|
|
419
|
-
if (this.portabilityAssessor) {
|
|
420
|
-
emitModuleStartedEvent("Portability", 10, toolCount);
|
|
421
|
-
assessmentPromises.push(this.portabilityAssessor.assess(context).then((r) => {
|
|
422
|
-
emitModuleProgress("Portability", r.status, r, this.portabilityAssessor.getTestCount());
|
|
423
|
-
return (assessmentResults.portability = r);
|
|
424
|
-
}));
|
|
425
|
-
}
|
|
426
|
-
if (this.externalAPIScannerAssessor) {
|
|
427
|
-
emitModuleStartedEvent("External APIs", 10, toolCount);
|
|
428
|
-
assessmentPromises.push(this.externalAPIScannerAssessor.assess(context).then((r) => {
|
|
429
|
-
emitModuleProgress("External APIs", r.status, r, this.externalAPIScannerAssessor.getTestCount());
|
|
430
|
-
return (assessmentResults.externalAPIScanner = r);
|
|
431
|
-
}));
|
|
432
|
-
}
|
|
433
|
-
if (this.authenticationAssessor) {
|
|
434
|
-
const sourceFileCount = context.sourceCodeFiles?.size || 0;
|
|
435
|
-
emitModuleStartedEvent("Authentication", sourceFileCount, sourceFileCount);
|
|
436
|
-
assessmentPromises.push(this.authenticationAssessor.assess(context).then((r) => {
|
|
437
|
-
emitModuleProgress("Authentication", r.status, r, this.authenticationAssessor.getTestCount());
|
|
438
|
-
return (assessmentResults.authentication = r);
|
|
439
|
-
}));
|
|
440
|
-
}
|
|
441
|
-
// NOTE: Temporal runs in PHASE 0 above, not in parallel with other modules
|
|
442
|
-
// New capability assessors
|
|
443
|
-
if (this.resourceAssessor) {
|
|
444
|
-
const resourceCount = (context.resources?.length || 0) +
|
|
445
|
-
(context.resourceTemplates?.length || 0);
|
|
446
|
-
emitModuleStartedEvent("Resources", resourceCount * 5, resourceCount);
|
|
447
|
-
assessmentPromises.push(this.resourceAssessor.assess(context).then((r) => {
|
|
448
|
-
emitModuleProgress("Resources", r.status, r, this.resourceAssessor.getTestCount());
|
|
449
|
-
return (assessmentResults.resources = r);
|
|
450
|
-
}));
|
|
451
|
-
}
|
|
452
|
-
if (this.promptAssessor) {
|
|
453
|
-
const promptCount = context.prompts?.length || 0;
|
|
454
|
-
emitModuleStartedEvent("Prompts", promptCount * 10, promptCount);
|
|
455
|
-
assessmentPromises.push(this.promptAssessor.assess(context).then((r) => {
|
|
456
|
-
emitModuleProgress("Prompts", r.status, r, this.promptAssessor.getTestCount());
|
|
457
|
-
return (assessmentResults.prompts = r);
|
|
458
|
-
}));
|
|
459
|
-
}
|
|
460
|
-
if (this.crossCapabilityAssessor) {
|
|
461
|
-
const capabilityCount = toolCount +
|
|
462
|
-
(context.resources?.length || 0) +
|
|
463
|
-
(context.prompts?.length || 0);
|
|
464
|
-
emitModuleStartedEvent("Cross-Capability", capabilityCount * 3, capabilityCount);
|
|
465
|
-
assessmentPromises.push(this.crossCapabilityAssessor.assess(context).then((r) => {
|
|
466
|
-
emitModuleProgress("Cross-Capability", r.status, r, this.crossCapabilityAssessor.getTestCount());
|
|
467
|
-
return (assessmentResults.crossCapability = r);
|
|
468
|
-
}));
|
|
469
|
-
}
|
|
470
|
-
if (this.fileModularizationAssessor) {
|
|
471
|
-
const sourceFileCount = context.sourceCodeFiles?.size || 0;
|
|
472
|
-
emitModuleStartedEvent("File Modularization", sourceFileCount, sourceFileCount);
|
|
473
|
-
assessmentPromises.push(this.fileModularizationAssessor.assess(context).then((r) => {
|
|
474
|
-
emitModuleProgress("File Modularization", r.status, r, this.fileModularizationAssessor.getTestCount());
|
|
475
|
-
return (assessmentResults.fileModularization = r);
|
|
476
|
-
}));
|
|
477
|
-
}
|
|
478
|
-
// Official MCP conformance testing (opt-in, requires HTTP/SSE transport)
|
|
479
|
-
if (this.conformanceAssessor) {
|
|
480
|
-
// Conformance tests ~7 server scenarios
|
|
481
|
-
emitModuleStartedEvent("Conformance", 7, toolCount);
|
|
482
|
-
assessmentPromises.push(this.conformanceAssessor.assess(context).then((r) => {
|
|
483
|
-
emitModuleProgress("Conformance", r.status, r, this.conformanceAssessor.getTestCount());
|
|
484
|
-
return (assessmentResults.conformance = r);
|
|
485
|
-
}));
|
|
486
|
-
}
|
|
487
|
-
// Note: Protocol Conformance now handled by unified ProtocolComplianceAssessor above
|
|
488
|
-
await Promise.all(assessmentPromises);
|
|
489
|
-
}
|
|
490
|
-
else {
|
|
491
|
-
// Sequential execution with module_started events
|
|
492
|
-
const toolCount = this.getToolCountForTesting(context.tools);
|
|
493
|
-
const securityPatterns = this.config.securityPatternsToTest || 17;
|
|
494
|
-
// NOTE: Temporal runs in PHASE 0 above, before sequential/parallel phases
|
|
495
|
-
// Core assessments - only emit and run if not skipped
|
|
496
|
-
if (this.functionalityAssessor) {
|
|
497
|
-
// Functionality: ~10 scenarios per tool
|
|
498
|
-
emitModuleStartedEvent("Functionality", toolCount * 10, toolCount);
|
|
499
|
-
assessmentResults.functionality =
|
|
500
|
-
await this.functionalityAssessor.assess(context);
|
|
501
|
-
emitModuleProgress("Functionality", assessmentResults.functionality.status, assessmentResults.functionality, this.functionalityAssessor.getTestCount());
|
|
502
|
-
}
|
|
503
|
-
if (this.securityAssessor) {
|
|
504
|
-
// Security: patterns × tools
|
|
505
|
-
emitModuleStartedEvent("Security", securityPatterns * toolCount, toolCount);
|
|
506
|
-
assessmentResults.security =
|
|
507
|
-
await this.securityAssessor.assess(context);
|
|
508
|
-
emitModuleProgress("Security", assessmentResults.security.status, assessmentResults.security, this.securityAssessor.getTestCount());
|
|
509
|
-
}
|
|
510
|
-
if (this.documentationAssessor) {
|
|
511
|
-
// Documentation: ~5 static tests
|
|
512
|
-
emitModuleStartedEvent("Documentation", 5, toolCount);
|
|
513
|
-
assessmentResults.documentation =
|
|
514
|
-
await this.documentationAssessor.assess(context);
|
|
515
|
-
emitModuleProgress("Documentation", assessmentResults.documentation.status, assessmentResults.documentation, this.documentationAssessor.getTestCount());
|
|
516
|
-
}
|
|
517
|
-
if (this.errorHandlingAssessor) {
|
|
518
|
-
// Error Handling: ~5 tests per tool
|
|
519
|
-
emitModuleStartedEvent("Error Handling", toolCount * 5, toolCount);
|
|
520
|
-
assessmentResults.errorHandling =
|
|
521
|
-
await this.errorHandlingAssessor.assess(context);
|
|
522
|
-
emitModuleProgress("Error Handling", assessmentResults.errorHandling.status, assessmentResults.errorHandling, this.errorHandlingAssessor.getTestCount());
|
|
523
|
-
}
|
|
524
|
-
if (this.usabilityAssessor) {
|
|
525
|
-
// Usability: ~10 static tests
|
|
526
|
-
emitModuleStartedEvent("Usability", 10, toolCount);
|
|
527
|
-
assessmentResults.usability =
|
|
528
|
-
await this.usabilityAssessor.assess(context);
|
|
529
|
-
emitModuleProgress("Usability", assessmentResults.usability.status, assessmentResults.usability, this.usabilityAssessor.getTestCount());
|
|
530
|
-
}
|
|
531
|
-
if (this.protocolComplianceAssessor) {
|
|
532
|
-
emitModuleStartedEvent("Protocol Compliance", 10, toolCount);
|
|
533
|
-
assessmentResults.mcpSpecCompliance =
|
|
534
|
-
await this.protocolComplianceAssessor.assess(context);
|
|
535
|
-
emitModuleProgress("Protocol Compliance", assessmentResults.mcpSpecCompliance.status, assessmentResults.mcpSpecCompliance, this.protocolComplianceAssessor.getTestCount());
|
|
536
|
-
}
|
|
537
|
-
// New MCP Directory Compliance Gap assessments (sequential)
|
|
538
|
-
if (this.aupComplianceAssessor) {
|
|
539
|
-
emitModuleStartedEvent("AUP", 20, toolCount);
|
|
540
|
-
assessmentResults.aupCompliance =
|
|
541
|
-
await this.aupComplianceAssessor.assess(context);
|
|
542
|
-
emitModuleProgress("AUP", assessmentResults.aupCompliance.status, assessmentResults.aupCompliance, this.aupComplianceAssessor.getTestCount());
|
|
543
|
-
}
|
|
544
|
-
if (this.toolAnnotationAssessor) {
|
|
545
|
-
emitModuleStartedEvent("Annotations", toolCount, toolCount);
|
|
546
|
-
assessmentResults.toolAnnotations =
|
|
547
|
-
await this.toolAnnotationAssessor.assess(context);
|
|
548
|
-
emitModuleProgress("Annotations", assessmentResults.toolAnnotations.status, assessmentResults.toolAnnotations, this.toolAnnotationAssessor.getTestCount());
|
|
549
|
-
}
|
|
550
|
-
if (this.prohibitedLibrariesAssessor) {
|
|
551
|
-
emitModuleStartedEvent("Libraries", 5, toolCount);
|
|
552
|
-
assessmentResults.prohibitedLibraries =
|
|
553
|
-
await this.prohibitedLibrariesAssessor.assess(context);
|
|
554
|
-
emitModuleProgress("Libraries", assessmentResults.prohibitedLibraries.status, assessmentResults.prohibitedLibraries, this.prohibitedLibrariesAssessor.getTestCount());
|
|
555
|
-
}
|
|
556
|
-
if (this.manifestValidationAssessor) {
|
|
557
|
-
emitModuleStartedEvent("Manifest", 10, toolCount);
|
|
558
|
-
assessmentResults.manifestValidation =
|
|
559
|
-
await this.manifestValidationAssessor.assess(context);
|
|
560
|
-
emitModuleProgress("Manifest", assessmentResults.manifestValidation.status, assessmentResults.manifestValidation, this.manifestValidationAssessor.getTestCount());
|
|
561
|
-
}
|
|
562
|
-
if (this.portabilityAssessor) {
|
|
563
|
-
emitModuleStartedEvent("Portability", 10, toolCount);
|
|
564
|
-
assessmentResults.portability =
|
|
565
|
-
await this.portabilityAssessor.assess(context);
|
|
566
|
-
emitModuleProgress("Portability", assessmentResults.portability.status, assessmentResults.portability, this.portabilityAssessor.getTestCount());
|
|
567
|
-
}
|
|
568
|
-
if (this.externalAPIScannerAssessor) {
|
|
569
|
-
emitModuleStartedEvent("External APIs", 10, toolCount);
|
|
570
|
-
assessmentResults.externalAPIScanner =
|
|
571
|
-
await this.externalAPIScannerAssessor.assess(context);
|
|
572
|
-
emitModuleProgress("External APIs", assessmentResults.externalAPIScanner.status, assessmentResults.externalAPIScanner, this.externalAPIScannerAssessor.getTestCount());
|
|
573
|
-
}
|
|
574
|
-
if (this.authenticationAssessor) {
|
|
575
|
-
const sourceFileCount = context.sourceCodeFiles?.size || 0;
|
|
576
|
-
emitModuleStartedEvent("Authentication", sourceFileCount, sourceFileCount);
|
|
577
|
-
assessmentResults.authentication =
|
|
578
|
-
await this.authenticationAssessor.assess(context);
|
|
579
|
-
emitModuleProgress("Authentication", assessmentResults.authentication.status, assessmentResults.authentication, this.authenticationAssessor.getTestCount());
|
|
580
|
-
}
|
|
581
|
-
// New capability assessors (sequential)
|
|
582
|
-
if (this.resourceAssessor) {
|
|
583
|
-
const resourceCount = (context.resources?.length || 0) +
|
|
584
|
-
(context.resourceTemplates?.length || 0);
|
|
585
|
-
emitModuleStartedEvent("Resources", resourceCount * 5, resourceCount);
|
|
586
|
-
assessmentResults.resources =
|
|
587
|
-
await this.resourceAssessor.assess(context);
|
|
588
|
-
emitModuleProgress("Resources", assessmentResults.resources.status, assessmentResults.resources, this.resourceAssessor.getTestCount());
|
|
589
|
-
}
|
|
590
|
-
if (this.promptAssessor) {
|
|
591
|
-
const promptCount = context.prompts?.length || 0;
|
|
592
|
-
emitModuleStartedEvent("Prompts", promptCount * 10, promptCount);
|
|
593
|
-
assessmentResults.prompts = await this.promptAssessor.assess(context);
|
|
594
|
-
emitModuleProgress("Prompts", assessmentResults.prompts.status, assessmentResults.prompts, this.promptAssessor.getTestCount());
|
|
595
|
-
}
|
|
596
|
-
if (this.crossCapabilityAssessor) {
|
|
597
|
-
const capabilityCount = toolCount +
|
|
598
|
-
(context.resources?.length || 0) +
|
|
599
|
-
(context.prompts?.length || 0);
|
|
600
|
-
emitModuleStartedEvent("Cross-Capability", capabilityCount * 3, capabilityCount);
|
|
601
|
-
assessmentResults.crossCapability =
|
|
602
|
-
await this.crossCapabilityAssessor.assess(context);
|
|
603
|
-
emitModuleProgress("Cross-Capability", assessmentResults.crossCapability.status, assessmentResults.crossCapability, this.crossCapabilityAssessor.getTestCount());
|
|
604
|
-
}
|
|
605
|
-
if (this.fileModularizationAssessor) {
|
|
606
|
-
const sourceFileCount = context.sourceCodeFiles?.size || 0;
|
|
607
|
-
emitModuleStartedEvent("File Modularization", sourceFileCount, sourceFileCount);
|
|
608
|
-
assessmentResults.fileModularization =
|
|
609
|
-
await this.fileModularizationAssessor.assess(context);
|
|
610
|
-
emitModuleProgress("File Modularization", assessmentResults.fileModularization.status, assessmentResults.fileModularization, this.fileModularizationAssessor.getTestCount());
|
|
611
|
-
}
|
|
612
|
-
// Official MCP conformance testing (sequential, opt-in)
|
|
613
|
-
if (this.conformanceAssessor) {
|
|
614
|
-
emitModuleStartedEvent("Conformance", 7, toolCount);
|
|
615
|
-
assessmentResults.conformance =
|
|
616
|
-
await this.conformanceAssessor.assess(context);
|
|
617
|
-
emitModuleProgress("Conformance", assessmentResults.conformance.status, assessmentResults.conformance, this.conformanceAssessor.getTestCount());
|
|
618
|
-
}
|
|
619
|
-
// Note: Protocol Conformance now handled by unified ProtocolComplianceAssessor above
|
|
620
|
-
}
|
|
135
|
+
this.registry.resetAllTestCounts();
|
|
136
|
+
// Execute all assessors via registry (Issue #91)
|
|
137
|
+
// Registry handles:
|
|
138
|
+
// - Phase-ordered execution (Phase 0/PRE always runs first and sequentially)
|
|
139
|
+
// - Parallel vs sequential based on config.parallelTesting
|
|
140
|
+
// - JSONL events (module_started, module_progress)
|
|
141
|
+
// - Test count tracking
|
|
142
|
+
const assessmentResults = await this.registry.executeAll(context);
|
|
621
143
|
// Integrate temporal findings into security.vulnerabilities for unified view
|
|
622
144
|
if (assessmentResults.temporal?.rugPullsDetected &&
|
|
623
145
|
assessmentResults.temporal.rugPullsDetected > 0 &&
|
|
@@ -626,6 +148,41 @@ export class AssessmentOrchestrator {
|
|
|
626
148
|
assessmentResults.security.vulnerabilities.push(`RUG_PULL_TEMPORAL: ${detail.tool} - Tool behavior changed after invocation ${detail.firstDeviationAt}. Requires immediate manual review.`);
|
|
627
149
|
}
|
|
628
150
|
}
|
|
151
|
+
// Issue #124: Dual-key output for v2.0.0 transition
|
|
152
|
+
// Output BOTH old and new keys to maintain backward compatibility
|
|
153
|
+
// Old keys (documentation, usability, mcpSpecCompliance) will be removed in v2.0.0
|
|
154
|
+
// developerExperience (new) = documentation + usability (deprecated)
|
|
155
|
+
if (assessmentResults.documentation && assessmentResults.usability) {
|
|
156
|
+
const docScore = calculateModuleScore(assessmentResults.documentation) ?? 50;
|
|
157
|
+
const usabilityScore = calculateModuleScore(assessmentResults.usability) ?? 50;
|
|
158
|
+
const combinedStatus = determineOverallStatus({
|
|
159
|
+
documentation: assessmentResults.documentation,
|
|
160
|
+
usability: assessmentResults.usability,
|
|
161
|
+
});
|
|
162
|
+
assessmentResults.developerExperience = {
|
|
163
|
+
documentation: assessmentResults.documentation,
|
|
164
|
+
usability: assessmentResults.usability,
|
|
165
|
+
status: combinedStatus,
|
|
166
|
+
score: Math.round((docScore + usabilityScore) / 2),
|
|
167
|
+
};
|
|
168
|
+
// Emit deprecation warning for old keys
|
|
169
|
+
this.logger.warn("Output keys 'documentation' and 'usability' are deprecated. " +
|
|
170
|
+
"Use 'developerExperience' instead. These keys will be removed in v2.0.0.", {
|
|
171
|
+
deprecated: ["documentation", "usability"],
|
|
172
|
+
replacement: "developerExperience",
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
// protocolCompliance (new) = mcpSpecCompliance (deprecated)
|
|
176
|
+
if (assessmentResults.mcpSpecCompliance) {
|
|
177
|
+
assessmentResults.protocolCompliance =
|
|
178
|
+
assessmentResults.mcpSpecCompliance;
|
|
179
|
+
// Emit deprecation warning for old key
|
|
180
|
+
this.logger.warn("Output key 'mcpSpecCompliance' is deprecated. " +
|
|
181
|
+
"Use 'protocolCompliance' instead. This key will be removed in v2.0.0.", {
|
|
182
|
+
deprecated: ["mcpSpecCompliance"],
|
|
183
|
+
replacement: "protocolCompliance",
|
|
184
|
+
});
|
|
185
|
+
}
|
|
629
186
|
// Collect test counts from all assessors
|
|
630
187
|
this.totalTestsRun = this.collectTotalTestCount();
|
|
631
188
|
// Determine overall status
|
|
@@ -674,75 +231,8 @@ export class AssessmentOrchestrator {
|
|
|
674
231
|
return this.runFullAssessment(context);
|
|
675
232
|
}
|
|
676
233
|
collectTotalTestCount() {
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
const functionalityCount = this.functionalityAssessor?.getTestCount() || 0;
|
|
680
|
-
const securityCount = this.securityAssessor?.getTestCount() || 0;
|
|
681
|
-
const documentationCount = this.documentationAssessor?.getTestCount() || 0;
|
|
682
|
-
const errorHandlingCount = this.errorHandlingAssessor?.getTestCount() || 0;
|
|
683
|
-
const usabilityCount = this.usabilityAssessor?.getTestCount() || 0;
|
|
684
|
-
const mcpSpecCount = this.protocolComplianceAssessor?.getTestCount() || 0;
|
|
685
|
-
// New assessor counts
|
|
686
|
-
const aupCount = this.aupComplianceAssessor?.getTestCount() || 0;
|
|
687
|
-
const annotationCount = this.toolAnnotationAssessor?.getTestCount() || 0;
|
|
688
|
-
const librariesCount = this.prohibitedLibrariesAssessor?.getTestCount() || 0;
|
|
689
|
-
const manifestCount = this.manifestValidationAssessor?.getTestCount() || 0;
|
|
690
|
-
const portabilityCount = this.portabilityAssessor?.getTestCount() || 0;
|
|
691
|
-
const authenticationCount = this.authenticationAssessor?.getTestCount() || 0;
|
|
692
|
-
const externalAPICount = this.externalAPIScannerAssessor?.getTestCount() || 0;
|
|
693
|
-
const temporalCount = this.temporalAssessor?.getTestCount() || 0;
|
|
694
|
-
// New capability assessor counts
|
|
695
|
-
const resourcesCount = this.resourceAssessor?.getTestCount() || 0;
|
|
696
|
-
const promptsCount = this.promptAssessor?.getTestCount() || 0;
|
|
697
|
-
const crossCapabilityCount = this.crossCapabilityAssessor?.getTestCount() || 0;
|
|
698
|
-
// Code quality assessor counts
|
|
699
|
-
const fileModularizationCount = this.fileModularizationAssessor?.getTestCount() || 0;
|
|
700
|
-
// Official MCP conformance test count
|
|
701
|
-
const conformanceCount = this.conformanceAssessor?.getTestCount() || 0;
|
|
702
|
-
// Note: Protocol conformance now included in mcpSpecCount (unified ProtocolComplianceAssessor)
|
|
703
|
-
this.logger.debug("Test counts by assessor", {
|
|
704
|
-
functionality: functionalityCount,
|
|
705
|
-
security: securityCount,
|
|
706
|
-
documentation: documentationCount,
|
|
707
|
-
errorHandling: errorHandlingCount,
|
|
708
|
-
usability: usabilityCount,
|
|
709
|
-
mcpSpec: mcpSpecCount,
|
|
710
|
-
aupCompliance: aupCount,
|
|
711
|
-
toolAnnotations: annotationCount,
|
|
712
|
-
prohibitedLibraries: librariesCount,
|
|
713
|
-
manifestValidation: manifestCount,
|
|
714
|
-
portability: portabilityCount,
|
|
715
|
-
authentication: authenticationCount,
|
|
716
|
-
externalAPIScanner: externalAPICount,
|
|
717
|
-
temporal: temporalCount,
|
|
718
|
-
resources: resourcesCount,
|
|
719
|
-
prompts: promptsCount,
|
|
720
|
-
crossCapability: crossCapabilityCount,
|
|
721
|
-
fileModularization: fileModularizationCount,
|
|
722
|
-
conformance: conformanceCount,
|
|
723
|
-
// Note: protocolConformance now included in mcpSpec (unified)
|
|
724
|
-
});
|
|
725
|
-
total =
|
|
726
|
-
functionalityCount +
|
|
727
|
-
securityCount +
|
|
728
|
-
documentationCount +
|
|
729
|
-
errorHandlingCount +
|
|
730
|
-
usabilityCount +
|
|
731
|
-
mcpSpecCount +
|
|
732
|
-
aupCount +
|
|
733
|
-
annotationCount +
|
|
734
|
-
librariesCount +
|
|
735
|
-
manifestCount +
|
|
736
|
-
portabilityCount +
|
|
737
|
-
authenticationCount +
|
|
738
|
-
externalAPICount +
|
|
739
|
-
temporalCount +
|
|
740
|
-
resourcesCount +
|
|
741
|
-
promptsCount +
|
|
742
|
-
crossCapabilityCount +
|
|
743
|
-
fileModularizationCount +
|
|
744
|
-
conformanceCount;
|
|
745
|
-
// Note: protocolConformance now included in mcpSpecCount (unified)
|
|
234
|
+
// Delegate to registry for centralized test count aggregation (Issue #91)
|
|
235
|
+
const total = this.registry.getTotalTestCount();
|
|
746
236
|
this.logger.debug("Total test count", { total });
|
|
747
237
|
return total;
|
|
748
238
|
}
|