@bryan-thompson/inspector-assessment-client 1.31.0 → 1.32.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/dist/assets/{OAuthCallback-CXcl26vR.js → OAuthCallback-DQqdKmh-.js} +1 -1
  2. package/dist/assets/{OAuthDebugCallback-J9s4SF_c.js → OAuthDebugCallback-CZ79tNPn.js} +1 -1
  3. package/dist/assets/{index-_HAw2b2G.js → index-7dGKjpGX.js} +4 -4
  4. package/dist/index.html +1 -1
  5. package/lib/lib/assessment/extendedTypes.d.ts +21 -0
  6. package/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
  7. package/lib/lib/assessment/resultTypes.d.ts +12 -2
  8. package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
  9. package/lib/lib/moduleScoring.d.ts.map +1 -1
  10. package/lib/lib/moduleScoring.js +5 -0
  11. package/lib/services/assessment/AssessmentOrchestrator.d.ts +1 -28
  12. package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
  13. package/lib/services/assessment/AssessmentOrchestrator.js +66 -576
  14. package/lib/services/assessment/modules/ConformanceAssessor.d.ts +4 -0
  15. package/lib/services/assessment/modules/ConformanceAssessor.d.ts.map +1 -1
  16. package/lib/services/assessment/modules/ConformanceAssessor.js +21 -0
  17. package/lib/services/assessment/modules/ResourceAssessor.d.ts.map +1 -1
  18. package/lib/services/assessment/modules/ResourceAssessor.js +6 -1
  19. package/lib/services/assessment/registry/AssessorDefinitions.d.ts +38 -0
  20. package/lib/services/assessment/registry/AssessorDefinitions.d.ts.map +1 -0
  21. package/lib/services/assessment/registry/AssessorDefinitions.js +370 -0
  22. package/lib/services/assessment/registry/AssessorRegistry.d.ts +157 -0
  23. package/lib/services/assessment/registry/AssessorRegistry.d.ts.map +1 -0
  24. package/lib/services/assessment/registry/AssessorRegistry.js +364 -0
  25. package/lib/services/assessment/registry/estimators.d.ts +93 -0
  26. package/lib/services/assessment/registry/estimators.d.ts.map +1 -0
  27. package/lib/services/assessment/registry/estimators.js +176 -0
  28. package/lib/services/assessment/registry/index.d.ts +13 -0
  29. package/lib/services/assessment/registry/index.d.ts.map +1 -0
  30. package/lib/services/assessment/registry/index.js +16 -0
  31. package/lib/services/assessment/registry/types.d.ts +180 -0
  32. package/lib/services/assessment/registry/types.d.ts.map +1 -0
  33. package/lib/services/assessment/registry/types.js +35 -0
  34. package/lib/services/assessment/responseValidatorSchemas.d.ts +4 -4
  35. package/package.json +1 -1
@@ -6,41 +6,18 @@
6
6
  * @module AssessmentOrchestrator
7
7
  */
8
8
  import { DEFAULT_ASSESSMENT_CONFIG, } from "../../lib/assessmentTypes.js";
9
- // Core assessment modules
10
- import { FunctionalityAssessor } from "./modules/FunctionalityAssessor.js";
11
- import { SecurityAssessor } from "./modules/SecurityAssessor.js";
12
- import { DocumentationAssessor } from "./modules/DocumentationAssessor.js";
13
- import { ErrorHandlingAssessor } from "./modules/ErrorHandlingAssessor.js";
14
- import { UsabilityAssessor } from "./modules/UsabilityAssessor.js";
15
- // Extended assessment modules - unified protocol compliance
16
- import { ProtocolComplianceAssessor } from "./modules/ProtocolComplianceAssessor.js";
17
- // New MCP Directory Compliance Gap assessors
18
- import { AUPComplianceAssessor } from "./modules/AUPComplianceAssessor.js";
19
- import { ToolAnnotationAssessor } from "./modules/ToolAnnotationAssessor.js";
20
- import { ProhibitedLibrariesAssessor } from "./modules/ProhibitedLibrariesAssessor.js";
21
- import { ManifestValidationAssessor } from "./modules/ManifestValidationAssessor.js";
22
- import { PortabilityAssessor } from "./modules/PortabilityAssessor.js";
23
- import { ExternalAPIScannerAssessor } from "./modules/ExternalAPIScannerAssessor.js";
24
- import { TemporalAssessor } from "./modules/TemporalAssessor.js";
25
- import { AuthenticationAssessor } from "./modules/AuthenticationAssessor.js";
26
- // New capability assessors
27
- import { ResourceAssessor } from "./modules/ResourceAssessor.js";
28
- import { PromptAssessor } from "./modules/PromptAssessor.js";
29
- import { CrossCapabilitySecurityAssessor } from "./modules/CrossCapabilitySecurityAssessor.js";
30
- // Code quality assessors
31
- import { FileModularizationAssessor } from "./modules/FileModularizationAssessor.js";
32
- // Official MCP conformance testing
33
- import { ConformanceAssessor } from "./modules/ConformanceAssessor.js";
34
- // Note: ProtocolConformanceAssessor merged into ProtocolComplianceAssessor (v1.25.2)
35
- // Pattern configuration for tool annotation assessment
36
- import { loadPatternConfig, compilePatterns, } from "./config/annotationPatterns.js";
9
+ // Note: All assessor module imports now handled by registry/AssessorDefinitions.ts (Issue #91)
37
10
  // Claude Code integration for intelligent analysis
38
11
  import { ClaudeCodeBridge, FULL_CLAUDE_CODE_CONFIG, } from "./lib/claudeCodeBridge.js";
39
12
  import { TestDataGenerator } from "./TestDataGenerator.js";
40
13
  // Structured logging
41
14
  import { createLogger, DEFAULT_LOGGING_CONFIG } from "./lib/logger.js";
42
15
  // Extracted helpers for testability
43
- import { emitModuleStartedEvent, emitModuleProgress, determineOverallStatus, generateSummary, generateRecommendations, } from "./orchestratorHelpers.js";
16
+ import { determineOverallStatus, generateSummary, generateRecommendations, } from "./orchestratorHelpers.js";
17
+ // Registry pattern for assessor management (Issue #91)
18
+ import { AssessorRegistry, ASSESSOR_DEFINITIONS } from "./registry/index.js";
19
+ // Module scoring for dual-key output (Issue #124)
20
+ import { calculateModuleScore } from "../../lib/moduleScoring.js";
44
21
  /**
45
22
  * Main orchestrator class for running MCP server assessments
46
23
  *
@@ -61,32 +38,9 @@ export class AssessmentOrchestrator {
61
38
  // Claude Code Bridge for intelligent analysis
62
39
  claudeBridge;
63
40
  claudeEnabled = false;
64
- // Core assessors (optional to support --skip-modules)
65
- functionalityAssessor;
66
- securityAssessor;
67
- documentationAssessor;
68
- errorHandlingAssessor;
69
- usabilityAssessor;
70
- // Extended assessors - unified protocol compliance
71
- protocolComplianceAssessor;
72
- // New MCP Directory Compliance Gap assessors
73
- aupComplianceAssessor;
74
- toolAnnotationAssessor;
75
- prohibitedLibrariesAssessor;
76
- manifestValidationAssessor;
77
- portabilityAssessor;
78
- externalAPIScannerAssessor;
79
- temporalAssessor;
80
- authenticationAssessor;
81
- // New capability assessors
82
- resourceAssessor;
83
- promptAssessor;
84
- crossCapabilityAssessor;
85
- // Code quality assessors
86
- fileModularizationAssessor;
87
- // Official MCP conformance testing (opt-in via --conformance flag)
88
- conformanceAssessor;
89
- // Note: protocolConformanceAssessor merged into protocolComplianceAssessor (v1.25.2)
41
+ // Registry for assessor management (Issue #91)
42
+ // Delegates construction, test count aggregation, and Claude bridge wiring
43
+ registry;
90
44
  constructor(config = {}) {
91
45
  this.config = { ...DEFAULT_ASSESSMENT_CONFIG, ...config };
92
46
  // Initialize logger
@@ -104,113 +58,22 @@ export class AssessmentOrchestrator {
104
58
  if (this.config.claudeCode?.enabled) {
105
59
  this.initializeClaudeBridge(this.config.claudeCode);
106
60
  }
107
- // Initialize core assessors (respects assessmentCategories config for --skip-modules)
108
- if (this.config.assessmentCategories?.functionality !== false) {
109
- this.functionalityAssessor = new FunctionalityAssessor(this.config);
110
- }
111
- if (this.config.assessmentCategories?.security !== false) {
112
- this.securityAssessor = new SecurityAssessor(this.config);
113
- // Wire up Claude bridge for security semantic analysis
114
- if (this.claudeBridge) {
115
- this.securityAssessor.setClaudeBridge(this.claudeBridge);
116
- }
117
- }
118
- if (this.config.assessmentCategories?.documentation !== false) {
119
- this.documentationAssessor = new DocumentationAssessor(this.config);
120
- }
121
- if (this.config.assessmentCategories?.errorHandling !== false) {
122
- this.errorHandlingAssessor = new ErrorHandlingAssessor(this.config);
123
- }
124
- if (this.config.assessmentCategories?.usability !== false) {
125
- this.usabilityAssessor = new UsabilityAssessor(this.config);
126
- }
127
- // Initialize extended assessors if enabled
128
- if (this.config.enableExtendedAssessment) {
129
- // Initialize unified protocol compliance assessor
130
- // Supports new protocolCompliance flag and deprecated mcpSpecCompliance/protocolConformance
131
- if (this.config.assessmentCategories?.protocolCompliance ||
132
- this.config.assessmentCategories?.mcpSpecCompliance ||
133
- this.config.assessmentCategories?.protocolConformance) {
134
- this.protocolComplianceAssessor = new ProtocolComplianceAssessor(this.config);
135
- }
136
- // Initialize new MCP Directory Compliance Gap assessors
137
- if (this.config.assessmentCategories?.aupCompliance) {
138
- this.aupComplianceAssessor = new AUPComplianceAssessor(this.config);
139
- // Wire up Claude bridge for semantic analysis
140
- if (this.claudeBridge) {
141
- this.aupComplianceAssessor.setClaudeBridge(this.claudeBridge);
142
- }
143
- }
144
- if (this.config.assessmentCategories?.toolAnnotations) {
145
- this.toolAnnotationAssessor = new ToolAnnotationAssessor(this.config);
146
- // Wire up Claude bridge for behavior inference
147
- if (this.claudeBridge) {
148
- this.toolAnnotationAssessor.setClaudeBridge(this.claudeBridge);
149
- }
150
- // Load custom pattern configuration if provided
151
- if (this.config.patternConfigPath) {
152
- const patternConfig = loadPatternConfig(this.config.patternConfigPath, this.logger);
153
- const compiledPatterns = compilePatterns(patternConfig);
154
- this.toolAnnotationAssessor.setPatterns(compiledPatterns);
155
- }
156
- }
157
- if (this.config.assessmentCategories?.prohibitedLibraries) {
158
- this.prohibitedLibrariesAssessor = new ProhibitedLibrariesAssessor(this.config);
159
- }
160
- if (this.config.assessmentCategories?.manifestValidation) {
161
- this.manifestValidationAssessor = new ManifestValidationAssessor(this.config);
162
- }
163
- if (this.config.assessmentCategories?.portability) {
164
- this.portabilityAssessor = new PortabilityAssessor(this.config);
165
- }
166
- if (this.config.assessmentCategories?.externalAPIScanner) {
167
- this.externalAPIScannerAssessor = new ExternalAPIScannerAssessor(this.config);
168
- }
169
- if (this.config.assessmentCategories?.temporal) {
170
- this.temporalAssessor = new TemporalAssessor(this.config);
171
- }
172
- if (this.config.assessmentCategories?.authentication) {
173
- this.authenticationAssessor = new AuthenticationAssessor(this.config);
174
- }
175
- // Initialize new capability assessors
176
- if (this.config.assessmentCategories?.resources) {
177
- this.resourceAssessor = new ResourceAssessor(this.config);
178
- }
179
- if (this.config.assessmentCategories?.prompts) {
180
- this.promptAssessor = new PromptAssessor(this.config);
181
- }
182
- if (this.config.assessmentCategories?.crossCapability) {
183
- this.crossCapabilityAssessor = new CrossCapabilitySecurityAssessor(this.config);
184
- }
185
- // Initialize code quality assessors
186
- if (this.config.assessmentCategories?.fileModularization) {
187
- this.fileModularizationAssessor = new FileModularizationAssessor(this.config);
188
- }
189
- // Initialize official MCP conformance testing (opt-in via --conformance flag)
190
- // Requires HTTP/SSE transport with serverUrl available
191
- if (this.config.assessmentCategories?.conformance) {
192
- this.conformanceAssessor = new ConformanceAssessor(this.config);
193
- }
194
- // Note: Protocol conformance now handled by unified ProtocolComplianceAssessor above
195
- }
196
- // Wire up Claude bridge to TestDataGenerator for intelligent test generation
61
+ // Initialize registry and register all enabled assessors (Issue #91)
62
+ // The registry handles:
63
+ // - Conditional instantiation based on config flags
64
+ // - Deprecated flag OR logic (e.g., protocolCompliance supports 3 flags)
65
+ // - Custom setup (e.g., ToolAnnotationAssessor pattern config)
66
+ // - Claude bridge wiring for supporting assessors
67
+ this.registry = new AssessorRegistry(this.config);
68
+ this.registry.registerAll(ASSESSOR_DEFINITIONS);
69
+ // Wire up Claude bridge to registry (handles all supporting assessors)
197
70
  if (this.claudeBridge) {
71
+ this.registry.setClaudeBridge(this.claudeBridge);
198
72
  TestDataGenerator.setClaudeBridge(this.claudeBridge);
199
73
  }
200
74
  // Set logger for TestDataGenerator diagnostic output
201
75
  TestDataGenerator.setLogger(this.logger);
202
76
  }
203
- /**
204
- * Get the count of tools that will actually be tested based on selectedToolsForTesting config.
205
- * Used for accurate progress estimation in emitModuleStartedEvent calls.
206
- */
207
- getToolCountForTesting(tools) {
208
- if (this.config.selectedToolsForTesting !== undefined) {
209
- const selectedNames = new Set(this.config.selectedToolsForTesting);
210
- return tools.filter((tool) => selectedNames.has(tool.name)).length;
211
- }
212
- return tools.length;
213
- }
214
77
  /**
215
78
  * Initialize Claude Code Bridge for intelligent analysis
216
79
  * This enables semantic AUP violation analysis, behavior inference, and intelligent test generation
@@ -242,17 +105,9 @@ export class AssessmentOrchestrator {
242
105
  enabled: true,
243
106
  };
244
107
  this.initializeClaudeBridge(bridgeConfig);
245
- // Wire up to existing assessors
108
+ // Wire up to all supporting assessors via registry
246
109
  if (this.claudeBridge) {
247
- if (this.aupComplianceAssessor) {
248
- this.aupComplianceAssessor.setClaudeBridge(this.claudeBridge);
249
- }
250
- if (this.toolAnnotationAssessor) {
251
- this.toolAnnotationAssessor.setClaudeBridge(this.claudeBridge);
252
- }
253
- if (this.securityAssessor) {
254
- this.securityAssessor.setClaudeBridge(this.claudeBridge);
255
- }
110
+ this.registry.setClaudeBridge(this.claudeBridge);
256
111
  TestDataGenerator.setClaudeBridge(this.claudeBridge);
257
112
  }
258
113
  }
@@ -270,55 +125,6 @@ export class AssessmentOrchestrator {
270
125
  getClaudeBridge() {
271
126
  return this.claudeBridge;
272
127
  }
273
- /**
274
- * Reset test counts for all assessors
275
- */
276
- resetAllTestCounts() {
277
- this.functionalityAssessor?.resetTestCount();
278
- this.securityAssessor?.resetTestCount();
279
- this.documentationAssessor?.resetTestCount();
280
- this.errorHandlingAssessor?.resetTestCount();
281
- this.usabilityAssessor?.resetTestCount();
282
- if (this.protocolComplianceAssessor) {
283
- this.protocolComplianceAssessor.resetTestCount();
284
- }
285
- // Reset new assessors
286
- if (this.aupComplianceAssessor) {
287
- this.aupComplianceAssessor.resetTestCount();
288
- }
289
- if (this.toolAnnotationAssessor) {
290
- this.toolAnnotationAssessor.resetTestCount();
291
- }
292
- if (this.prohibitedLibrariesAssessor) {
293
- this.prohibitedLibrariesAssessor.resetTestCount();
294
- }
295
- if (this.manifestValidationAssessor) {
296
- this.manifestValidationAssessor.resetTestCount();
297
- }
298
- if (this.portabilityAssessor) {
299
- this.portabilityAssessor.resetTestCount();
300
- }
301
- if (this.authenticationAssessor) {
302
- this.authenticationAssessor.resetTestCount();
303
- }
304
- // Reset new capability assessors
305
- if (this.resourceAssessor) {
306
- this.resourceAssessor.resetTestCount();
307
- }
308
- if (this.promptAssessor) {
309
- this.promptAssessor.resetTestCount();
310
- }
311
- if (this.crossCapabilityAssessor) {
312
- this.crossCapabilityAssessor.resetTestCount();
313
- }
314
- if (this.fileModularizationAssessor) {
315
- this.fileModularizationAssessor.resetTestCount();
316
- }
317
- // Reset official conformance assessor
318
- if (this.conformanceAssessor) {
319
- this.conformanceAssessor.resetTestCount();
320
- }
321
- }
322
128
  /**
323
129
  * Run a complete assessment on an MCP server
324
130
  * @public
@@ -326,298 +132,14 @@ export class AssessmentOrchestrator {
326
132
  async runFullAssessment(context) {
327
133
  this.startTime = Date.now();
328
134
  this.totalTestsRun = 0;
329
- this.resetAllTestCounts();
330
- // Run assessments in parallel if enabled
331
- const assessmentPromises = [];
332
- const assessmentResults = {};
333
- // PHASE 0: Temporal Assessment (ALWAYS runs first, before parallel/sequential phases)
334
- // This ensures temporal captures clean baseline before other modules trigger rug pulls
335
- if (this.temporalAssessor) {
336
- const toolCount = this.getToolCountForTesting(context.tools);
337
- const invocationsPerTool = this.config.temporalInvocations ?? 25;
338
- emitModuleStartedEvent("Temporal", toolCount * invocationsPerTool, toolCount);
339
- assessmentResults.temporal = await this.temporalAssessor.assess(context);
340
- emitModuleProgress("Temporal", assessmentResults.temporal.status, assessmentResults.temporal, this.temporalAssessor.getTestCount());
341
- }
342
- if (this.config.parallelTesting) {
343
- // Calculate estimates for module_started events
344
- const toolCount = this.getToolCountForTesting(context.tools);
345
- const securityPatterns = this.config.securityPatternsToTest || 17;
346
- // Core assessments - only emit and run if not skipped
347
- if (this.functionalityAssessor) {
348
- emitModuleStartedEvent("Functionality", toolCount * 10, toolCount);
349
- assessmentPromises.push(this.functionalityAssessor.assess(context).then((r) => {
350
- emitModuleProgress("Functionality", r.status, r, this.functionalityAssessor.getTestCount());
351
- return (assessmentResults.functionality = r);
352
- }));
353
- }
354
- if (this.securityAssessor) {
355
- emitModuleStartedEvent("Security", securityPatterns * toolCount, toolCount);
356
- assessmentPromises.push(this.securityAssessor.assess(context).then((r) => {
357
- emitModuleProgress("Security", r.status, r, this.securityAssessor.getTestCount());
358
- return (assessmentResults.security = r);
359
- }));
360
- }
361
- if (this.documentationAssessor) {
362
- emitModuleStartedEvent("Documentation", 5, toolCount);
363
- assessmentPromises.push(this.documentationAssessor.assess(context).then((r) => {
364
- emitModuleProgress("Documentation", r.status, r, this.documentationAssessor.getTestCount());
365
- return (assessmentResults.documentation = r);
366
- }));
367
- }
368
- if (this.errorHandlingAssessor) {
369
- emitModuleStartedEvent("Error Handling", toolCount * 5, toolCount);
370
- assessmentPromises.push(this.errorHandlingAssessor.assess(context).then((r) => {
371
- emitModuleProgress("Error Handling", r.status, r, this.errorHandlingAssessor.getTestCount());
372
- return (assessmentResults.errorHandling = r);
373
- }));
374
- }
375
- if (this.usabilityAssessor) {
376
- emitModuleStartedEvent("Usability", 10, toolCount);
377
- assessmentPromises.push(this.usabilityAssessor.assess(context).then((r) => {
378
- emitModuleProgress("Usability", r.status, r, this.usabilityAssessor.getTestCount());
379
- return (assessmentResults.usability = r);
380
- }));
381
- }
382
- // Extended assessments - unified protocol compliance
383
- if (this.protocolComplianceAssessor) {
384
- emitModuleStartedEvent("Protocol Compliance", 10, toolCount);
385
- assessmentPromises.push(this.protocolComplianceAssessor.assess(context).then((r) => {
386
- emitModuleProgress("Protocol Compliance", r.status, r, this.protocolComplianceAssessor.getTestCount());
387
- return (assessmentResults.mcpSpecCompliance = r);
388
- }));
389
- }
390
- // New MCP Directory Compliance Gap assessments
391
- if (this.aupComplianceAssessor) {
392
- emitModuleStartedEvent("AUP", 20, toolCount);
393
- assessmentPromises.push(this.aupComplianceAssessor.assess(context).then((r) => {
394
- emitModuleProgress("AUP", r.status, r, this.aupComplianceAssessor.getTestCount());
395
- return (assessmentResults.aupCompliance = r);
396
- }));
397
- }
398
- if (this.toolAnnotationAssessor) {
399
- emitModuleStartedEvent("Annotations", toolCount, toolCount);
400
- assessmentPromises.push(this.toolAnnotationAssessor.assess(context).then((r) => {
401
- emitModuleProgress("Annotations", r.status, r, this.toolAnnotationAssessor.getTestCount());
402
- return (assessmentResults.toolAnnotations = r);
403
- }));
404
- }
405
- if (this.prohibitedLibrariesAssessor) {
406
- emitModuleStartedEvent("Libraries", 5, toolCount);
407
- assessmentPromises.push(this.prohibitedLibrariesAssessor.assess(context).then((r) => {
408
- emitModuleProgress("Libraries", r.status, r, this.prohibitedLibrariesAssessor.getTestCount());
409
- return (assessmentResults.prohibitedLibraries = r);
410
- }));
411
- }
412
- if (this.manifestValidationAssessor) {
413
- emitModuleStartedEvent("Manifest", 10, toolCount);
414
- assessmentPromises.push(this.manifestValidationAssessor.assess(context).then((r) => {
415
- emitModuleProgress("Manifest", r.status, r, this.manifestValidationAssessor.getTestCount());
416
- return (assessmentResults.manifestValidation = r);
417
- }));
418
- }
419
- if (this.portabilityAssessor) {
420
- emitModuleStartedEvent("Portability", 10, toolCount);
421
- assessmentPromises.push(this.portabilityAssessor.assess(context).then((r) => {
422
- emitModuleProgress("Portability", r.status, r, this.portabilityAssessor.getTestCount());
423
- return (assessmentResults.portability = r);
424
- }));
425
- }
426
- if (this.externalAPIScannerAssessor) {
427
- emitModuleStartedEvent("External APIs", 10, toolCount);
428
- assessmentPromises.push(this.externalAPIScannerAssessor.assess(context).then((r) => {
429
- emitModuleProgress("External APIs", r.status, r, this.externalAPIScannerAssessor.getTestCount());
430
- return (assessmentResults.externalAPIScanner = r);
431
- }));
432
- }
433
- if (this.authenticationAssessor) {
434
- const sourceFileCount = context.sourceCodeFiles?.size || 0;
435
- emitModuleStartedEvent("Authentication", sourceFileCount, sourceFileCount);
436
- assessmentPromises.push(this.authenticationAssessor.assess(context).then((r) => {
437
- emitModuleProgress("Authentication", r.status, r, this.authenticationAssessor.getTestCount());
438
- return (assessmentResults.authentication = r);
439
- }));
440
- }
441
- // NOTE: Temporal runs in PHASE 0 above, not in parallel with other modules
442
- // New capability assessors
443
- if (this.resourceAssessor) {
444
- const resourceCount = (context.resources?.length || 0) +
445
- (context.resourceTemplates?.length || 0);
446
- emitModuleStartedEvent("Resources", resourceCount * 5, resourceCount);
447
- assessmentPromises.push(this.resourceAssessor.assess(context).then((r) => {
448
- emitModuleProgress("Resources", r.status, r, this.resourceAssessor.getTestCount());
449
- return (assessmentResults.resources = r);
450
- }));
451
- }
452
- if (this.promptAssessor) {
453
- const promptCount = context.prompts?.length || 0;
454
- emitModuleStartedEvent("Prompts", promptCount * 10, promptCount);
455
- assessmentPromises.push(this.promptAssessor.assess(context).then((r) => {
456
- emitModuleProgress("Prompts", r.status, r, this.promptAssessor.getTestCount());
457
- return (assessmentResults.prompts = r);
458
- }));
459
- }
460
- if (this.crossCapabilityAssessor) {
461
- const capabilityCount = toolCount +
462
- (context.resources?.length || 0) +
463
- (context.prompts?.length || 0);
464
- emitModuleStartedEvent("Cross-Capability", capabilityCount * 3, capabilityCount);
465
- assessmentPromises.push(this.crossCapabilityAssessor.assess(context).then((r) => {
466
- emitModuleProgress("Cross-Capability", r.status, r, this.crossCapabilityAssessor.getTestCount());
467
- return (assessmentResults.crossCapability = r);
468
- }));
469
- }
470
- if (this.fileModularizationAssessor) {
471
- const sourceFileCount = context.sourceCodeFiles?.size || 0;
472
- emitModuleStartedEvent("File Modularization", sourceFileCount, sourceFileCount);
473
- assessmentPromises.push(this.fileModularizationAssessor.assess(context).then((r) => {
474
- emitModuleProgress("File Modularization", r.status, r, this.fileModularizationAssessor.getTestCount());
475
- return (assessmentResults.fileModularization = r);
476
- }));
477
- }
478
- // Official MCP conformance testing (opt-in, requires HTTP/SSE transport)
479
- if (this.conformanceAssessor) {
480
- // Conformance tests ~7 server scenarios
481
- emitModuleStartedEvent("Conformance", 7, toolCount);
482
- assessmentPromises.push(this.conformanceAssessor.assess(context).then((r) => {
483
- emitModuleProgress("Conformance", r.status, r, this.conformanceAssessor.getTestCount());
484
- return (assessmentResults.conformance = r);
485
- }));
486
- }
487
- // Note: Protocol Conformance now handled by unified ProtocolComplianceAssessor above
488
- await Promise.all(assessmentPromises);
489
- }
490
- else {
491
- // Sequential execution with module_started events
492
- const toolCount = this.getToolCountForTesting(context.tools);
493
- const securityPatterns = this.config.securityPatternsToTest || 17;
494
- // NOTE: Temporal runs in PHASE 0 above, before sequential/parallel phases
495
- // Core assessments - only emit and run if not skipped
496
- if (this.functionalityAssessor) {
497
- // Functionality: ~10 scenarios per tool
498
- emitModuleStartedEvent("Functionality", toolCount * 10, toolCount);
499
- assessmentResults.functionality =
500
- await this.functionalityAssessor.assess(context);
501
- emitModuleProgress("Functionality", assessmentResults.functionality.status, assessmentResults.functionality, this.functionalityAssessor.getTestCount());
502
- }
503
- if (this.securityAssessor) {
504
- // Security: patterns × tools
505
- emitModuleStartedEvent("Security", securityPatterns * toolCount, toolCount);
506
- assessmentResults.security =
507
- await this.securityAssessor.assess(context);
508
- emitModuleProgress("Security", assessmentResults.security.status, assessmentResults.security, this.securityAssessor.getTestCount());
509
- }
510
- if (this.documentationAssessor) {
511
- // Documentation: ~5 static tests
512
- emitModuleStartedEvent("Documentation", 5, toolCount);
513
- assessmentResults.documentation =
514
- await this.documentationAssessor.assess(context);
515
- emitModuleProgress("Documentation", assessmentResults.documentation.status, assessmentResults.documentation, this.documentationAssessor.getTestCount());
516
- }
517
- if (this.errorHandlingAssessor) {
518
- // Error Handling: ~5 tests per tool
519
- emitModuleStartedEvent("Error Handling", toolCount * 5, toolCount);
520
- assessmentResults.errorHandling =
521
- await this.errorHandlingAssessor.assess(context);
522
- emitModuleProgress("Error Handling", assessmentResults.errorHandling.status, assessmentResults.errorHandling, this.errorHandlingAssessor.getTestCount());
523
- }
524
- if (this.usabilityAssessor) {
525
- // Usability: ~10 static tests
526
- emitModuleStartedEvent("Usability", 10, toolCount);
527
- assessmentResults.usability =
528
- await this.usabilityAssessor.assess(context);
529
- emitModuleProgress("Usability", assessmentResults.usability.status, assessmentResults.usability, this.usabilityAssessor.getTestCount());
530
- }
531
- if (this.protocolComplianceAssessor) {
532
- emitModuleStartedEvent("Protocol Compliance", 10, toolCount);
533
- assessmentResults.mcpSpecCompliance =
534
- await this.protocolComplianceAssessor.assess(context);
535
- emitModuleProgress("Protocol Compliance", assessmentResults.mcpSpecCompliance.status, assessmentResults.mcpSpecCompliance, this.protocolComplianceAssessor.getTestCount());
536
- }
537
- // New MCP Directory Compliance Gap assessments (sequential)
538
- if (this.aupComplianceAssessor) {
539
- emitModuleStartedEvent("AUP", 20, toolCount);
540
- assessmentResults.aupCompliance =
541
- await this.aupComplianceAssessor.assess(context);
542
- emitModuleProgress("AUP", assessmentResults.aupCompliance.status, assessmentResults.aupCompliance, this.aupComplianceAssessor.getTestCount());
543
- }
544
- if (this.toolAnnotationAssessor) {
545
- emitModuleStartedEvent("Annotations", toolCount, toolCount);
546
- assessmentResults.toolAnnotations =
547
- await this.toolAnnotationAssessor.assess(context);
548
- emitModuleProgress("Annotations", assessmentResults.toolAnnotations.status, assessmentResults.toolAnnotations, this.toolAnnotationAssessor.getTestCount());
549
- }
550
- if (this.prohibitedLibrariesAssessor) {
551
- emitModuleStartedEvent("Libraries", 5, toolCount);
552
- assessmentResults.prohibitedLibraries =
553
- await this.prohibitedLibrariesAssessor.assess(context);
554
- emitModuleProgress("Libraries", assessmentResults.prohibitedLibraries.status, assessmentResults.prohibitedLibraries, this.prohibitedLibrariesAssessor.getTestCount());
555
- }
556
- if (this.manifestValidationAssessor) {
557
- emitModuleStartedEvent("Manifest", 10, toolCount);
558
- assessmentResults.manifestValidation =
559
- await this.manifestValidationAssessor.assess(context);
560
- emitModuleProgress("Manifest", assessmentResults.manifestValidation.status, assessmentResults.manifestValidation, this.manifestValidationAssessor.getTestCount());
561
- }
562
- if (this.portabilityAssessor) {
563
- emitModuleStartedEvent("Portability", 10, toolCount);
564
- assessmentResults.portability =
565
- await this.portabilityAssessor.assess(context);
566
- emitModuleProgress("Portability", assessmentResults.portability.status, assessmentResults.portability, this.portabilityAssessor.getTestCount());
567
- }
568
- if (this.externalAPIScannerAssessor) {
569
- emitModuleStartedEvent("External APIs", 10, toolCount);
570
- assessmentResults.externalAPIScanner =
571
- await this.externalAPIScannerAssessor.assess(context);
572
- emitModuleProgress("External APIs", assessmentResults.externalAPIScanner.status, assessmentResults.externalAPIScanner, this.externalAPIScannerAssessor.getTestCount());
573
- }
574
- if (this.authenticationAssessor) {
575
- const sourceFileCount = context.sourceCodeFiles?.size || 0;
576
- emitModuleStartedEvent("Authentication", sourceFileCount, sourceFileCount);
577
- assessmentResults.authentication =
578
- await this.authenticationAssessor.assess(context);
579
- emitModuleProgress("Authentication", assessmentResults.authentication.status, assessmentResults.authentication, this.authenticationAssessor.getTestCount());
580
- }
581
- // New capability assessors (sequential)
582
- if (this.resourceAssessor) {
583
- const resourceCount = (context.resources?.length || 0) +
584
- (context.resourceTemplates?.length || 0);
585
- emitModuleStartedEvent("Resources", resourceCount * 5, resourceCount);
586
- assessmentResults.resources =
587
- await this.resourceAssessor.assess(context);
588
- emitModuleProgress("Resources", assessmentResults.resources.status, assessmentResults.resources, this.resourceAssessor.getTestCount());
589
- }
590
- if (this.promptAssessor) {
591
- const promptCount = context.prompts?.length || 0;
592
- emitModuleStartedEvent("Prompts", promptCount * 10, promptCount);
593
- assessmentResults.prompts = await this.promptAssessor.assess(context);
594
- emitModuleProgress("Prompts", assessmentResults.prompts.status, assessmentResults.prompts, this.promptAssessor.getTestCount());
595
- }
596
- if (this.crossCapabilityAssessor) {
597
- const capabilityCount = toolCount +
598
- (context.resources?.length || 0) +
599
- (context.prompts?.length || 0);
600
- emitModuleStartedEvent("Cross-Capability", capabilityCount * 3, capabilityCount);
601
- assessmentResults.crossCapability =
602
- await this.crossCapabilityAssessor.assess(context);
603
- emitModuleProgress("Cross-Capability", assessmentResults.crossCapability.status, assessmentResults.crossCapability, this.crossCapabilityAssessor.getTestCount());
604
- }
605
- if (this.fileModularizationAssessor) {
606
- const sourceFileCount = context.sourceCodeFiles?.size || 0;
607
- emitModuleStartedEvent("File Modularization", sourceFileCount, sourceFileCount);
608
- assessmentResults.fileModularization =
609
- await this.fileModularizationAssessor.assess(context);
610
- emitModuleProgress("File Modularization", assessmentResults.fileModularization.status, assessmentResults.fileModularization, this.fileModularizationAssessor.getTestCount());
611
- }
612
- // Official MCP conformance testing (sequential, opt-in)
613
- if (this.conformanceAssessor) {
614
- emitModuleStartedEvent("Conformance", 7, toolCount);
615
- assessmentResults.conformance =
616
- await this.conformanceAssessor.assess(context);
617
- emitModuleProgress("Conformance", assessmentResults.conformance.status, assessmentResults.conformance, this.conformanceAssessor.getTestCount());
618
- }
619
- // Note: Protocol Conformance now handled by unified ProtocolComplianceAssessor above
620
- }
135
+ this.registry.resetAllTestCounts();
136
+ // Execute all assessors via registry (Issue #91)
137
+ // Registry handles:
138
+ // - Phase-ordered execution (Phase 0/PRE always runs first and sequentially)
139
+ // - Parallel vs sequential based on config.parallelTesting
140
+ // - JSONL events (module_started, module_progress)
141
+ // - Test count tracking
142
+ const assessmentResults = await this.registry.executeAll(context);
621
143
  // Integrate temporal findings into security.vulnerabilities for unified view
622
144
  if (assessmentResults.temporal?.rugPullsDetected &&
623
145
  assessmentResults.temporal.rugPullsDetected > 0 &&
@@ -626,6 +148,41 @@ export class AssessmentOrchestrator {
626
148
  assessmentResults.security.vulnerabilities.push(`RUG_PULL_TEMPORAL: ${detail.tool} - Tool behavior changed after invocation ${detail.firstDeviationAt}. Requires immediate manual review.`);
627
149
  }
628
150
  }
151
+ // Issue #124: Dual-key output for v2.0.0 transition
152
+ // Output BOTH old and new keys to maintain backward compatibility
153
+ // Old keys (documentation, usability, mcpSpecCompliance) will be removed in v2.0.0
154
+ // developerExperience (new) = documentation + usability (deprecated)
155
+ if (assessmentResults.documentation && assessmentResults.usability) {
156
+ const docScore = calculateModuleScore(assessmentResults.documentation) ?? 50;
157
+ const usabilityScore = calculateModuleScore(assessmentResults.usability) ?? 50;
158
+ const combinedStatus = determineOverallStatus({
159
+ documentation: assessmentResults.documentation,
160
+ usability: assessmentResults.usability,
161
+ });
162
+ assessmentResults.developerExperience = {
163
+ documentation: assessmentResults.documentation,
164
+ usability: assessmentResults.usability,
165
+ status: combinedStatus,
166
+ score: Math.round((docScore + usabilityScore) / 2),
167
+ };
168
+ // Emit deprecation warning for old keys
169
+ this.logger.warn("Output keys 'documentation' and 'usability' are deprecated. " +
170
+ "Use 'developerExperience' instead. These keys will be removed in v2.0.0.", {
171
+ deprecated: ["documentation", "usability"],
172
+ replacement: "developerExperience",
173
+ });
174
+ }
175
+ // protocolCompliance (new) = mcpSpecCompliance (deprecated)
176
+ if (assessmentResults.mcpSpecCompliance) {
177
+ assessmentResults.protocolCompliance =
178
+ assessmentResults.mcpSpecCompliance;
179
+ // Emit deprecation warning for old key
180
+ this.logger.warn("Output key 'mcpSpecCompliance' is deprecated. " +
181
+ "Use 'protocolCompliance' instead. This key will be removed in v2.0.0.", {
182
+ deprecated: ["mcpSpecCompliance"],
183
+ replacement: "protocolCompliance",
184
+ });
185
+ }
629
186
  // Collect test counts from all assessors
630
187
  this.totalTestsRun = this.collectTotalTestCount();
631
188
  // Determine overall status
@@ -674,75 +231,8 @@ export class AssessmentOrchestrator {
674
231
  return this.runFullAssessment(context);
675
232
  }
676
233
  collectTotalTestCount() {
677
- let total = 0;
678
- // Get actual test counts from assessors (optional for --skip-modules support)
679
- const functionalityCount = this.functionalityAssessor?.getTestCount() || 0;
680
- const securityCount = this.securityAssessor?.getTestCount() || 0;
681
- const documentationCount = this.documentationAssessor?.getTestCount() || 0;
682
- const errorHandlingCount = this.errorHandlingAssessor?.getTestCount() || 0;
683
- const usabilityCount = this.usabilityAssessor?.getTestCount() || 0;
684
- const mcpSpecCount = this.protocolComplianceAssessor?.getTestCount() || 0;
685
- // New assessor counts
686
- const aupCount = this.aupComplianceAssessor?.getTestCount() || 0;
687
- const annotationCount = this.toolAnnotationAssessor?.getTestCount() || 0;
688
- const librariesCount = this.prohibitedLibrariesAssessor?.getTestCount() || 0;
689
- const manifestCount = this.manifestValidationAssessor?.getTestCount() || 0;
690
- const portabilityCount = this.portabilityAssessor?.getTestCount() || 0;
691
- const authenticationCount = this.authenticationAssessor?.getTestCount() || 0;
692
- const externalAPICount = this.externalAPIScannerAssessor?.getTestCount() || 0;
693
- const temporalCount = this.temporalAssessor?.getTestCount() || 0;
694
- // New capability assessor counts
695
- const resourcesCount = this.resourceAssessor?.getTestCount() || 0;
696
- const promptsCount = this.promptAssessor?.getTestCount() || 0;
697
- const crossCapabilityCount = this.crossCapabilityAssessor?.getTestCount() || 0;
698
- // Code quality assessor counts
699
- const fileModularizationCount = this.fileModularizationAssessor?.getTestCount() || 0;
700
- // Official MCP conformance test count
701
- const conformanceCount = this.conformanceAssessor?.getTestCount() || 0;
702
- // Note: Protocol conformance now included in mcpSpecCount (unified ProtocolComplianceAssessor)
703
- this.logger.debug("Test counts by assessor", {
704
- functionality: functionalityCount,
705
- security: securityCount,
706
- documentation: documentationCount,
707
- errorHandling: errorHandlingCount,
708
- usability: usabilityCount,
709
- mcpSpec: mcpSpecCount,
710
- aupCompliance: aupCount,
711
- toolAnnotations: annotationCount,
712
- prohibitedLibraries: librariesCount,
713
- manifestValidation: manifestCount,
714
- portability: portabilityCount,
715
- authentication: authenticationCount,
716
- externalAPIScanner: externalAPICount,
717
- temporal: temporalCount,
718
- resources: resourcesCount,
719
- prompts: promptsCount,
720
- crossCapability: crossCapabilityCount,
721
- fileModularization: fileModularizationCount,
722
- conformance: conformanceCount,
723
- // Note: protocolConformance now included in mcpSpec (unified)
724
- });
725
- total =
726
- functionalityCount +
727
- securityCount +
728
- documentationCount +
729
- errorHandlingCount +
730
- usabilityCount +
731
- mcpSpecCount +
732
- aupCount +
733
- annotationCount +
734
- librariesCount +
735
- manifestCount +
736
- portabilityCount +
737
- authenticationCount +
738
- externalAPICount +
739
- temporalCount +
740
- resourcesCount +
741
- promptsCount +
742
- crossCapabilityCount +
743
- fileModularizationCount +
744
- conformanceCount;
745
- // Note: protocolConformance now included in mcpSpecCount (unified)
234
+ // Delegate to registry for centralized test count aggregation (Issue #91)
235
+ const total = this.registry.getTotalTestCount();
746
236
  this.logger.debug("Total test count", { total });
747
237
  return total;
748
238
  }