@bryan-thompson/inspector-assessment 1.43.2 → 1.43.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +1062 -224
  2. package/cli/build/assess-full.js +532 -106
  3. package/cli/build/assess-security.js +54 -90
  4. package/cli/build/lib/cli-parser.js +14 -1
  5. package/cli/build/lib/cli-parserSchemas.js +1 -0
  6. package/cli/build/lib/result-output.js +21 -0
  7. package/cli/build/profiles.js +20 -0
  8. package/cli/build/validate-testbed.js +0 -0
  9. package/cli/package.json +1 -1
  10. package/client/dist/assets/{OAuthCallback-BS8-A1sU.js → OAuthCallback-Chi58kRc.js} +1 -1
  11. package/client/dist/assets/{OAuthDebugCallback-025_TM2i.js → OAuthDebugCallback-BluD_Wxg.js} +1 -1
  12. package/client/dist/assets/{index-DEhlIjy-.js → index-KW2LwGdp.js} +4 -4
  13. package/client/dist/index.html +1 -1
  14. package/client/lib/lib/assessment/configSchemas.d.ts +64 -64
  15. package/client/lib/lib/assessment/jsonlEventSchemas.d.ts +286 -286
  16. package/client/lib/lib/assessment/resultTypes.d.ts +10 -0
  17. package/client/lib/lib/assessment/resultTypes.d.ts.map +1 -1
  18. package/client/lib/lib/assessmentTypes.d.ts +1 -20
  19. package/client/lib/lib/assessmentTypes.d.ts.map +1 -1
  20. package/client/lib/lib/assessmentTypes.js +1 -20
  21. package/client/lib/services/assessment/AssessmentOrchestrator.d.ts +57 -104
  22. package/client/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
  23. package/client/lib/services/assessment/AssessmentOrchestrator.js +298 -133
  24. package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts +25 -0
  25. package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts.map +1 -0
  26. package/client/lib/services/assessment/modules/ErrorHandlingAssessor.deprecated.js +1 -1
  27. package/client/lib/services/assessment/modules/ErrorHandlingAssessor.js +564 -0
  28. package/client/lib/services/assessment/modules/SecurityAssessor.d.ts +5 -0
  29. package/client/lib/services/assessment/modules/SecurityAssessor.d.ts.map +1 -1
  30. package/client/lib/services/assessment/modules/SecurityAssessor.js +62 -0
  31. package/client/lib/services/assessment/modules/index.d.ts +1 -1
  32. package/client/lib/services/assessment/modules/index.js +1 -1
  33. package/client/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts +15 -0
  34. package/client/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts.map +1 -1
  35. package/client/lib/services/assessment/modules/securityTests/SecurityPayloadTester.js +72 -0
  36. package/client/lib/services/assessment/modules/securityTests/factory.d.ts +2 -0
  37. package/client/lib/services/assessment/modules/securityTests/factory.d.ts.map +1 -1
  38. package/client/lib/services/assessment/modules/securityTests/factory.js +1 -0
  39. package/client/lib/services/assessment/registry/AssessorDefinitions.js +1 -1
  40. package/client/lib/services/assessment/responseValidatorSchemas.d.ts +12 -12
  41. package/client/package.json +3 -3
  42. package/package.json +4 -2
  43. package/server/package.json +1 -1
  44. package/cli/build/lib/__tests__/zodErrorFormatter.test.js +0 -282
  45. package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts +0 -109
  46. package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts.map +0 -1
  47. package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.deprecated.d.ts +0 -109
  48. package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.deprecated.d.ts.map +0 -1
  49. package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.deprecated.js +0 -852
  50. package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.js +0 -852
@@ -1,78 +1,92 @@
1
1
  /**
2
2
  * Assessment Orchestrator
3
3
  * Coordinates all assessment modules and manages the assessment workflow
4
- *
5
- * @public
6
- * @module AssessmentOrchestrator
7
4
  */
8
5
  import { DEFAULT_ASSESSMENT_CONFIG, } from "../../lib/assessmentTypes.js";
9
- // Note: All assessor module imports now handled by registry/AssessorDefinitions.ts (Issue #91)
6
+ // Core assessment modules
7
+ import { FunctionalityAssessor } from "./modules/FunctionalityAssessor.js";
8
+ import { SecurityAssessor } from "./modules/SecurityAssessor.js";
9
+ import { DocumentationAssessor } from "./modules/DocumentationAssessor.js";
10
+ import { ErrorHandlingAssessor } from "./modules/ErrorHandlingAssessor.js";
11
+ import { UsabilityAssessor } from "./modules/UsabilityAssessor.js";
12
+ // Extended assessment modules
13
+ import { MCPSpecComplianceAssessor } from "./modules/MCPSpecComplianceAssessor.js";
14
+ // New MCP Directory Compliance Gap assessors
15
+ import { AUPComplianceAssessor } from "./modules/AUPComplianceAssessor.js";
16
+ import { ToolAnnotationAssessor } from "./modules/ToolAnnotationAssessor.js";
17
+ import { ProhibitedLibrariesAssessor } from "./modules/ProhibitedLibrariesAssessor.js";
18
+ import { ManifestValidationAssessor } from "./modules/ManifestValidationAssessor.js";
19
+ import { PortabilityAssessor } from "./modules/PortabilityAssessor.js";
10
20
  // Claude Code integration for intelligent analysis
11
21
  import { ClaudeCodeBridge, FULL_CLAUDE_CODE_CONFIG, } from "./lib/claudeCodeBridge.js";
12
22
  import { TestDataGenerator } from "./TestDataGenerator.js";
13
- // Structured logging
14
- import { createLogger, DEFAULT_LOGGING_CONFIG } from "./lib/logger.js";
15
- // Extracted helpers for testability
16
- import { determineOverallStatus, generateSummary, generateRecommendations, } from "./orchestratorHelpers.js";
17
- // Registry pattern for assessor management (Issue #91)
18
- import { AssessorRegistry, ASSESSOR_DEFINITIONS } from "./registry/index.js";
19
- // Module scoring for dual-key output (Issue #124)
20
- import { calculateModuleScore } from "../../lib/moduleScoring.js";
21
- /**
22
- * Main orchestrator class for running MCP server assessments
23
- *
24
- * @public
25
- * @example
26
- * ```typescript
27
- * import { AssessmentOrchestrator, AssessmentContext } from '@bryan-thompson/inspector-assessment';
28
- *
29
- * const orchestrator = new AssessmentOrchestrator();
30
- * const result = await orchestrator.runFullAssessment(context);
31
- * ```
32
- */
33
23
  export class AssessmentOrchestrator {
34
24
  config;
35
- logger;
36
25
  startTime = 0;
37
26
  totalTestsRun = 0;
38
27
  // Claude Code Bridge for intelligent analysis
39
28
  claudeBridge;
40
29
  claudeEnabled = false;
41
- // Registry for assessor management (Issue #91)
42
- // Delegates construction, test count aggregation, and Claude bridge wiring
43
- registry;
30
+ // Core assessors
31
+ functionalityAssessor;
32
+ securityAssessor;
33
+ documentationAssessor;
34
+ errorHandlingAssessor;
35
+ usabilityAssessor;
36
+ // Extended assessors
37
+ mcpSpecAssessor;
38
+ // New MCP Directory Compliance Gap assessors
39
+ aupComplianceAssessor;
40
+ toolAnnotationAssessor;
41
+ prohibitedLibrariesAssessor;
42
+ manifestValidationAssessor;
43
+ portabilityAssessor;
44
44
  constructor(config = {}) {
45
45
  this.config = { ...DEFAULT_ASSESSMENT_CONFIG, ...config };
46
- // Initialize logger
47
- this.logger = createLogger("AssessmentOrchestrator", this.config.logging ?? DEFAULT_LOGGING_CONFIG);
48
- // Emit deprecation warnings for deprecated config flags
49
- if (this.config.assessmentCategories?.mcpSpecCompliance !== undefined) {
50
- this.logger.warn("Config flag 'mcpSpecCompliance' is deprecated. Use 'protocolCompliance' instead. " +
51
- "This flag will be removed in v2.0.0.", { flag: "mcpSpecCompliance", replacement: "protocolCompliance" });
52
- }
53
- if (this.config.assessmentCategories?.protocolConformance !== undefined) {
54
- this.logger.warn("Config flag 'protocolConformance' is deprecated. Use 'protocolCompliance' instead. " +
55
- "This flag will be removed in v2.0.0.", { flag: "protocolConformance", replacement: "protocolCompliance" });
56
- }
57
46
  // Initialize Claude Code Bridge if enabled in config
58
47
  if (this.config.claudeCode?.enabled) {
59
48
  this.initializeClaudeBridge(this.config.claudeCode);
60
49
  }
61
- // Initialize registry and register all enabled assessors (Issue #91)
62
- // The registry handles:
63
- // - Conditional instantiation based on config flags
64
- // - Deprecated flag OR logic (e.g., protocolCompliance supports 3 flags)
65
- // - Custom setup (e.g., ToolAnnotationAssessor pattern config)
66
- // - Claude bridge wiring for supporting assessors
67
- this.registry = new AssessorRegistry(this.config);
68
- this.registry.registerAll(ASSESSOR_DEFINITIONS);
69
- // Wire up Claude bridge to registry (handles all supporting assessors)
50
+ // Initialize core assessors
51
+ this.functionalityAssessor = new FunctionalityAssessor(this.config);
52
+ this.securityAssessor = new SecurityAssessor(this.config);
53
+ this.documentationAssessor = new DocumentationAssessor(this.config);
54
+ this.errorHandlingAssessor = new ErrorHandlingAssessor(this.config);
55
+ this.usabilityAssessor = new UsabilityAssessor(this.config);
56
+ // Initialize extended assessors if enabled
57
+ if (this.config.enableExtendedAssessment) {
58
+ if (this.config.assessmentCategories?.mcpSpecCompliance) {
59
+ this.mcpSpecAssessor = new MCPSpecComplianceAssessor(this.config);
60
+ }
61
+ // Initialize new MCP Directory Compliance Gap assessors
62
+ if (this.config.assessmentCategories?.aupCompliance) {
63
+ this.aupComplianceAssessor = new AUPComplianceAssessor(this.config);
64
+ // Wire up Claude bridge for semantic analysis
65
+ if (this.claudeBridge) {
66
+ this.aupComplianceAssessor.setClaudeBridge(this.claudeBridge);
67
+ }
68
+ }
69
+ if (this.config.assessmentCategories?.toolAnnotations) {
70
+ this.toolAnnotationAssessor = new ToolAnnotationAssessor(this.config);
71
+ // Wire up Claude bridge for behavior inference
72
+ if (this.claudeBridge) {
73
+ this.toolAnnotationAssessor.setClaudeBridge(this.claudeBridge);
74
+ }
75
+ }
76
+ if (this.config.assessmentCategories?.prohibitedLibraries) {
77
+ this.prohibitedLibrariesAssessor = new ProhibitedLibrariesAssessor(this.config);
78
+ }
79
+ if (this.config.assessmentCategories?.manifestValidation) {
80
+ this.manifestValidationAssessor = new ManifestValidationAssessor(this.config);
81
+ }
82
+ if (this.config.assessmentCategories?.portability) {
83
+ this.portabilityAssessor = new PortabilityAssessor(this.config);
84
+ }
85
+ }
86
+ // Wire up Claude bridge to TestDataGenerator for intelligent test generation
70
87
  if (this.claudeBridge) {
71
- this.registry.setClaudeBridge(this.claudeBridge);
72
88
  TestDataGenerator.setClaudeBridge(this.claudeBridge);
73
89
  }
74
- // Set logger for TestDataGenerator diagnostic output
75
- TestDataGenerator.setLogger(this.logger);
76
90
  }
77
91
  /**
78
92
  * Initialize Claude Code Bridge for intelligent analysis
@@ -80,23 +94,18 @@ export class AssessmentOrchestrator {
80
94
  */
81
95
  initializeClaudeBridge(bridgeConfig) {
82
96
  try {
83
- this.claudeBridge = new ClaudeCodeBridge(bridgeConfig, this.logger);
97
+ this.claudeBridge = new ClaudeCodeBridge(bridgeConfig);
84
98
  this.claudeEnabled = true;
85
- this.logger.info("Claude Code Bridge initialized", {
86
- features: bridgeConfig.features,
87
- });
99
+ console.log("[AssessmentOrchestrator] Claude Code Bridge initialized with features:", bridgeConfig.features);
88
100
  }
89
101
  catch (error) {
90
- this.logger.warn("Failed to initialize Claude Code Bridge", {
91
- error: String(error),
92
- });
102
+ console.warn("[AssessmentOrchestrator] Failed to initialize Claude Code Bridge:", error);
93
103
  this.claudeEnabled = false;
94
104
  }
95
105
  }
96
106
  /**
97
107
  * Enable Claude Code integration programmatically
98
108
  * Call this method to enable Claude features after construction
99
- * @public
100
109
  */
101
110
  enableClaudeCode(config) {
102
111
  const bridgeConfig = {
@@ -105,94 +114,160 @@ export class AssessmentOrchestrator {
105
114
  enabled: true,
106
115
  };
107
116
  this.initializeClaudeBridge(bridgeConfig);
108
- // Wire up to all supporting assessors via registry
117
+ // Wire up to existing assessors
109
118
  if (this.claudeBridge) {
110
- this.registry.setClaudeBridge(this.claudeBridge);
119
+ if (this.aupComplianceAssessor) {
120
+ this.aupComplianceAssessor.setClaudeBridge(this.claudeBridge);
121
+ }
122
+ if (this.toolAnnotationAssessor) {
123
+ this.toolAnnotationAssessor.setClaudeBridge(this.claudeBridge);
124
+ }
111
125
  TestDataGenerator.setClaudeBridge(this.claudeBridge);
112
126
  }
113
127
  }
114
128
  /**
115
129
  * Check if Claude Code integration is enabled and available
116
- * @public
117
130
  */
118
131
  isClaudeEnabled() {
119
132
  return this.claudeEnabled && this.claudeBridge !== undefined;
120
133
  }
121
134
  /**
122
135
  * Get Claude Code Bridge for external access
123
- * @public
124
136
  */
125
137
  getClaudeBridge() {
126
138
  return this.claudeBridge;
127
139
  }
140
+ /**
141
+ * Reset test counts for all assessors
142
+ */
143
+ resetAllTestCounts() {
144
+ this.functionalityAssessor.resetTestCount();
145
+ this.securityAssessor.resetTestCount();
146
+ this.documentationAssessor.resetTestCount();
147
+ this.errorHandlingAssessor.resetTestCount();
148
+ this.usabilityAssessor.resetTestCount();
149
+ if (this.mcpSpecAssessor) {
150
+ this.mcpSpecAssessor.resetTestCount();
151
+ }
152
+ // Reset new assessors
153
+ if (this.aupComplianceAssessor) {
154
+ this.aupComplianceAssessor.resetTestCount();
155
+ }
156
+ if (this.toolAnnotationAssessor) {
157
+ this.toolAnnotationAssessor.resetTestCount();
158
+ }
159
+ if (this.prohibitedLibrariesAssessor) {
160
+ this.prohibitedLibrariesAssessor.resetTestCount();
161
+ }
162
+ if (this.manifestValidationAssessor) {
163
+ this.manifestValidationAssessor.resetTestCount();
164
+ }
165
+ if (this.portabilityAssessor) {
166
+ this.portabilityAssessor.resetTestCount();
167
+ }
168
+ }
128
169
  /**
129
170
  * Run a complete assessment on an MCP server
130
- * @public
131
171
  */
132
172
  async runFullAssessment(context) {
133
173
  this.startTime = Date.now();
134
174
  this.totalTestsRun = 0;
135
- this.registry.resetAllTestCounts();
136
- // Execute all assessors via registry (Issue #91)
137
- // Registry handles:
138
- // - Phase-ordered execution (Phase 0/PRE always runs first and sequentially)
139
- // - Parallel vs sequential based on config.parallelTesting
140
- // - JSONL events (module_started, module_progress)
141
- // - Test count tracking
142
- const assessmentResults = await this.registry.executeAll(context);
143
- // Integrate temporal findings into security.vulnerabilities for unified view
144
- if (assessmentResults.temporal?.rugPullsDetected &&
145
- assessmentResults.temporal.rugPullsDetected > 0 &&
146
- assessmentResults.security) {
147
- for (const detail of assessmentResults.temporal.details.filter((d) => d.vulnerable)) {
148
- assessmentResults.security.vulnerabilities.push(`RUG_PULL_TEMPORAL: ${detail.tool} - Tool behavior changed after invocation ${detail.firstDeviationAt}. Requires immediate manual review.`);
149
- }
150
- }
151
- // Issue #124: Dual-key output for v2.0.0 transition
152
- // Output BOTH old and new keys to maintain backward compatibility
153
- // Old keys (documentation, usability, mcpSpecCompliance) will be removed in v2.0.0
154
- // developerExperience (new) = documentation + usability (deprecated)
155
- if (assessmentResults.documentation && assessmentResults.usability) {
156
- const docScore = calculateModuleScore(assessmentResults.documentation) ?? 50;
157
- const usabilityScore = calculateModuleScore(assessmentResults.usability) ?? 50;
158
- const combinedStatus = determineOverallStatus({
159
- documentation: assessmentResults.documentation,
160
- usability: assessmentResults.usability,
161
- });
162
- assessmentResults.developerExperience = {
163
- documentation: assessmentResults.documentation,
164
- usability: assessmentResults.usability,
165
- status: combinedStatus,
166
- score: Math.round((docScore + usabilityScore) / 2),
167
- };
168
- // Emit deprecation warning for old keys
169
- this.logger.warn("Output keys 'documentation' and 'usability' are deprecated. " +
170
- "Use 'developerExperience' instead. These keys will be removed in v2.0.0.", {
171
- deprecated: ["documentation", "usability"],
172
- replacement: "developerExperience",
173
- });
174
- }
175
- // protocolCompliance (new) = mcpSpecCompliance (deprecated)
176
- if (assessmentResults.mcpSpecCompliance) {
177
- assessmentResults.protocolCompliance =
178
- assessmentResults.mcpSpecCompliance;
179
- // Emit deprecation warning for old key
180
- this.logger.warn("Output key 'mcpSpecCompliance' is deprecated. " +
181
- "Use 'protocolCompliance' instead. This key will be removed in v2.0.0.", {
182
- deprecated: ["mcpSpecCompliance"],
183
- replacement: "protocolCompliance",
184
- });
175
+ this.resetAllTestCounts();
176
+ // Run assessments in parallel if enabled
177
+ const assessmentPromises = [];
178
+ const assessmentResults = {};
179
+ if (this.config.parallelTesting) {
180
+ // Core assessments
181
+ assessmentPromises.push(this.functionalityAssessor
182
+ .assess(context)
183
+ .then((r) => (assessmentResults.functionality = r)), this.securityAssessor
184
+ .assess(context)
185
+ .then((r) => (assessmentResults.security = r)), this.documentationAssessor
186
+ .assess(context)
187
+ .then((r) => (assessmentResults.documentation = r)), this.errorHandlingAssessor
188
+ .assess(context)
189
+ .then((r) => (assessmentResults.errorHandling = r)), this.usabilityAssessor
190
+ .assess(context)
191
+ .then((r) => (assessmentResults.usability = r)));
192
+ // Extended assessments
193
+ if (this.mcpSpecAssessor) {
194
+ assessmentPromises.push(this.mcpSpecAssessor
195
+ .assess(context)
196
+ .then((r) => (assessmentResults.mcpSpecCompliance = r)));
197
+ }
198
+ // New MCP Directory Compliance Gap assessments
199
+ if (this.aupComplianceAssessor) {
200
+ assessmentPromises.push(this.aupComplianceAssessor
201
+ .assess(context)
202
+ .then((r) => (assessmentResults.aupCompliance = r)));
203
+ }
204
+ if (this.toolAnnotationAssessor) {
205
+ assessmentPromises.push(this.toolAnnotationAssessor
206
+ .assess(context)
207
+ .then((r) => (assessmentResults.toolAnnotations = r)));
208
+ }
209
+ if (this.prohibitedLibrariesAssessor) {
210
+ assessmentPromises.push(this.prohibitedLibrariesAssessor
211
+ .assess(context)
212
+ .then((r) => (assessmentResults.prohibitedLibraries = r)));
213
+ }
214
+ if (this.manifestValidationAssessor) {
215
+ assessmentPromises.push(this.manifestValidationAssessor
216
+ .assess(context)
217
+ .then((r) => (assessmentResults.manifestValidation = r)));
218
+ }
219
+ if (this.portabilityAssessor) {
220
+ assessmentPromises.push(this.portabilityAssessor
221
+ .assess(context)
222
+ .then((r) => (assessmentResults.portability = r)));
223
+ }
224
+ await Promise.all(assessmentPromises);
225
+ }
226
+ else {
227
+ // Sequential execution
228
+ assessmentResults.functionality =
229
+ await this.functionalityAssessor.assess(context);
230
+ assessmentResults.security = await this.securityAssessor.assess(context);
231
+ assessmentResults.documentation =
232
+ await this.documentationAssessor.assess(context);
233
+ assessmentResults.errorHandling =
234
+ await this.errorHandlingAssessor.assess(context);
235
+ assessmentResults.usability =
236
+ await this.usabilityAssessor.assess(context);
237
+ if (this.mcpSpecAssessor) {
238
+ assessmentResults.mcpSpecCompliance =
239
+ await this.mcpSpecAssessor.assess(context);
240
+ }
241
+ // New MCP Directory Compliance Gap assessments (sequential)
242
+ if (this.aupComplianceAssessor) {
243
+ assessmentResults.aupCompliance =
244
+ await this.aupComplianceAssessor.assess(context);
245
+ }
246
+ if (this.toolAnnotationAssessor) {
247
+ assessmentResults.toolAnnotations =
248
+ await this.toolAnnotationAssessor.assess(context);
249
+ }
250
+ if (this.prohibitedLibrariesAssessor) {
251
+ assessmentResults.prohibitedLibraries =
252
+ await this.prohibitedLibrariesAssessor.assess(context);
253
+ }
254
+ if (this.manifestValidationAssessor) {
255
+ assessmentResults.manifestValidation =
256
+ await this.manifestValidationAssessor.assess(context);
257
+ }
258
+ if (this.portabilityAssessor) {
259
+ assessmentResults.portability =
260
+ await this.portabilityAssessor.assess(context);
261
+ }
185
262
  }
186
263
  // Collect test counts from all assessors
187
264
  this.totalTestsRun = this.collectTotalTestCount();
188
265
  // Determine overall status
189
- const overallStatus = determineOverallStatus(assessmentResults);
266
+ const overallStatus = this.determineOverallStatus(assessmentResults);
190
267
  // Generate summary and recommendations
191
- const summary = generateSummary(assessmentResults);
192
- const recommendations = generateRecommendations(assessmentResults);
268
+ const summary = this.generateSummary(assessmentResults);
269
+ const recommendations = this.generateRecommendations(assessmentResults);
193
270
  const executionTime = Date.now() - this.startTime;
194
- // Type assertion needed because Partial<MCPDirectoryAssessment> has optional required fields
195
- // When modules are skipped via --skip-modules, not all fields will be present
196
271
  return {
197
272
  serverName: context.serverName,
198
273
  assessmentDate: new Date().toISOString(),
@@ -204,19 +279,10 @@ export class AssessmentOrchestrator {
204
279
  executionTime,
205
280
  totalTestsRun: this.totalTestsRun,
206
281
  mcpProtocolVersion: this.config.mcpProtocolVersion,
207
- assessmentMetadata: {
208
- // Source code is available if we have a path OR loaded source files
209
- sourceCodeAvailable: !!context.sourceCodePath || (context.sourceCodeFiles?.size ?? 0) > 0,
210
- // Use explicit transport type, or infer from available context
211
- transportType: context.transportConfig?.type ??
212
- (context.transportConfig?.url ? "streamable-http" : undefined),
213
- },
214
282
  };
215
283
  }
216
284
  /**
217
285
  * Legacy assess method for backward compatibility
218
- * @public
219
- * @deprecated Use runFullAssessment() with AssessmentContext instead
220
286
  */
221
287
  async assess(serverName, tools, callTool, serverInfo, readmeContent, packageJson) {
222
288
  const context = {
@@ -231,21 +297,120 @@ export class AssessmentOrchestrator {
231
297
  return this.runFullAssessment(context);
232
298
  }
233
299
  collectTotalTestCount() {
234
- // Delegate to registry for centralized test count aggregation (Issue #91)
235
- const total = this.registry.getTotalTestCount();
236
- this.logger.debug("Total test count", { total });
300
+ let total = 0;
301
+ // Get actual test counts from assessors
302
+ const functionalityCount = this.functionalityAssessor.getTestCount();
303
+ const securityCount = this.securityAssessor.getTestCount();
304
+ const documentationCount = this.documentationAssessor.getTestCount();
305
+ const errorHandlingCount = this.errorHandlingAssessor.getTestCount();
306
+ const usabilityCount = this.usabilityAssessor.getTestCount();
307
+ const mcpSpecCount = this.mcpSpecAssessor?.getTestCount() || 0;
308
+ // New assessor counts
309
+ const aupCount = this.aupComplianceAssessor?.getTestCount() || 0;
310
+ const annotationCount = this.toolAnnotationAssessor?.getTestCount() || 0;
311
+ const librariesCount = this.prohibitedLibrariesAssessor?.getTestCount() || 0;
312
+ const manifestCount = this.manifestValidationAssessor?.getTestCount() || 0;
313
+ const portabilityCount = this.portabilityAssessor?.getTestCount() || 0;
314
+ console.log("[AssessmentOrchestrator] Test counts by assessor:", {
315
+ functionality: functionalityCount,
316
+ security: securityCount,
317
+ documentation: documentationCount,
318
+ errorHandling: errorHandlingCount,
319
+ usability: usabilityCount,
320
+ mcpSpec: mcpSpecCount,
321
+ aupCompliance: aupCount,
322
+ toolAnnotations: annotationCount,
323
+ prohibitedLibraries: librariesCount,
324
+ manifestValidation: manifestCount,
325
+ portability: portabilityCount,
326
+ });
327
+ total =
328
+ functionalityCount +
329
+ securityCount +
330
+ documentationCount +
331
+ errorHandlingCount +
332
+ usabilityCount +
333
+ mcpSpecCount +
334
+ aupCount +
335
+ annotationCount +
336
+ librariesCount +
337
+ manifestCount +
338
+ portabilityCount;
339
+ console.log("[AssessmentOrchestrator] Total test count:", total);
237
340
  return total;
238
341
  }
342
+ determineOverallStatus(results) {
343
+ const statuses = [];
344
+ // Collect all statuses
345
+ Object.values(results).forEach((assessment) => {
346
+ if (assessment?.status) {
347
+ statuses.push(assessment.status);
348
+ }
349
+ });
350
+ // If any critical category fails, overall fails
351
+ if (statuses.includes("FAIL"))
352
+ return "FAIL";
353
+ // If any category needs more info, overall needs more info
354
+ if (statuses.includes("NEED_MORE_INFO"))
355
+ return "NEED_MORE_INFO";
356
+ // All must pass for overall pass
357
+ return "PASS";
358
+ }
359
+ generateSummary(results) {
360
+ const parts = [];
361
+ const totalCategories = Object.keys(results).length;
362
+ const passedCategories = Object.values(results).filter((r) => r?.status === "PASS").length;
363
+ parts.push(`Assessment complete: ${passedCategories}/${totalCategories} categories passed.`);
364
+ // Add key findings
365
+ if (results.security?.vulnerabilities?.length > 0) {
366
+ parts.push(`Found ${results.security.vulnerabilities.length} security vulnerabilities.`);
367
+ }
368
+ if (results.functionality?.brokenTools?.length > 0) {
369
+ parts.push(`${results.functionality.brokenTools.length} tools are not functioning correctly.`);
370
+ }
371
+ // New assessor findings
372
+ if (results.aupCompliance?.violations?.length > 0) {
373
+ const criticalCount = results.aupCompliance.violations.filter((v) => v.severity === "CRITICAL").length;
374
+ if (criticalCount > 0) {
375
+ parts.push(`CRITICAL: ${criticalCount} AUP violation(s) detected.`);
376
+ }
377
+ else {
378
+ parts.push(`${results.aupCompliance.violations.length} AUP item(s) flagged for review.`);
379
+ }
380
+ }
381
+ if (results.toolAnnotations?.missingAnnotationsCount > 0) {
382
+ parts.push(`${results.toolAnnotations.missingAnnotationsCount} tools missing annotations.`);
383
+ }
384
+ if (results.prohibitedLibraries?.matches?.length > 0) {
385
+ const blockingCount = results.prohibitedLibraries.matches.filter((m) => m.severity === "BLOCKING").length;
386
+ if (blockingCount > 0) {
387
+ parts.push(`BLOCKING: ${blockingCount} prohibited library/libraries detected.`);
388
+ }
389
+ }
390
+ if (results.portability?.usesBundleRoot) {
391
+ parts.push("Uses ${BUNDLE_ROOT} anti-pattern.");
392
+ }
393
+ return parts.join(" ");
394
+ }
395
+ generateRecommendations(results) {
396
+ const recommendations = [];
397
+ // Aggregate recommendations from all assessments
398
+ Object.values(results).forEach((assessment) => {
399
+ if (assessment?.recommendations) {
400
+ recommendations.push(...assessment.recommendations);
401
+ }
402
+ });
403
+ // De-duplicate and prioritize
404
+ return [...new Set(recommendations)].slice(0, 10);
405
+ }
239
406
  /**
240
407
  * Get assessment configuration
241
- * @public
242
408
  */
243
409
  getConfig() {
244
410
  return this.config;
245
411
  }
246
412
  /**
247
413
  * Update assessment configuration
248
- * @public
249
414
  */
250
415
  updateConfig(config) {
251
416
  this.config = { ...this.config, ...config };
@@ -0,0 +1,25 @@
1
+ /**
2
+ * Error Handling Assessor Module
3
+ * Tests error handling and input validation
4
+ */
5
+ import { ErrorHandlingAssessment } from "../../../lib/assessmentTypes.js";
6
+ import { BaseAssessor } from "./BaseAssessor.js";
7
+ import { AssessmentContext } from "../AssessmentOrchestrator.js";
8
+ export declare class ErrorHandlingAssessor extends BaseAssessor {
9
+ assess(context: AssessmentContext): Promise<ErrorHandlingAssessment>;
10
+ private selectToolsForTesting;
11
+ private testToolErrorHandling;
12
+ private testMissingParameters;
13
+ private testWrongTypes;
14
+ private testInvalidValues;
15
+ private testExcessiveInput;
16
+ private getToolSchema;
17
+ private generateWrongTypeParams;
18
+ private generateInvalidValueParams;
19
+ private generateParamsWithValue;
20
+ private calculateMetrics;
21
+ private determineErrorHandlingStatus;
22
+ private generateExplanation;
23
+ private generateRecommendations;
24
+ }
25
+ //# sourceMappingURL=ErrorHandlingAssessor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ErrorHandlingAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ErrorHandlingAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAIxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D,qBAAa,qBAAsB,SAAQ,YAAY;IAC/C,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAyC1E,OAAO,CAAC,qBAAqB;YA8Cf,qBAAqB;YAuBrB,qBAAqB;YAmGrB,cAAc;YAmFd,iBAAiB;YA8DjB,kBAAkB;IA6DhC,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,uBAAuB;IAgC/B,OAAO,CAAC,0BAA0B;IAgClC,OAAO,CAAC,uBAAuB;IA4B/B,OAAO,CAAC,gBAAgB;IAoGxB,OAAO,CAAC,4BAA4B;IAapC,OAAO,CAAC,mBAAmB;IAuE3B,OAAO,CAAC,uBAAuB;CA4ChC"}
@@ -10,7 +10,7 @@
10
10
  * @module assessment/modules/ErrorHandlingAssessor
11
11
  */
12
12
  import { BaseAssessor } from "./BaseAssessor.js";
13
- import { ProtocolComplianceAssessor } from "./ProtocolComplianceAssessor.js";
13
+ import { ProtocolComplianceAssessor } from "./ProtocolComplianceAssessor/index.js";
14
14
  /**
15
15
  * @deprecated Use ProtocolComplianceAssessor instead.
16
16
  *