@bryan-thompson/inspector-assessment 1.43.2 → 1.43.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1062 -224
- package/cli/build/assess-full.js +532 -106
- package/cli/build/assess-security.js +54 -90
- package/cli/build/lib/cli-parser.js +14 -1
- package/cli/build/lib/cli-parserSchemas.js +1 -0
- package/cli/build/lib/result-output.js +21 -0
- package/cli/build/profiles.js +20 -0
- package/cli/build/validate-testbed.js +0 -0
- package/cli/package.json +1 -1
- package/client/dist/assets/{OAuthCallback-BS8-A1sU.js → OAuthCallback-Chi58kRc.js} +1 -1
- package/client/dist/assets/{OAuthDebugCallback-025_TM2i.js → OAuthDebugCallback-BluD_Wxg.js} +1 -1
- package/client/dist/assets/{index-DEhlIjy-.js → index-KW2LwGdp.js} +4 -4
- package/client/dist/index.html +1 -1
- package/client/lib/lib/assessment/configSchemas.d.ts +64 -64
- package/client/lib/lib/assessment/jsonlEventSchemas.d.ts +286 -286
- package/client/lib/lib/assessment/resultTypes.d.ts +10 -0
- package/client/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/client/lib/lib/assessmentTypes.d.ts +1 -20
- package/client/lib/lib/assessmentTypes.d.ts.map +1 -1
- package/client/lib/lib/assessmentTypes.js +1 -20
- package/client/lib/services/assessment/AssessmentOrchestrator.d.ts +57 -104
- package/client/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/client/lib/services/assessment/AssessmentOrchestrator.js +298 -133
- package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts +25 -0
- package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts.map +1 -0
- package/client/lib/services/assessment/modules/ErrorHandlingAssessor.deprecated.js +1 -1
- package/client/lib/services/assessment/modules/ErrorHandlingAssessor.js +564 -0
- package/client/lib/services/assessment/modules/SecurityAssessor.d.ts +5 -0
- package/client/lib/services/assessment/modules/SecurityAssessor.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/SecurityAssessor.js +62 -0
- package/client/lib/services/assessment/modules/index.d.ts +1 -1
- package/client/lib/services/assessment/modules/index.js +1 -1
- package/client/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts +15 -0
- package/client/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/securityTests/SecurityPayloadTester.js +72 -0
- package/client/lib/services/assessment/modules/securityTests/factory.d.ts +2 -0
- package/client/lib/services/assessment/modules/securityTests/factory.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/securityTests/factory.js +1 -0
- package/client/lib/services/assessment/registry/AssessorDefinitions.js +1 -1
- package/client/lib/services/assessment/responseValidatorSchemas.d.ts +12 -12
- package/client/package.json +3 -3
- package/package.json +4 -2
- package/server/package.json +1 -1
- package/cli/build/lib/__tests__/zodErrorFormatter.test.js +0 -282
- package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts +0 -109
- package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts.map +0 -1
- package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.deprecated.d.ts +0 -109
- package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.deprecated.d.ts.map +0 -1
- package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.deprecated.js +0 -852
- package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.js +0 -852
|
@@ -1,78 +1,92 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Assessment Orchestrator
|
|
3
3
|
* Coordinates all assessment modules and manages the assessment workflow
|
|
4
|
-
*
|
|
5
|
-
* @public
|
|
6
|
-
* @module AssessmentOrchestrator
|
|
7
4
|
*/
|
|
8
5
|
import { DEFAULT_ASSESSMENT_CONFIG, } from "../../lib/assessmentTypes.js";
|
|
9
|
-
//
|
|
6
|
+
// Core assessment modules
|
|
7
|
+
import { FunctionalityAssessor } from "./modules/FunctionalityAssessor.js";
|
|
8
|
+
import { SecurityAssessor } from "./modules/SecurityAssessor.js";
|
|
9
|
+
import { DocumentationAssessor } from "./modules/DocumentationAssessor.js";
|
|
10
|
+
import { ErrorHandlingAssessor } from "./modules/ErrorHandlingAssessor.js";
|
|
11
|
+
import { UsabilityAssessor } from "./modules/UsabilityAssessor.js";
|
|
12
|
+
// Extended assessment modules
|
|
13
|
+
import { MCPSpecComplianceAssessor } from "./modules/MCPSpecComplianceAssessor.js";
|
|
14
|
+
// New MCP Directory Compliance Gap assessors
|
|
15
|
+
import { AUPComplianceAssessor } from "./modules/AUPComplianceAssessor.js";
|
|
16
|
+
import { ToolAnnotationAssessor } from "./modules/ToolAnnotationAssessor.js";
|
|
17
|
+
import { ProhibitedLibrariesAssessor } from "./modules/ProhibitedLibrariesAssessor.js";
|
|
18
|
+
import { ManifestValidationAssessor } from "./modules/ManifestValidationAssessor.js";
|
|
19
|
+
import { PortabilityAssessor } from "./modules/PortabilityAssessor.js";
|
|
10
20
|
// Claude Code integration for intelligent analysis
|
|
11
21
|
import { ClaudeCodeBridge, FULL_CLAUDE_CODE_CONFIG, } from "./lib/claudeCodeBridge.js";
|
|
12
22
|
import { TestDataGenerator } from "./TestDataGenerator.js";
|
|
13
|
-
// Structured logging
|
|
14
|
-
import { createLogger, DEFAULT_LOGGING_CONFIG } from "./lib/logger.js";
|
|
15
|
-
// Extracted helpers for testability
|
|
16
|
-
import { determineOverallStatus, generateSummary, generateRecommendations, } from "./orchestratorHelpers.js";
|
|
17
|
-
// Registry pattern for assessor management (Issue #91)
|
|
18
|
-
import { AssessorRegistry, ASSESSOR_DEFINITIONS } from "./registry/index.js";
|
|
19
|
-
// Module scoring for dual-key output (Issue #124)
|
|
20
|
-
import { calculateModuleScore } from "../../lib/moduleScoring.js";
|
|
21
|
-
/**
|
|
22
|
-
* Main orchestrator class for running MCP server assessments
|
|
23
|
-
*
|
|
24
|
-
* @public
|
|
25
|
-
* @example
|
|
26
|
-
* ```typescript
|
|
27
|
-
* import { AssessmentOrchestrator, AssessmentContext } from '@bryan-thompson/inspector-assessment';
|
|
28
|
-
*
|
|
29
|
-
* const orchestrator = new AssessmentOrchestrator();
|
|
30
|
-
* const result = await orchestrator.runFullAssessment(context);
|
|
31
|
-
* ```
|
|
32
|
-
*/
|
|
33
23
|
export class AssessmentOrchestrator {
|
|
34
24
|
config;
|
|
35
|
-
logger;
|
|
36
25
|
startTime = 0;
|
|
37
26
|
totalTestsRun = 0;
|
|
38
27
|
// Claude Code Bridge for intelligent analysis
|
|
39
28
|
claudeBridge;
|
|
40
29
|
claudeEnabled = false;
|
|
41
|
-
//
|
|
42
|
-
|
|
43
|
-
|
|
30
|
+
// Core assessors
|
|
31
|
+
functionalityAssessor;
|
|
32
|
+
securityAssessor;
|
|
33
|
+
documentationAssessor;
|
|
34
|
+
errorHandlingAssessor;
|
|
35
|
+
usabilityAssessor;
|
|
36
|
+
// Extended assessors
|
|
37
|
+
mcpSpecAssessor;
|
|
38
|
+
// New MCP Directory Compliance Gap assessors
|
|
39
|
+
aupComplianceAssessor;
|
|
40
|
+
toolAnnotationAssessor;
|
|
41
|
+
prohibitedLibrariesAssessor;
|
|
42
|
+
manifestValidationAssessor;
|
|
43
|
+
portabilityAssessor;
|
|
44
44
|
constructor(config = {}) {
|
|
45
45
|
this.config = { ...DEFAULT_ASSESSMENT_CONFIG, ...config };
|
|
46
|
-
// Initialize logger
|
|
47
|
-
this.logger = createLogger("AssessmentOrchestrator", this.config.logging ?? DEFAULT_LOGGING_CONFIG);
|
|
48
|
-
// Emit deprecation warnings for deprecated config flags
|
|
49
|
-
if (this.config.assessmentCategories?.mcpSpecCompliance !== undefined) {
|
|
50
|
-
this.logger.warn("Config flag 'mcpSpecCompliance' is deprecated. Use 'protocolCompliance' instead. " +
|
|
51
|
-
"This flag will be removed in v2.0.0.", { flag: "mcpSpecCompliance", replacement: "protocolCompliance" });
|
|
52
|
-
}
|
|
53
|
-
if (this.config.assessmentCategories?.protocolConformance !== undefined) {
|
|
54
|
-
this.logger.warn("Config flag 'protocolConformance' is deprecated. Use 'protocolCompliance' instead. " +
|
|
55
|
-
"This flag will be removed in v2.0.0.", { flag: "protocolConformance", replacement: "protocolCompliance" });
|
|
56
|
-
}
|
|
57
46
|
// Initialize Claude Code Bridge if enabled in config
|
|
58
47
|
if (this.config.claudeCode?.enabled) {
|
|
59
48
|
this.initializeClaudeBridge(this.config.claudeCode);
|
|
60
49
|
}
|
|
61
|
-
// Initialize
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
this.
|
|
69
|
-
|
|
50
|
+
// Initialize core assessors
|
|
51
|
+
this.functionalityAssessor = new FunctionalityAssessor(this.config);
|
|
52
|
+
this.securityAssessor = new SecurityAssessor(this.config);
|
|
53
|
+
this.documentationAssessor = new DocumentationAssessor(this.config);
|
|
54
|
+
this.errorHandlingAssessor = new ErrorHandlingAssessor(this.config);
|
|
55
|
+
this.usabilityAssessor = new UsabilityAssessor(this.config);
|
|
56
|
+
// Initialize extended assessors if enabled
|
|
57
|
+
if (this.config.enableExtendedAssessment) {
|
|
58
|
+
if (this.config.assessmentCategories?.mcpSpecCompliance) {
|
|
59
|
+
this.mcpSpecAssessor = new MCPSpecComplianceAssessor(this.config);
|
|
60
|
+
}
|
|
61
|
+
// Initialize new MCP Directory Compliance Gap assessors
|
|
62
|
+
if (this.config.assessmentCategories?.aupCompliance) {
|
|
63
|
+
this.aupComplianceAssessor = new AUPComplianceAssessor(this.config);
|
|
64
|
+
// Wire up Claude bridge for semantic analysis
|
|
65
|
+
if (this.claudeBridge) {
|
|
66
|
+
this.aupComplianceAssessor.setClaudeBridge(this.claudeBridge);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (this.config.assessmentCategories?.toolAnnotations) {
|
|
70
|
+
this.toolAnnotationAssessor = new ToolAnnotationAssessor(this.config);
|
|
71
|
+
// Wire up Claude bridge for behavior inference
|
|
72
|
+
if (this.claudeBridge) {
|
|
73
|
+
this.toolAnnotationAssessor.setClaudeBridge(this.claudeBridge);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
if (this.config.assessmentCategories?.prohibitedLibraries) {
|
|
77
|
+
this.prohibitedLibrariesAssessor = new ProhibitedLibrariesAssessor(this.config);
|
|
78
|
+
}
|
|
79
|
+
if (this.config.assessmentCategories?.manifestValidation) {
|
|
80
|
+
this.manifestValidationAssessor = new ManifestValidationAssessor(this.config);
|
|
81
|
+
}
|
|
82
|
+
if (this.config.assessmentCategories?.portability) {
|
|
83
|
+
this.portabilityAssessor = new PortabilityAssessor(this.config);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
// Wire up Claude bridge to TestDataGenerator for intelligent test generation
|
|
70
87
|
if (this.claudeBridge) {
|
|
71
|
-
this.registry.setClaudeBridge(this.claudeBridge);
|
|
72
88
|
TestDataGenerator.setClaudeBridge(this.claudeBridge);
|
|
73
89
|
}
|
|
74
|
-
// Set logger for TestDataGenerator diagnostic output
|
|
75
|
-
TestDataGenerator.setLogger(this.logger);
|
|
76
90
|
}
|
|
77
91
|
/**
|
|
78
92
|
* Initialize Claude Code Bridge for intelligent analysis
|
|
@@ -80,23 +94,18 @@ export class AssessmentOrchestrator {
|
|
|
80
94
|
*/
|
|
81
95
|
initializeClaudeBridge(bridgeConfig) {
|
|
82
96
|
try {
|
|
83
|
-
this.claudeBridge = new ClaudeCodeBridge(bridgeConfig
|
|
97
|
+
this.claudeBridge = new ClaudeCodeBridge(bridgeConfig);
|
|
84
98
|
this.claudeEnabled = true;
|
|
85
|
-
|
|
86
|
-
features: bridgeConfig.features,
|
|
87
|
-
});
|
|
99
|
+
console.log("[AssessmentOrchestrator] Claude Code Bridge initialized with features:", bridgeConfig.features);
|
|
88
100
|
}
|
|
89
101
|
catch (error) {
|
|
90
|
-
|
|
91
|
-
error: String(error),
|
|
92
|
-
});
|
|
102
|
+
console.warn("[AssessmentOrchestrator] Failed to initialize Claude Code Bridge:", error);
|
|
93
103
|
this.claudeEnabled = false;
|
|
94
104
|
}
|
|
95
105
|
}
|
|
96
106
|
/**
|
|
97
107
|
* Enable Claude Code integration programmatically
|
|
98
108
|
* Call this method to enable Claude features after construction
|
|
99
|
-
* @public
|
|
100
109
|
*/
|
|
101
110
|
enableClaudeCode(config) {
|
|
102
111
|
const bridgeConfig = {
|
|
@@ -105,94 +114,160 @@ export class AssessmentOrchestrator {
|
|
|
105
114
|
enabled: true,
|
|
106
115
|
};
|
|
107
116
|
this.initializeClaudeBridge(bridgeConfig);
|
|
108
|
-
// Wire up to
|
|
117
|
+
// Wire up to existing assessors
|
|
109
118
|
if (this.claudeBridge) {
|
|
110
|
-
|
|
119
|
+
if (this.aupComplianceAssessor) {
|
|
120
|
+
this.aupComplianceAssessor.setClaudeBridge(this.claudeBridge);
|
|
121
|
+
}
|
|
122
|
+
if (this.toolAnnotationAssessor) {
|
|
123
|
+
this.toolAnnotationAssessor.setClaudeBridge(this.claudeBridge);
|
|
124
|
+
}
|
|
111
125
|
TestDataGenerator.setClaudeBridge(this.claudeBridge);
|
|
112
126
|
}
|
|
113
127
|
}
|
|
114
128
|
/**
|
|
115
129
|
* Check if Claude Code integration is enabled and available
|
|
116
|
-
* @public
|
|
117
130
|
*/
|
|
118
131
|
isClaudeEnabled() {
|
|
119
132
|
return this.claudeEnabled && this.claudeBridge !== undefined;
|
|
120
133
|
}
|
|
121
134
|
/**
|
|
122
135
|
* Get Claude Code Bridge for external access
|
|
123
|
-
* @public
|
|
124
136
|
*/
|
|
125
137
|
getClaudeBridge() {
|
|
126
138
|
return this.claudeBridge;
|
|
127
139
|
}
|
|
140
|
+
/**
|
|
141
|
+
* Reset test counts for all assessors
|
|
142
|
+
*/
|
|
143
|
+
resetAllTestCounts() {
|
|
144
|
+
this.functionalityAssessor.resetTestCount();
|
|
145
|
+
this.securityAssessor.resetTestCount();
|
|
146
|
+
this.documentationAssessor.resetTestCount();
|
|
147
|
+
this.errorHandlingAssessor.resetTestCount();
|
|
148
|
+
this.usabilityAssessor.resetTestCount();
|
|
149
|
+
if (this.mcpSpecAssessor) {
|
|
150
|
+
this.mcpSpecAssessor.resetTestCount();
|
|
151
|
+
}
|
|
152
|
+
// Reset new assessors
|
|
153
|
+
if (this.aupComplianceAssessor) {
|
|
154
|
+
this.aupComplianceAssessor.resetTestCount();
|
|
155
|
+
}
|
|
156
|
+
if (this.toolAnnotationAssessor) {
|
|
157
|
+
this.toolAnnotationAssessor.resetTestCount();
|
|
158
|
+
}
|
|
159
|
+
if (this.prohibitedLibrariesAssessor) {
|
|
160
|
+
this.prohibitedLibrariesAssessor.resetTestCount();
|
|
161
|
+
}
|
|
162
|
+
if (this.manifestValidationAssessor) {
|
|
163
|
+
this.manifestValidationAssessor.resetTestCount();
|
|
164
|
+
}
|
|
165
|
+
if (this.portabilityAssessor) {
|
|
166
|
+
this.portabilityAssessor.resetTestCount();
|
|
167
|
+
}
|
|
168
|
+
}
|
|
128
169
|
/**
|
|
129
170
|
* Run a complete assessment on an MCP server
|
|
130
|
-
* @public
|
|
131
171
|
*/
|
|
132
172
|
async runFullAssessment(context) {
|
|
133
173
|
this.startTime = Date.now();
|
|
134
174
|
this.totalTestsRun = 0;
|
|
135
|
-
this.
|
|
136
|
-
//
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
this.
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
175
|
+
this.resetAllTestCounts();
|
|
176
|
+
// Run assessments in parallel if enabled
|
|
177
|
+
const assessmentPromises = [];
|
|
178
|
+
const assessmentResults = {};
|
|
179
|
+
if (this.config.parallelTesting) {
|
|
180
|
+
// Core assessments
|
|
181
|
+
assessmentPromises.push(this.functionalityAssessor
|
|
182
|
+
.assess(context)
|
|
183
|
+
.then((r) => (assessmentResults.functionality = r)), this.securityAssessor
|
|
184
|
+
.assess(context)
|
|
185
|
+
.then((r) => (assessmentResults.security = r)), this.documentationAssessor
|
|
186
|
+
.assess(context)
|
|
187
|
+
.then((r) => (assessmentResults.documentation = r)), this.errorHandlingAssessor
|
|
188
|
+
.assess(context)
|
|
189
|
+
.then((r) => (assessmentResults.errorHandling = r)), this.usabilityAssessor
|
|
190
|
+
.assess(context)
|
|
191
|
+
.then((r) => (assessmentResults.usability = r)));
|
|
192
|
+
// Extended assessments
|
|
193
|
+
if (this.mcpSpecAssessor) {
|
|
194
|
+
assessmentPromises.push(this.mcpSpecAssessor
|
|
195
|
+
.assess(context)
|
|
196
|
+
.then((r) => (assessmentResults.mcpSpecCompliance = r)));
|
|
197
|
+
}
|
|
198
|
+
// New MCP Directory Compliance Gap assessments
|
|
199
|
+
if (this.aupComplianceAssessor) {
|
|
200
|
+
assessmentPromises.push(this.aupComplianceAssessor
|
|
201
|
+
.assess(context)
|
|
202
|
+
.then((r) => (assessmentResults.aupCompliance = r)));
|
|
203
|
+
}
|
|
204
|
+
if (this.toolAnnotationAssessor) {
|
|
205
|
+
assessmentPromises.push(this.toolAnnotationAssessor
|
|
206
|
+
.assess(context)
|
|
207
|
+
.then((r) => (assessmentResults.toolAnnotations = r)));
|
|
208
|
+
}
|
|
209
|
+
if (this.prohibitedLibrariesAssessor) {
|
|
210
|
+
assessmentPromises.push(this.prohibitedLibrariesAssessor
|
|
211
|
+
.assess(context)
|
|
212
|
+
.then((r) => (assessmentResults.prohibitedLibraries = r)));
|
|
213
|
+
}
|
|
214
|
+
if (this.manifestValidationAssessor) {
|
|
215
|
+
assessmentPromises.push(this.manifestValidationAssessor
|
|
216
|
+
.assess(context)
|
|
217
|
+
.then((r) => (assessmentResults.manifestValidation = r)));
|
|
218
|
+
}
|
|
219
|
+
if (this.portabilityAssessor) {
|
|
220
|
+
assessmentPromises.push(this.portabilityAssessor
|
|
221
|
+
.assess(context)
|
|
222
|
+
.then((r) => (assessmentResults.portability = r)));
|
|
223
|
+
}
|
|
224
|
+
await Promise.all(assessmentPromises);
|
|
225
|
+
}
|
|
226
|
+
else {
|
|
227
|
+
// Sequential execution
|
|
228
|
+
assessmentResults.functionality =
|
|
229
|
+
await this.functionalityAssessor.assess(context);
|
|
230
|
+
assessmentResults.security = await this.securityAssessor.assess(context);
|
|
231
|
+
assessmentResults.documentation =
|
|
232
|
+
await this.documentationAssessor.assess(context);
|
|
233
|
+
assessmentResults.errorHandling =
|
|
234
|
+
await this.errorHandlingAssessor.assess(context);
|
|
235
|
+
assessmentResults.usability =
|
|
236
|
+
await this.usabilityAssessor.assess(context);
|
|
237
|
+
if (this.mcpSpecAssessor) {
|
|
238
|
+
assessmentResults.mcpSpecCompliance =
|
|
239
|
+
await this.mcpSpecAssessor.assess(context);
|
|
240
|
+
}
|
|
241
|
+
// New MCP Directory Compliance Gap assessments (sequential)
|
|
242
|
+
if (this.aupComplianceAssessor) {
|
|
243
|
+
assessmentResults.aupCompliance =
|
|
244
|
+
await this.aupComplianceAssessor.assess(context);
|
|
245
|
+
}
|
|
246
|
+
if (this.toolAnnotationAssessor) {
|
|
247
|
+
assessmentResults.toolAnnotations =
|
|
248
|
+
await this.toolAnnotationAssessor.assess(context);
|
|
249
|
+
}
|
|
250
|
+
if (this.prohibitedLibrariesAssessor) {
|
|
251
|
+
assessmentResults.prohibitedLibraries =
|
|
252
|
+
await this.prohibitedLibrariesAssessor.assess(context);
|
|
253
|
+
}
|
|
254
|
+
if (this.manifestValidationAssessor) {
|
|
255
|
+
assessmentResults.manifestValidation =
|
|
256
|
+
await this.manifestValidationAssessor.assess(context);
|
|
257
|
+
}
|
|
258
|
+
if (this.portabilityAssessor) {
|
|
259
|
+
assessmentResults.portability =
|
|
260
|
+
await this.portabilityAssessor.assess(context);
|
|
261
|
+
}
|
|
185
262
|
}
|
|
186
263
|
// Collect test counts from all assessors
|
|
187
264
|
this.totalTestsRun = this.collectTotalTestCount();
|
|
188
265
|
// Determine overall status
|
|
189
|
-
const overallStatus = determineOverallStatus(assessmentResults);
|
|
266
|
+
const overallStatus = this.determineOverallStatus(assessmentResults);
|
|
190
267
|
// Generate summary and recommendations
|
|
191
|
-
const summary = generateSummary(assessmentResults);
|
|
192
|
-
const recommendations = generateRecommendations(assessmentResults);
|
|
268
|
+
const summary = this.generateSummary(assessmentResults);
|
|
269
|
+
const recommendations = this.generateRecommendations(assessmentResults);
|
|
193
270
|
const executionTime = Date.now() - this.startTime;
|
|
194
|
-
// Type assertion needed because Partial<MCPDirectoryAssessment> has optional required fields
|
|
195
|
-
// When modules are skipped via --skip-modules, not all fields will be present
|
|
196
271
|
return {
|
|
197
272
|
serverName: context.serverName,
|
|
198
273
|
assessmentDate: new Date().toISOString(),
|
|
@@ -204,19 +279,10 @@ export class AssessmentOrchestrator {
|
|
|
204
279
|
executionTime,
|
|
205
280
|
totalTestsRun: this.totalTestsRun,
|
|
206
281
|
mcpProtocolVersion: this.config.mcpProtocolVersion,
|
|
207
|
-
assessmentMetadata: {
|
|
208
|
-
// Source code is available if we have a path OR loaded source files
|
|
209
|
-
sourceCodeAvailable: !!context.sourceCodePath || (context.sourceCodeFiles?.size ?? 0) > 0,
|
|
210
|
-
// Use explicit transport type, or infer from available context
|
|
211
|
-
transportType: context.transportConfig?.type ??
|
|
212
|
-
(context.transportConfig?.url ? "streamable-http" : undefined),
|
|
213
|
-
},
|
|
214
282
|
};
|
|
215
283
|
}
|
|
216
284
|
/**
|
|
217
285
|
* Legacy assess method for backward compatibility
|
|
218
|
-
* @public
|
|
219
|
-
* @deprecated Use runFullAssessment() with AssessmentContext instead
|
|
220
286
|
*/
|
|
221
287
|
async assess(serverName, tools, callTool, serverInfo, readmeContent, packageJson) {
|
|
222
288
|
const context = {
|
|
@@ -231,21 +297,120 @@ export class AssessmentOrchestrator {
|
|
|
231
297
|
return this.runFullAssessment(context);
|
|
232
298
|
}
|
|
233
299
|
collectTotalTestCount() {
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
this.
|
|
300
|
+
let total = 0;
|
|
301
|
+
// Get actual test counts from assessors
|
|
302
|
+
const functionalityCount = this.functionalityAssessor.getTestCount();
|
|
303
|
+
const securityCount = this.securityAssessor.getTestCount();
|
|
304
|
+
const documentationCount = this.documentationAssessor.getTestCount();
|
|
305
|
+
const errorHandlingCount = this.errorHandlingAssessor.getTestCount();
|
|
306
|
+
const usabilityCount = this.usabilityAssessor.getTestCount();
|
|
307
|
+
const mcpSpecCount = this.mcpSpecAssessor?.getTestCount() || 0;
|
|
308
|
+
// New assessor counts
|
|
309
|
+
const aupCount = this.aupComplianceAssessor?.getTestCount() || 0;
|
|
310
|
+
const annotationCount = this.toolAnnotationAssessor?.getTestCount() || 0;
|
|
311
|
+
const librariesCount = this.prohibitedLibrariesAssessor?.getTestCount() || 0;
|
|
312
|
+
const manifestCount = this.manifestValidationAssessor?.getTestCount() || 0;
|
|
313
|
+
const portabilityCount = this.portabilityAssessor?.getTestCount() || 0;
|
|
314
|
+
console.log("[AssessmentOrchestrator] Test counts by assessor:", {
|
|
315
|
+
functionality: functionalityCount,
|
|
316
|
+
security: securityCount,
|
|
317
|
+
documentation: documentationCount,
|
|
318
|
+
errorHandling: errorHandlingCount,
|
|
319
|
+
usability: usabilityCount,
|
|
320
|
+
mcpSpec: mcpSpecCount,
|
|
321
|
+
aupCompliance: aupCount,
|
|
322
|
+
toolAnnotations: annotationCount,
|
|
323
|
+
prohibitedLibraries: librariesCount,
|
|
324
|
+
manifestValidation: manifestCount,
|
|
325
|
+
portability: portabilityCount,
|
|
326
|
+
});
|
|
327
|
+
total =
|
|
328
|
+
functionalityCount +
|
|
329
|
+
securityCount +
|
|
330
|
+
documentationCount +
|
|
331
|
+
errorHandlingCount +
|
|
332
|
+
usabilityCount +
|
|
333
|
+
mcpSpecCount +
|
|
334
|
+
aupCount +
|
|
335
|
+
annotationCount +
|
|
336
|
+
librariesCount +
|
|
337
|
+
manifestCount +
|
|
338
|
+
portabilityCount;
|
|
339
|
+
console.log("[AssessmentOrchestrator] Total test count:", total);
|
|
237
340
|
return total;
|
|
238
341
|
}
|
|
342
|
+
determineOverallStatus(results) {
|
|
343
|
+
const statuses = [];
|
|
344
|
+
// Collect all statuses
|
|
345
|
+
Object.values(results).forEach((assessment) => {
|
|
346
|
+
if (assessment?.status) {
|
|
347
|
+
statuses.push(assessment.status);
|
|
348
|
+
}
|
|
349
|
+
});
|
|
350
|
+
// If any critical category fails, overall fails
|
|
351
|
+
if (statuses.includes("FAIL"))
|
|
352
|
+
return "FAIL";
|
|
353
|
+
// If any category needs more info, overall needs more info
|
|
354
|
+
if (statuses.includes("NEED_MORE_INFO"))
|
|
355
|
+
return "NEED_MORE_INFO";
|
|
356
|
+
// All must pass for overall pass
|
|
357
|
+
return "PASS";
|
|
358
|
+
}
|
|
359
|
+
generateSummary(results) {
|
|
360
|
+
const parts = [];
|
|
361
|
+
const totalCategories = Object.keys(results).length;
|
|
362
|
+
const passedCategories = Object.values(results).filter((r) => r?.status === "PASS").length;
|
|
363
|
+
parts.push(`Assessment complete: ${passedCategories}/${totalCategories} categories passed.`);
|
|
364
|
+
// Add key findings
|
|
365
|
+
if (results.security?.vulnerabilities?.length > 0) {
|
|
366
|
+
parts.push(`Found ${results.security.vulnerabilities.length} security vulnerabilities.`);
|
|
367
|
+
}
|
|
368
|
+
if (results.functionality?.brokenTools?.length > 0) {
|
|
369
|
+
parts.push(`${results.functionality.brokenTools.length} tools are not functioning correctly.`);
|
|
370
|
+
}
|
|
371
|
+
// New assessor findings
|
|
372
|
+
if (results.aupCompliance?.violations?.length > 0) {
|
|
373
|
+
const criticalCount = results.aupCompliance.violations.filter((v) => v.severity === "CRITICAL").length;
|
|
374
|
+
if (criticalCount > 0) {
|
|
375
|
+
parts.push(`CRITICAL: ${criticalCount} AUP violation(s) detected.`);
|
|
376
|
+
}
|
|
377
|
+
else {
|
|
378
|
+
parts.push(`${results.aupCompliance.violations.length} AUP item(s) flagged for review.`);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
if (results.toolAnnotations?.missingAnnotationsCount > 0) {
|
|
382
|
+
parts.push(`${results.toolAnnotations.missingAnnotationsCount} tools missing annotations.`);
|
|
383
|
+
}
|
|
384
|
+
if (results.prohibitedLibraries?.matches?.length > 0) {
|
|
385
|
+
const blockingCount = results.prohibitedLibraries.matches.filter((m) => m.severity === "BLOCKING").length;
|
|
386
|
+
if (blockingCount > 0) {
|
|
387
|
+
parts.push(`BLOCKING: ${blockingCount} prohibited library/libraries detected.`);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
if (results.portability?.usesBundleRoot) {
|
|
391
|
+
parts.push("Uses ${BUNDLE_ROOT} anti-pattern.");
|
|
392
|
+
}
|
|
393
|
+
return parts.join(" ");
|
|
394
|
+
}
|
|
395
|
+
generateRecommendations(results) {
|
|
396
|
+
const recommendations = [];
|
|
397
|
+
// Aggregate recommendations from all assessments
|
|
398
|
+
Object.values(results).forEach((assessment) => {
|
|
399
|
+
if (assessment?.recommendations) {
|
|
400
|
+
recommendations.push(...assessment.recommendations);
|
|
401
|
+
}
|
|
402
|
+
});
|
|
403
|
+
// De-duplicate and prioritize
|
|
404
|
+
return [...new Set(recommendations)].slice(0, 10);
|
|
405
|
+
}
|
|
239
406
|
/**
|
|
240
407
|
* Get assessment configuration
|
|
241
|
-
* @public
|
|
242
408
|
*/
|
|
243
409
|
getConfig() {
|
|
244
410
|
return this.config;
|
|
245
411
|
}
|
|
246
412
|
/**
|
|
247
413
|
* Update assessment configuration
|
|
248
|
-
* @public
|
|
249
414
|
*/
|
|
250
415
|
updateConfig(config) {
|
|
251
416
|
this.config = { ...this.config, ...config };
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Error Handling Assessor Module
|
|
3
|
+
* Tests error handling and input validation
|
|
4
|
+
*/
|
|
5
|
+
import { ErrorHandlingAssessment } from "../../../lib/assessmentTypes.js";
|
|
6
|
+
import { BaseAssessor } from "./BaseAssessor.js";
|
|
7
|
+
import { AssessmentContext } from "../AssessmentOrchestrator.js";
|
|
8
|
+
export declare class ErrorHandlingAssessor extends BaseAssessor {
|
|
9
|
+
assess(context: AssessmentContext): Promise<ErrorHandlingAssessment>;
|
|
10
|
+
private selectToolsForTesting;
|
|
11
|
+
private testToolErrorHandling;
|
|
12
|
+
private testMissingParameters;
|
|
13
|
+
private testWrongTypes;
|
|
14
|
+
private testInvalidValues;
|
|
15
|
+
private testExcessiveInput;
|
|
16
|
+
private getToolSchema;
|
|
17
|
+
private generateWrongTypeParams;
|
|
18
|
+
private generateInvalidValueParams;
|
|
19
|
+
private generateParamsWithValue;
|
|
20
|
+
private calculateMetrics;
|
|
21
|
+
private determineErrorHandlingStatus;
|
|
22
|
+
private generateExplanation;
|
|
23
|
+
private generateRecommendations;
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=ErrorHandlingAssessor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ErrorHandlingAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ErrorHandlingAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAIxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D,qBAAa,qBAAsB,SAAQ,YAAY;IAC/C,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAyC1E,OAAO,CAAC,qBAAqB;YA8Cf,qBAAqB;YAuBrB,qBAAqB;YAmGrB,cAAc;YAmFd,iBAAiB;YA8DjB,kBAAkB;IA6DhC,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,uBAAuB;IAgC/B,OAAO,CAAC,0BAA0B;IAgClC,OAAO,CAAC,uBAAuB;IA4B/B,OAAO,CAAC,gBAAgB;IAoGxB,OAAO,CAAC,4BAA4B;IAapC,OAAO,CAAC,mBAAmB;IAuE3B,OAAO,CAAC,uBAAuB;CA4ChC"}
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* @module assessment/modules/ErrorHandlingAssessor
|
|
11
11
|
*/
|
|
12
12
|
import { BaseAssessor } from "./BaseAssessor.js";
|
|
13
|
-
import { ProtocolComplianceAssessor } from "./ProtocolComplianceAssessor.js";
|
|
13
|
+
import { ProtocolComplianceAssessor } from "./ProtocolComplianceAssessor/index.js";
|
|
14
14
|
/**
|
|
15
15
|
* @deprecated Use ProtocolComplianceAssessor instead.
|
|
16
16
|
*
|