@bryan-thompson/inspector-assessment-client 1.30.1 → 1.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-BbE88qbF.js → OAuthCallback-Cl2ANLYP.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-CfRYq1JG.js → OAuthDebugCallback-Ws62T4Ns.js} +1 -1
- package/dist/assets/{index-cHhcEXbr.css → index-BoUA5OL1.css} +3 -0
- package/dist/assets/{index-CsUB73MT.js → index-DaPIdOcS.js} +3746 -115
- package/dist/index.html +2 -2
- package/lib/lib/assessment/configTypes.d.ts +6 -0
- package/lib/lib/assessment/configTypes.d.ts.map +1 -1
- package/lib/lib/assessment/extendedTypes.d.ts +95 -0
- package/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
- package/lib/lib/assessment/resultTypes.d.ts +14 -2
- package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/lib/lib/assessment/sharedSchemas.d.ts +140 -0
- package/lib/lib/assessment/sharedSchemas.d.ts.map +1 -0
- package/lib/lib/assessment/sharedSchemas.js +113 -0
- package/lib/lib/moduleScoring.d.ts.map +1 -1
- package/lib/lib/moduleScoring.js +5 -0
- package/lib/lib/securityPatterns.d.ts.map +1 -1
- package/lib/lib/securityPatterns.js +2 -2
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +20 -18
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +143 -144
- package/lib/services/assessment/ResponseValidator.d.ts +10 -0
- package/lib/services/assessment/ResponseValidator.d.ts.map +1 -1
- package/lib/services/assessment/ResponseValidator.js +30 -6
- package/lib/services/assessment/config/performanceConfig.d.ts +2 -0
- package/lib/services/assessment/config/performanceConfig.d.ts.map +1 -1
- package/lib/services/assessment/config/performanceConfig.js +5 -33
- package/lib/services/assessment/config/performanceConfigSchemas.d.ts +111 -0
- package/lib/services/assessment/config/performanceConfigSchemas.d.ts.map +1 -0
- package/lib/services/assessment/config/performanceConfigSchemas.js +123 -0
- package/lib/services/assessment/modules/ConformanceAssessor.d.ts +64 -0
- package/lib/services/assessment/modules/ConformanceAssessor.d.ts.map +1 -0
- package/lib/services/assessment/modules/ConformanceAssessor.js +329 -0
- package/lib/services/assessment/modules/ResourceAssessor.d.ts +14 -0
- package/lib/services/assessment/modules/ResourceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ResourceAssessor.js +226 -0
- package/lib/services/assessment/modules/TemporalAssessor.d.ts +14 -0
- package/lib/services/assessment/modules/TemporalAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/TemporalAssessor.js +29 -1
- package/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts +9 -0
- package/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts.map +1 -1
- package/lib/services/assessment/modules/annotations/AlignmentChecker.js +97 -5
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts +6 -4
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts.map +1 -1
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.js +58 -0
- package/lib/services/assessment/modules/annotations/index.d.ts +1 -1
- package/lib/services/assessment/modules/annotations/index.d.ts.map +1 -1
- package/lib/services/assessment/modules/annotations/index.js +2 -1
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.js +3 -3
- package/lib/services/assessment/registry/AssessorDefinitions.d.ts +38 -0
- package/lib/services/assessment/registry/AssessorDefinitions.d.ts.map +1 -0
- package/lib/services/assessment/registry/AssessorDefinitions.js +370 -0
- package/lib/services/assessment/registry/AssessorRegistry.d.ts +124 -0
- package/lib/services/assessment/registry/AssessorRegistry.d.ts.map +1 -0
- package/lib/services/assessment/registry/AssessorRegistry.js +321 -0
- package/lib/services/assessment/registry/estimators.d.ts +93 -0
- package/lib/services/assessment/registry/estimators.d.ts.map +1 -0
- package/lib/services/assessment/registry/estimators.js +176 -0
- package/lib/services/assessment/registry/index.d.ts +13 -0
- package/lib/services/assessment/registry/index.d.ts.map +1 -0
- package/lib/services/assessment/registry/index.js +16 -0
- package/lib/services/assessment/registry/types.d.ts +180 -0
- package/lib/services/assessment/registry/types.d.ts.map +1 -0
- package/lib/services/assessment/registry/types.js +35 -0
- package/lib/services/assessment/responseValidatorSchemas.d.ts +751 -0
- package/lib/services/assessment/responseValidatorSchemas.d.ts.map +1 -0
- package/lib/services/assessment/responseValidatorSchemas.js +244 -0
- package/package.json +1 -1
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assessor Registry
|
|
3
|
+
*
|
|
4
|
+
* Central registry for managing assessor instances with metadata.
|
|
5
|
+
* Provides phase-ordered execution, Claude bridge wiring, and test count aggregation.
|
|
6
|
+
*
|
|
7
|
+
* @module assessment/registry/AssessorRegistry
|
|
8
|
+
* @see GitHub Issue #91
|
|
9
|
+
*/
|
|
10
|
+
import { createLogger, DEFAULT_LOGGING_CONFIG } from "../lib/logger.js";
|
|
11
|
+
import { AssessmentPhase, supportsClaudeBridge, } from "./types.js";
|
|
12
|
+
import { ASSESSOR_DEFINITIONS, getOrderedPhases } from "./AssessorDefinitions.js";
|
|
13
|
+
import { emitModuleStartedEvent, emitModuleProgress, } from "../orchestratorHelpers.js";
|
|
14
|
+
/**
|
|
15
|
+
* AssessorRegistry manages assessor instances and their execution.
|
|
16
|
+
*
|
|
17
|
+
* Key responsibilities:
|
|
18
|
+
* 1. Lazy instantiation based on configuration flags
|
|
19
|
+
* 2. Phase-ordered execution with Phase 0 always first and sequential
|
|
20
|
+
* 3. Claude bridge wiring to supporting assessors
|
|
21
|
+
* 4. Test count aggregation from all assessors
|
|
22
|
+
* 5. Backward-compatible property access via getAssessor()
|
|
23
|
+
*/
|
|
24
|
+
export class AssessorRegistry {
|
|
25
|
+
config;
|
|
26
|
+
logger;
|
|
27
|
+
assessors = new Map();
|
|
28
|
+
claudeBridge;
|
|
29
|
+
constructor(config) {
|
|
30
|
+
this.config = config;
|
|
31
|
+
this.logger = createLogger("AssessorRegistry", config.logging ?? DEFAULT_LOGGING_CONFIG);
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Register all enabled assessors based on configuration.
|
|
35
|
+
* Called during orchestrator initialization.
|
|
36
|
+
*/
|
|
37
|
+
registerAll(definitions = ASSESSOR_DEFINITIONS) {
|
|
38
|
+
this.logger.debug(`Registering ${definitions.length} assessor definitions`);
|
|
39
|
+
for (const definition of definitions) {
|
|
40
|
+
if (this.isEnabled(definition)) {
|
|
41
|
+
this.register(definition);
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
this.logger.debug(`Skipping disabled assessor: ${definition.id}`);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
this.logger.info(`Registered ${this.assessors.size} assessors`);
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Register a single assessor.
|
|
51
|
+
*/
|
|
52
|
+
register(definition) {
|
|
53
|
+
try {
|
|
54
|
+
// Instantiate the assessor
|
|
55
|
+
const instance = new definition.assessorClass(this.config);
|
|
56
|
+
// Run custom setup if defined
|
|
57
|
+
if (definition.customSetup) {
|
|
58
|
+
definition.customSetup(instance, this.config, this.logger);
|
|
59
|
+
}
|
|
60
|
+
// Wire Claude bridge if already set and assessor supports it
|
|
61
|
+
if (this.claudeBridge && definition.supportsClaudeBridge) {
|
|
62
|
+
this.wireClaudeBridge(instance);
|
|
63
|
+
}
|
|
64
|
+
const registered = {
|
|
65
|
+
definition,
|
|
66
|
+
instance,
|
|
67
|
+
enabled: true,
|
|
68
|
+
};
|
|
69
|
+
this.assessors.set(definition.id, registered);
|
|
70
|
+
this.logger.debug(`Registered assessor: ${definition.id}`);
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
this.logger.error(`Failed to register assessor: ${definition.id}`, {
|
|
74
|
+
error,
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Check if an assessor should be enabled based on configuration.
|
|
80
|
+
* Implements OR logic for backward-compatible deprecated flags.
|
|
81
|
+
*/
|
|
82
|
+
isEnabled(definition) {
|
|
83
|
+
const categories = this.config.assessmentCategories;
|
|
84
|
+
// Check if extended assessment is required but not enabled
|
|
85
|
+
if (definition.requiresExtended && !this.config.enableExtendedAssessment) {
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
// Check primary flag
|
|
89
|
+
const primaryEnabled = this.checkFlag(categories?.[definition.configFlags.primary], definition.configFlags.defaultEnabled ?? false);
|
|
90
|
+
if (primaryEnabled) {
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
93
|
+
// Check deprecated flags (OR logic for BC)
|
|
94
|
+
if (definition.configFlags.deprecated) {
|
|
95
|
+
for (const deprecatedFlag of definition.configFlags.deprecated) {
|
|
96
|
+
const deprecatedEnabled = categories?.[deprecatedFlag];
|
|
97
|
+
if (deprecatedEnabled === true) {
|
|
98
|
+
this.logger.warn(`Using deprecated flag '${String(deprecatedFlag)}' for ${definition.id}. ` +
|
|
99
|
+
`Please use '${String(definition.configFlags.primary)}' instead.`);
|
|
100
|
+
return true;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Check a single config flag value.
|
|
108
|
+
* If defaultEnabled is true, returns true unless flag is explicitly false.
|
|
109
|
+
* If defaultEnabled is false, returns true only if flag is explicitly true.
|
|
110
|
+
*/
|
|
111
|
+
checkFlag(flagValue, defaultEnabled) {
|
|
112
|
+
if (defaultEnabled) {
|
|
113
|
+
// Default enabled: check !== false
|
|
114
|
+
return flagValue !== false;
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
// Default disabled: check === true
|
|
118
|
+
return flagValue === true;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Get a registered assessor by ID.
|
|
123
|
+
* Returns undefined if not registered or disabled.
|
|
124
|
+
*/
|
|
125
|
+
getAssessor(id) {
|
|
126
|
+
const registered = this.assessors.get(id);
|
|
127
|
+
return registered?.instance;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Get all registered assessors for a specific phase.
|
|
131
|
+
*/
|
|
132
|
+
getByPhase(phase) {
|
|
133
|
+
return Array.from(this.assessors.values()).filter((r) => r.definition.phase === phase);
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Set Claude bridge for all supporting assessors.
|
|
137
|
+
* Called when Claude Code is enabled.
|
|
138
|
+
*/
|
|
139
|
+
setClaudeBridge(bridge) {
|
|
140
|
+
this.claudeBridge = bridge;
|
|
141
|
+
for (const registered of this.assessors.values()) {
|
|
142
|
+
if (registered.definition.supportsClaudeBridge) {
|
|
143
|
+
this.wireClaudeBridge(registered.instance);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Wire Claude bridge to a single assessor.
|
|
149
|
+
*/
|
|
150
|
+
wireClaudeBridge(assessor) {
|
|
151
|
+
if (this.claudeBridge && supportsClaudeBridge(assessor)) {
|
|
152
|
+
assessor.setClaudeBridge(this.claudeBridge);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Execute all assessors in phase order.
|
|
157
|
+
* Phase 0 (PRE) always runs first and sequentially.
|
|
158
|
+
* Other phases run based on parallelTesting config.
|
|
159
|
+
*
|
|
160
|
+
* @returns Partial MCPDirectoryAssessment with results from all assessors
|
|
161
|
+
*/
|
|
162
|
+
async executeAll(context) {
|
|
163
|
+
const results = {};
|
|
164
|
+
for (const phase of getOrderedPhases()) {
|
|
165
|
+
const phaseAssessors = this.getByPhase(phase);
|
|
166
|
+
if (phaseAssessors.length === 0) {
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
// Phase 0 (PRE) always runs sequentially for baseline capture
|
|
170
|
+
// Other phases respect parallelTesting config
|
|
171
|
+
const useParallel = phase !== AssessmentPhase.PRE && this.config.parallelTesting === true;
|
|
172
|
+
const phaseResults = await this.executePhase(phase, phaseAssessors, context, useParallel);
|
|
173
|
+
// Merge phase results into main results
|
|
174
|
+
for (const result of phaseResults) {
|
|
175
|
+
results[result.resultField] = result.result;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return results;
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Execute all assessors in a single phase.
|
|
182
|
+
*/
|
|
183
|
+
async executePhase(phase, assessors, context, parallel) {
|
|
184
|
+
const phaseName = AssessmentPhase[phase];
|
|
185
|
+
this.logger.debug(`Executing phase ${phaseName} with ${assessors.length} assessors (parallel: ${parallel})`);
|
|
186
|
+
const toolCount = this.getToolCountForContext(context);
|
|
187
|
+
if (parallel) {
|
|
188
|
+
return this.executeParallel(assessors, context, toolCount);
|
|
189
|
+
}
|
|
190
|
+
else {
|
|
191
|
+
return this.executeSequential(assessors, context, toolCount);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Execute assessors in parallel.
|
|
196
|
+
*/
|
|
197
|
+
async executeParallel(assessors, context, toolCount) {
|
|
198
|
+
const promises = assessors.map(async (registered) => {
|
|
199
|
+
const { definition, instance } = registered;
|
|
200
|
+
// Emit start event (writes to stderr)
|
|
201
|
+
const estimatedTests = definition.estimateTests(context, this.config);
|
|
202
|
+
emitModuleStartedEvent(definition.displayName, estimatedTests, toolCount);
|
|
203
|
+
// Execute
|
|
204
|
+
const startTime = Date.now();
|
|
205
|
+
try {
|
|
206
|
+
const result = await instance.assess(context);
|
|
207
|
+
const executionTime = Date.now() - startTime;
|
|
208
|
+
// Emit progress event (writes to stderr)
|
|
209
|
+
// Result should have a 'status' property
|
|
210
|
+
const status = this.extractStatus(result);
|
|
211
|
+
emitModuleProgress(definition.displayName, status, result, instance.getTestCount());
|
|
212
|
+
return {
|
|
213
|
+
id: definition.id,
|
|
214
|
+
resultField: definition.resultField,
|
|
215
|
+
result,
|
|
216
|
+
executionTime,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
catch (error) {
|
|
220
|
+
this.logger.error(`Assessor ${definition.id} failed`, { error });
|
|
221
|
+
throw error;
|
|
222
|
+
}
|
|
223
|
+
});
|
|
224
|
+
return Promise.all(promises);
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Execute assessors sequentially.
|
|
228
|
+
*/
|
|
229
|
+
async executeSequential(assessors, context, toolCount) {
|
|
230
|
+
const results = [];
|
|
231
|
+
for (const registered of assessors) {
|
|
232
|
+
const { definition, instance } = registered;
|
|
233
|
+
// Emit start event (writes to stderr)
|
|
234
|
+
const estimatedTests = definition.estimateTests(context, this.config);
|
|
235
|
+
emitModuleStartedEvent(definition.displayName, estimatedTests, toolCount);
|
|
236
|
+
// Execute
|
|
237
|
+
const startTime = Date.now();
|
|
238
|
+
try {
|
|
239
|
+
const result = await instance.assess(context);
|
|
240
|
+
const executionTime = Date.now() - startTime;
|
|
241
|
+
// Emit progress event (writes to stderr)
|
|
242
|
+
// Result should have a 'status' property
|
|
243
|
+
const status = this.extractStatus(result);
|
|
244
|
+
emitModuleProgress(definition.displayName, status, result, instance.getTestCount());
|
|
245
|
+
results.push({
|
|
246
|
+
id: definition.id,
|
|
247
|
+
resultField: definition.resultField,
|
|
248
|
+
result,
|
|
249
|
+
executionTime,
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
catch (error) {
|
|
253
|
+
this.logger.error(`Assessor ${definition.id} failed`, { error });
|
|
254
|
+
throw error;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
return results;
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* Extract status string from assessment result.
|
|
261
|
+
* Most results have a 'status' property.
|
|
262
|
+
*/
|
|
263
|
+
extractStatus(result) {
|
|
264
|
+
if (result && typeof result === "object" && "status" in result) {
|
|
265
|
+
return String(result.status);
|
|
266
|
+
}
|
|
267
|
+
return "UNKNOWN";
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Get tool count from context (respects selectedToolsForTesting config).
|
|
271
|
+
*/
|
|
272
|
+
getToolCountForContext(context) {
|
|
273
|
+
const tools = context.tools ?? [];
|
|
274
|
+
if (this.config.selectedToolsForTesting !== undefined) {
|
|
275
|
+
const selectedNames = new Set(this.config.selectedToolsForTesting);
|
|
276
|
+
return tools.filter((tool) => selectedNames.has(tool.name)).length;
|
|
277
|
+
}
|
|
278
|
+
return tools.length;
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* Get total test count from all registered assessors.
|
|
282
|
+
*/
|
|
283
|
+
getTotalTestCount() {
|
|
284
|
+
let total = 0;
|
|
285
|
+
for (const registered of this.assessors.values()) {
|
|
286
|
+
total += registered.instance.getTestCount();
|
|
287
|
+
}
|
|
288
|
+
return total;
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* Get test count for a specific assessor.
|
|
292
|
+
*/
|
|
293
|
+
getTestCount(id) {
|
|
294
|
+
const registered = this.assessors.get(id);
|
|
295
|
+
return registered?.instance.getTestCount() ?? 0;
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Get all registered assessor IDs.
|
|
299
|
+
*/
|
|
300
|
+
getRegisteredIds() {
|
|
301
|
+
return Array.from(this.assessors.keys());
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* Check if a specific assessor is registered.
|
|
305
|
+
*/
|
|
306
|
+
isRegistered(id) {
|
|
307
|
+
return this.assessors.has(id);
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Get the count of registered assessors.
|
|
311
|
+
*/
|
|
312
|
+
get size() {
|
|
313
|
+
return this.assessors.size;
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* Update configuration (e.g., for dynamic config changes).
|
|
317
|
+
*/
|
|
318
|
+
updateConfig(config) {
|
|
319
|
+
this.config = config;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test Count Estimators for AssessorRegistry
|
|
3
|
+
*
|
|
4
|
+
* Functions that estimate the number of tests each assessor will run.
|
|
5
|
+
* Used for progress event emission (emitModuleStartedEvent).
|
|
6
|
+
* Extracted from AssessmentOrchestrator for reuse in registry pattern.
|
|
7
|
+
*
|
|
8
|
+
* @module assessment/registry/estimators
|
|
9
|
+
* @see GitHub Issue #91
|
|
10
|
+
*/
|
|
11
|
+
import type { TestEstimatorFn } from "./types.js";
|
|
12
|
+
/**
|
|
13
|
+
* Temporal assessor: toolCount × temporalInvocations (default 25)
|
|
14
|
+
*/
|
|
15
|
+
export declare const estimateTemporalTests: TestEstimatorFn;
|
|
16
|
+
/**
|
|
17
|
+
* Functionality assessor: toolCount × 10 scenarios
|
|
18
|
+
*/
|
|
19
|
+
export declare const estimateFunctionalityTests: TestEstimatorFn;
|
|
20
|
+
/**
|
|
21
|
+
* Security assessor: toolCount × securityPatternsToTest (default 8)
|
|
22
|
+
*/
|
|
23
|
+
export declare const estimateSecurityTests: TestEstimatorFn;
|
|
24
|
+
/**
|
|
25
|
+
* Documentation assessor: fixed 5 checks
|
|
26
|
+
*/
|
|
27
|
+
export declare const estimateDocumentationTests: TestEstimatorFn;
|
|
28
|
+
/**
|
|
29
|
+
* Error handling assessor: toolCount × 5 error scenarios
|
|
30
|
+
*/
|
|
31
|
+
export declare const estimateErrorHandlingTests: TestEstimatorFn;
|
|
32
|
+
/**
|
|
33
|
+
* Usability assessor: fixed 10 checks
|
|
34
|
+
*/
|
|
35
|
+
export declare const estimateUsabilityTests: TestEstimatorFn;
|
|
36
|
+
/**
|
|
37
|
+
* Protocol compliance assessor: fixed 10 protocol checks
|
|
38
|
+
*/
|
|
39
|
+
export declare const estimateProtocolComplianceTests: TestEstimatorFn;
|
|
40
|
+
/**
|
|
41
|
+
* AUP compliance assessor: fixed 20 checks (14 AUP categories + violations)
|
|
42
|
+
*/
|
|
43
|
+
export declare const estimateAUPComplianceTests: TestEstimatorFn;
|
|
44
|
+
/**
|
|
45
|
+
* Tool annotations assessor: 1 check per tool
|
|
46
|
+
*/
|
|
47
|
+
export declare const estimateToolAnnotationTests: TestEstimatorFn;
|
|
48
|
+
/**
|
|
49
|
+
* Prohibited libraries assessor: fixed 5 library checks
|
|
50
|
+
*/
|
|
51
|
+
export declare const estimateProhibitedLibrariesTests: TestEstimatorFn;
|
|
52
|
+
/**
|
|
53
|
+
* Manifest validation assessor: fixed 10 manifest checks
|
|
54
|
+
*/
|
|
55
|
+
export declare const estimateManifestValidationTests: TestEstimatorFn;
|
|
56
|
+
/**
|
|
57
|
+
* Portability assessor: fixed 10 portability checks
|
|
58
|
+
*/
|
|
59
|
+
export declare const estimatePortabilityTests: TestEstimatorFn;
|
|
60
|
+
/**
|
|
61
|
+
* External API scanner assessor: fixed 10 API checks
|
|
62
|
+
*/
|
|
63
|
+
export declare const estimateExternalAPIScannerTests: TestEstimatorFn;
|
|
64
|
+
/**
|
|
65
|
+
* Authentication assessor: toolCount × 3 auth scenarios
|
|
66
|
+
*/
|
|
67
|
+
export declare const estimateAuthenticationTests: TestEstimatorFn;
|
|
68
|
+
/**
|
|
69
|
+
* Resource assessor: resourceCount × 5 resource tests
|
|
70
|
+
*/
|
|
71
|
+
export declare const estimateResourceTests: TestEstimatorFn;
|
|
72
|
+
/**
|
|
73
|
+
* Prompt assessor: promptCount × 10 prompt tests
|
|
74
|
+
*/
|
|
75
|
+
export declare const estimatePromptTests: TestEstimatorFn;
|
|
76
|
+
/**
|
|
77
|
+
* Cross-capability assessor: sum of capabilities × 5
|
|
78
|
+
*/
|
|
79
|
+
export declare const estimateCrossCapabilityTests: TestEstimatorFn;
|
|
80
|
+
/**
|
|
81
|
+
* File modularization assessor: 1 check per source file (or fixed 10 if no source)
|
|
82
|
+
*/
|
|
83
|
+
export declare const estimateFileModularizationTests: TestEstimatorFn;
|
|
84
|
+
/**
|
|
85
|
+
* Conformance assessor: fixed 7 conformance tests
|
|
86
|
+
*/
|
|
87
|
+
export declare const estimateConformanceTests: TestEstimatorFn;
|
|
88
|
+
/**
|
|
89
|
+
* Map of assessor ID to test estimator function.
|
|
90
|
+
* Used by AssessorRegistry for dynamic test count estimation.
|
|
91
|
+
*/
|
|
92
|
+
export declare const ESTIMATOR_MAP: Record<string, TestEstimatorFn>;
|
|
93
|
+
//# sourceMappingURL=estimators.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"estimators.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/registry/estimators.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAmC/C;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,eAInC,CAAC;AAMF;;GAEG;AACH,eAAO,MAAM,0BAA0B,EAAE,eACL,CAAC;AAErC;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,eAInC,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,0BAA0B,EAAE,eAAyB,CAAC;AAEnE;;GAEG;AACH,eAAO,MAAM,0BAA0B,EAAE,eACN,CAAC;AAEpC;;GAEG;AACH,eAAO,MAAM,sBAAsB,EAAE,eAA0B,CAAC;AAMhE;;GAEG;AACH,eAAO,MAAM,+BAA+B,EAAE,eAA0B,CAAC;AAMzE;;GAEG;AACH,eAAO,MAAM,0BAA0B,EAAE,eAA0B,CAAC;AAEpE;;GAEG;AACH,eAAO,MAAM,2BAA2B,EAAE,eACX,CAAC;AAEhC;;GAEG;AACH,eAAO,MAAM,gCAAgC,EAAE,eAAyB,CAAC;AAEzE;;GAEG;AACH,eAAO,MAAM,+BAA+B,EAAE,eAA0B,CAAC;AAEzE;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,eAA0B,CAAC;AAElE;;GAEG;AACH,eAAO,MAAM,+BAA+B,EAAE,eAA0B,CAAC;AAEzE;;GAEG;AACH,eAAO,MAAM,2BAA2B,EAAE,eACP,CAAC;AAMpC;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,eACL,CAAC;AAEhC;;GAEG;AACH,eAAO,MAAM,mBAAmB,EAAE,eACJ,CAAC;AAE/B;;GAEG;AACH,eAAO,MAAM,4BAA4B,EAAE,eAI1C,CAAC;AAMF;;GAEG;AACH,eAAO,MAAM,+BAA+B,EAAE,eAG7C,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,eAAyB,CAAC;AAMjE;;;GAGG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CA+BzD,CAAC"}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test Count Estimators for AssessorRegistry
|
|
3
|
+
*
|
|
4
|
+
* Functions that estimate the number of tests each assessor will run.
|
|
5
|
+
* Used for progress event emission (emitModuleStartedEvent).
|
|
6
|
+
* Extracted from AssessmentOrchestrator for reuse in registry pattern.
|
|
7
|
+
*
|
|
8
|
+
* @module assessment/registry/estimators
|
|
9
|
+
* @see GitHub Issue #91
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* Helper to get filtered tool count based on selectedToolsForTesting config.
|
|
13
|
+
*/
|
|
14
|
+
function getToolCount(context, config) {
|
|
15
|
+
const tools = context.tools ?? [];
|
|
16
|
+
if (config.selectedToolsForTesting !== undefined) {
|
|
17
|
+
const selectedNames = new Set(config.selectedToolsForTesting);
|
|
18
|
+
return tools.filter((tool) => selectedNames.has(tool.name)).length;
|
|
19
|
+
}
|
|
20
|
+
return tools.length;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Get resource count from context (for ResourceAssessor).
|
|
24
|
+
*/
|
|
25
|
+
function getResourceCount(context) {
|
|
26
|
+
return context.resources?.length ?? 0;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Get prompt count from context (for PromptAssessor).
|
|
30
|
+
*/
|
|
31
|
+
function getPromptCount(context) {
|
|
32
|
+
return context.prompts?.length ?? 0;
|
|
33
|
+
}
|
|
34
|
+
// ============================================================================
|
|
35
|
+
// Phase 0: PRE (Temporal)
|
|
36
|
+
// ============================================================================
|
|
37
|
+
/**
|
|
38
|
+
* Temporal assessor: toolCount × temporalInvocations (default 25)
|
|
39
|
+
*/
|
|
40
|
+
export const estimateTemporalTests = (context, config) => {
|
|
41
|
+
const toolCount = getToolCount(context, config);
|
|
42
|
+
const invocationsPerTool = config.temporalInvocations ?? 25;
|
|
43
|
+
return toolCount * invocationsPerTool;
|
|
44
|
+
};
|
|
45
|
+
// ============================================================================
|
|
46
|
+
// Phase 1: CORE (Functionality, Security, Documentation, ErrorHandling, Usability)
|
|
47
|
+
// ============================================================================
|
|
48
|
+
/**
|
|
49
|
+
* Functionality assessor: toolCount × 10 scenarios
|
|
50
|
+
*/
|
|
51
|
+
export const estimateFunctionalityTests = (context, config) => getToolCount(context, config) * 10;
|
|
52
|
+
/**
|
|
53
|
+
* Security assessor: toolCount × securityPatternsToTest (default 8)
|
|
54
|
+
*/
|
|
55
|
+
export const estimateSecurityTests = (context, config) => {
|
|
56
|
+
const toolCount = getToolCount(context, config);
|
|
57
|
+
const patternsToTest = config.securityPatternsToTest ?? 8;
|
|
58
|
+
return toolCount * patternsToTest;
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* Documentation assessor: fixed 5 checks
|
|
62
|
+
*/
|
|
63
|
+
export const estimateDocumentationTests = () => 5;
|
|
64
|
+
/**
|
|
65
|
+
* Error handling assessor: toolCount × 5 error scenarios
|
|
66
|
+
*/
|
|
67
|
+
export const estimateErrorHandlingTests = (context, config) => getToolCount(context, config) * 5;
|
|
68
|
+
/**
|
|
69
|
+
* Usability assessor: fixed 10 checks
|
|
70
|
+
*/
|
|
71
|
+
export const estimateUsabilityTests = () => 10;
|
|
72
|
+
// ============================================================================
|
|
73
|
+
// Phase 2: PROTOCOL (Protocol Compliance)
|
|
74
|
+
// ============================================================================
|
|
75
|
+
/**
|
|
76
|
+
* Protocol compliance assessor: fixed 10 protocol checks
|
|
77
|
+
*/
|
|
78
|
+
export const estimateProtocolComplianceTests = () => 10;
|
|
79
|
+
// ============================================================================
|
|
80
|
+
// Phase 3: COMPLIANCE (AUP, Annotations, Libraries, Manifest, Portability, APIs, Auth)
|
|
81
|
+
// ============================================================================
|
|
82
|
+
/**
|
|
83
|
+
* AUP compliance assessor: fixed 20 checks (14 AUP categories + violations)
|
|
84
|
+
*/
|
|
85
|
+
export const estimateAUPComplianceTests = () => 20;
|
|
86
|
+
/**
|
|
87
|
+
* Tool annotations assessor: 1 check per tool
|
|
88
|
+
*/
|
|
89
|
+
export const estimateToolAnnotationTests = (context, config) => getToolCount(context, config);
|
|
90
|
+
/**
|
|
91
|
+
* Prohibited libraries assessor: fixed 5 library checks
|
|
92
|
+
*/
|
|
93
|
+
export const estimateProhibitedLibrariesTests = () => 5;
|
|
94
|
+
/**
|
|
95
|
+
* Manifest validation assessor: fixed 10 manifest checks
|
|
96
|
+
*/
|
|
97
|
+
export const estimateManifestValidationTests = () => 10;
|
|
98
|
+
/**
|
|
99
|
+
* Portability assessor: fixed 10 portability checks
|
|
100
|
+
*/
|
|
101
|
+
export const estimatePortabilityTests = () => 10;
|
|
102
|
+
/**
|
|
103
|
+
* External API scanner assessor: fixed 10 API checks
|
|
104
|
+
*/
|
|
105
|
+
export const estimateExternalAPIScannerTests = () => 10;
|
|
106
|
+
/**
|
|
107
|
+
* Authentication assessor: toolCount × 3 auth scenarios
|
|
108
|
+
*/
|
|
109
|
+
export const estimateAuthenticationTests = (context, config) => getToolCount(context, config) * 3;
|
|
110
|
+
// ============================================================================
|
|
111
|
+
// Phase 4: CAPABILITY (Resources, Prompts, CrossCapability)
|
|
112
|
+
// ============================================================================
|
|
113
|
+
/**
|
|
114
|
+
* Resource assessor: resourceCount × 5 resource tests
|
|
115
|
+
*/
|
|
116
|
+
export const estimateResourceTests = (context) => getResourceCount(context) * 5;
|
|
117
|
+
/**
|
|
118
|
+
* Prompt assessor: promptCount × 10 prompt tests
|
|
119
|
+
*/
|
|
120
|
+
export const estimatePromptTests = (context) => getPromptCount(context) * 10;
|
|
121
|
+
/**
|
|
122
|
+
* Cross-capability assessor: sum of capabilities × 5
|
|
123
|
+
*/
|
|
124
|
+
export const estimateCrossCapabilityTests = (context) => {
|
|
125
|
+
const resourceCount = getResourceCount(context);
|
|
126
|
+
const promptCount = getPromptCount(context);
|
|
127
|
+
return (resourceCount + promptCount) * 5;
|
|
128
|
+
};
|
|
129
|
+
// ============================================================================
|
|
130
|
+
// Phase 5: QUALITY (FileModularization, Conformance)
|
|
131
|
+
// ============================================================================
|
|
132
|
+
/**
|
|
133
|
+
* File modularization assessor: 1 check per source file (or fixed 10 if no source)
|
|
134
|
+
*/
|
|
135
|
+
export const estimateFileModularizationTests = (context) => {
|
|
136
|
+
const sourceFiles = context.sourceCodeFiles?.size ?? 0;
|
|
137
|
+
return sourceFiles > 0 ? sourceFiles : 10;
|
|
138
|
+
};
|
|
139
|
+
/**
|
|
140
|
+
* Conformance assessor: fixed 7 conformance tests
|
|
141
|
+
*/
|
|
142
|
+
export const estimateConformanceTests = () => 7;
|
|
143
|
+
// ============================================================================
|
|
144
|
+
// Estimator Map (for dynamic lookup by assessor ID)
|
|
145
|
+
// ============================================================================
|
|
146
|
+
/**
|
|
147
|
+
* Map of assessor ID to test estimator function.
|
|
148
|
+
* Used by AssessorRegistry for dynamic test count estimation.
|
|
149
|
+
*/
|
|
150
|
+
export const ESTIMATOR_MAP = {
|
|
151
|
+
// Phase 0: PRE
|
|
152
|
+
temporal: estimateTemporalTests,
|
|
153
|
+
// Phase 1: CORE
|
|
154
|
+
functionality: estimateFunctionalityTests,
|
|
155
|
+
security: estimateSecurityTests,
|
|
156
|
+
documentation: estimateDocumentationTests,
|
|
157
|
+
errorHandling: estimateErrorHandlingTests,
|
|
158
|
+
usability: estimateUsabilityTests,
|
|
159
|
+
// Phase 2: PROTOCOL
|
|
160
|
+
protocolCompliance: estimateProtocolComplianceTests,
|
|
161
|
+
// Phase 3: COMPLIANCE
|
|
162
|
+
aupCompliance: estimateAUPComplianceTests,
|
|
163
|
+
toolAnnotations: estimateToolAnnotationTests,
|
|
164
|
+
prohibitedLibraries: estimateProhibitedLibrariesTests,
|
|
165
|
+
manifestValidation: estimateManifestValidationTests,
|
|
166
|
+
portability: estimatePortabilityTests,
|
|
167
|
+
externalAPIScanner: estimateExternalAPIScannerTests,
|
|
168
|
+
authentication: estimateAuthenticationTests,
|
|
169
|
+
// Phase 4: CAPABILITY
|
|
170
|
+
resources: estimateResourceTests,
|
|
171
|
+
prompts: estimatePromptTests,
|
|
172
|
+
crossCapability: estimateCrossCapabilityTests,
|
|
173
|
+
// Phase 5: QUALITY
|
|
174
|
+
fileModularization: estimateFileModularizationTests,
|
|
175
|
+
conformance: estimateConformanceTests,
|
|
176
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assessment Registry Module
|
|
3
|
+
*
|
|
4
|
+
* Exports all registry components for use by AssessmentOrchestrator.
|
|
5
|
+
*
|
|
6
|
+
* @module assessment/registry
|
|
7
|
+
* @see GitHub Issue #91
|
|
8
|
+
*/
|
|
9
|
+
export { AssessmentPhase, type AssessorDefinition, type AssessorConfigFlags, type AssessorConstructor, type AssessorSetupFn, type TestEstimatorFn, type RegisteredAssessor, type AssessorExecutionResult, type ClaudeBridgeCapable, supportsClaudeBridge, } from "./types.js";
|
|
10
|
+
export { ASSESSOR_DEFINITIONS, ASSESSOR_DEFINITION_MAP, getDefinitionsByPhase, getOrderedPhases, } from "./AssessorDefinitions.js";
|
|
11
|
+
export { AssessorRegistry } from "./AssessorRegistry.js";
|
|
12
|
+
export { estimateTemporalTests, estimateFunctionalityTests, estimateSecurityTests, estimateDocumentationTests, estimateErrorHandlingTests, estimateUsabilityTests, estimateProtocolComplianceTests, estimateAUPComplianceTests, estimateToolAnnotationTests, estimateProhibitedLibrariesTests, estimateManifestValidationTests, estimatePortabilityTests, estimateExternalAPIScannerTests, estimateAuthenticationTests, estimateResourceTests, estimatePromptTests, estimateCrossCapabilityTests, estimateFileModularizationTests, estimateConformanceTests, ESTIMATOR_MAP, } from "./estimators.js";
|
|
13
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/registry/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EACL,eAAe,EACf,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,kBAAkB,EACvB,KAAK,uBAAuB,EAC5B,KAAK,mBAAmB,EACxB,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAGjB,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,qBAAqB,EACrB,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAGtD,OAAO,EACL,qBAAqB,EACrB,0BAA0B,EAC1B,qBAAqB,EACrB,0BAA0B,EAC1B,0BAA0B,EAC1B,sBAAsB,EACtB,+BAA+B,EAC/B,0BAA0B,EAC1B,2BAA2B,EAC3B,gCAAgC,EAChC,+BAA+B,EAC/B,wBAAwB,EACxB,+BAA+B,EAC/B,2BAA2B,EAC3B,qBAAqB,EACrB,mBAAmB,EACnB,4BAA4B,EAC5B,+BAA+B,EAC/B,wBAAwB,EACxB,aAAa,GACd,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assessment Registry Module
|
|
3
|
+
*
|
|
4
|
+
* Exports all registry components for use by AssessmentOrchestrator.
|
|
5
|
+
*
|
|
6
|
+
* @module assessment/registry
|
|
7
|
+
* @see GitHub Issue #91
|
|
8
|
+
*/
|
|
9
|
+
// Types
|
|
10
|
+
export { AssessmentPhase, supportsClaudeBridge, } from "./types.js";
|
|
11
|
+
// Definitions
|
|
12
|
+
export { ASSESSOR_DEFINITIONS, ASSESSOR_DEFINITION_MAP, getDefinitionsByPhase, getOrderedPhases, } from "./AssessorDefinitions.js";
|
|
13
|
+
// Registry
|
|
14
|
+
export { AssessorRegistry } from "./AssessorRegistry.js";
|
|
15
|
+
// Estimators
|
|
16
|
+
export { estimateTemporalTests, estimateFunctionalityTests, estimateSecurityTests, estimateDocumentationTests, estimateErrorHandlingTests, estimateUsabilityTests, estimateProtocolComplianceTests, estimateAUPComplianceTests, estimateToolAnnotationTests, estimateProhibitedLibrariesTests, estimateManifestValidationTests, estimatePortabilityTests, estimateExternalAPIScannerTests, estimateAuthenticationTests, estimateResourceTests, estimatePromptTests, estimateCrossCapabilityTests, estimateFileModularizationTests, estimateConformanceTests, ESTIMATOR_MAP, } from "./estimators.js";
|