@bryan-thompson/inspector-assessment-client 1.31.0 → 1.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-CXcl26vR.js → OAuthCallback-DQqdKmh-.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-J9s4SF_c.js → OAuthDebugCallback-CZ79tNPn.js} +1 -1
- package/dist/assets/{index-_HAw2b2G.js → index-7dGKjpGX.js} +4 -4
- package/dist/index.html +1 -1
- package/lib/lib/assessment/extendedTypes.d.ts +21 -0
- package/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
- package/lib/lib/assessment/resultTypes.d.ts +12 -2
- package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/lib/lib/moduleScoring.d.ts.map +1 -1
- package/lib/lib/moduleScoring.js +5 -0
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +1 -28
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +66 -576
- package/lib/services/assessment/modules/ConformanceAssessor.d.ts +4 -0
- package/lib/services/assessment/modules/ConformanceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ConformanceAssessor.js +21 -0
- package/lib/services/assessment/modules/ResourceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ResourceAssessor.js +6 -1
- package/lib/services/assessment/registry/AssessorDefinitions.d.ts +38 -0
- package/lib/services/assessment/registry/AssessorDefinitions.d.ts.map +1 -0
- package/lib/services/assessment/registry/AssessorDefinitions.js +370 -0
- package/lib/services/assessment/registry/AssessorRegistry.d.ts +157 -0
- package/lib/services/assessment/registry/AssessorRegistry.d.ts.map +1 -0
- package/lib/services/assessment/registry/AssessorRegistry.js +364 -0
- package/lib/services/assessment/registry/estimators.d.ts +93 -0
- package/lib/services/assessment/registry/estimators.d.ts.map +1 -0
- package/lib/services/assessment/registry/estimators.js +176 -0
- package/lib/services/assessment/registry/index.d.ts +13 -0
- package/lib/services/assessment/registry/index.d.ts.map +1 -0
- package/lib/services/assessment/registry/index.js +16 -0
- package/lib/services/assessment/registry/types.d.ts +180 -0
- package/lib/services/assessment/registry/types.d.ts.map +1 -0
- package/lib/services/assessment/registry/types.js +35 -0
- package/lib/services/assessment/responseValidatorSchemas.d.ts +4 -4
- package/package.json +1 -1
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assessor Registry
|
|
3
|
+
*
|
|
4
|
+
* Central registry for managing assessor instances with metadata.
|
|
5
|
+
* Provides phase-ordered execution, Claude bridge wiring, and test count aggregation.
|
|
6
|
+
*
|
|
7
|
+
* @module assessment/registry/AssessorRegistry
|
|
8
|
+
* @see GitHub Issue #91
|
|
9
|
+
*/
|
|
10
|
+
import { createLogger, DEFAULT_LOGGING_CONFIG } from "../lib/logger.js";
|
|
11
|
+
import { AssessmentPhase, supportsClaudeBridge, } from "./types.js";
|
|
12
|
+
import { ASSESSOR_DEFINITIONS, getOrderedPhases } from "./AssessorDefinitions.js";
|
|
13
|
+
import { emitModuleStartedEvent, emitModuleProgress, } from "../orchestratorHelpers.js";
|
|
14
|
+
export class AssessorRegistry {
|
|
15
|
+
config;
|
|
16
|
+
logger;
|
|
17
|
+
assessors = new Map();
|
|
18
|
+
claudeBridge;
|
|
19
|
+
failedRegistrations = [];
|
|
20
|
+
constructor(config) {
|
|
21
|
+
this.config = config;
|
|
22
|
+
this.logger = createLogger("AssessorRegistry", config.logging ?? DEFAULT_LOGGING_CONFIG);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Register all enabled assessors based on configuration.
|
|
26
|
+
* Called during orchestrator initialization.
|
|
27
|
+
*/
|
|
28
|
+
registerAll(definitions = ASSESSOR_DEFINITIONS) {
|
|
29
|
+
this.logger.debug(`Registering ${definitions.length} assessor definitions`);
|
|
30
|
+
for (const definition of definitions) {
|
|
31
|
+
if (this.isEnabled(definition)) {
|
|
32
|
+
this.register(definition);
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
this.logger.debug(`Skipping disabled assessor: ${definition.id}`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
this.logger.info(`Registered ${this.assessors.size} assessors`);
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Register a single assessor.
|
|
42
|
+
*/
|
|
43
|
+
register(definition) {
|
|
44
|
+
try {
|
|
45
|
+
// Instantiate the assessor
|
|
46
|
+
const instance = new definition.assessorClass(this.config);
|
|
47
|
+
// Run custom setup if defined
|
|
48
|
+
if (definition.customSetup) {
|
|
49
|
+
definition.customSetup(instance, this.config, this.logger);
|
|
50
|
+
}
|
|
51
|
+
// Wire Claude bridge if already set and assessor supports it
|
|
52
|
+
if (this.claudeBridge && definition.supportsClaudeBridge) {
|
|
53
|
+
this.wireClaudeBridge(instance);
|
|
54
|
+
}
|
|
55
|
+
const registered = {
|
|
56
|
+
definition,
|
|
57
|
+
instance,
|
|
58
|
+
enabled: true,
|
|
59
|
+
};
|
|
60
|
+
this.assessors.set(definition.id, registered);
|
|
61
|
+
this.logger.debug(`Registered assessor: ${definition.id}`);
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
this.logger.error(`Failed to register assessor: ${definition.id}`, {
|
|
65
|
+
error,
|
|
66
|
+
});
|
|
67
|
+
// Track failed registrations for summary reporting (P1 fix)
|
|
68
|
+
this.failedRegistrations.push({
|
|
69
|
+
id: definition.id,
|
|
70
|
+
error: error instanceof Error ? error.message : String(error),
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Check if an assessor should be enabled based on configuration.
|
|
76
|
+
* Implements OR logic for backward-compatible deprecated flags.
|
|
77
|
+
*/
|
|
78
|
+
isEnabled(definition) {
|
|
79
|
+
const categories = this.config.assessmentCategories;
|
|
80
|
+
// Check if extended assessment is required but not enabled
|
|
81
|
+
if (definition.requiresExtended && !this.config.enableExtendedAssessment) {
|
|
82
|
+
return false;
|
|
83
|
+
}
|
|
84
|
+
// Check primary flag
|
|
85
|
+
const primaryEnabled = this.checkFlag(categories?.[definition.configFlags.primary], definition.configFlags.defaultEnabled ?? false);
|
|
86
|
+
if (primaryEnabled) {
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
// Check deprecated flags (OR logic for BC)
|
|
90
|
+
if (definition.configFlags.deprecated) {
|
|
91
|
+
for (const deprecatedFlag of definition.configFlags.deprecated) {
|
|
92
|
+
const deprecatedEnabled = categories?.[deprecatedFlag];
|
|
93
|
+
if (deprecatedEnabled === true) {
|
|
94
|
+
this.logger.warn(`Using deprecated flag '${String(deprecatedFlag)}' for ${definition.id}. ` +
|
|
95
|
+
`Please use '${String(definition.configFlags.primary)}' instead.`);
|
|
96
|
+
return true;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Check a single config flag value.
|
|
104
|
+
* If defaultEnabled is true, returns true unless flag is explicitly false.
|
|
105
|
+
* If defaultEnabled is false, returns true only if flag is explicitly true.
|
|
106
|
+
*/
|
|
107
|
+
checkFlag(flagValue, defaultEnabled) {
|
|
108
|
+
if (defaultEnabled) {
|
|
109
|
+
// Default enabled: check !== false
|
|
110
|
+
return flagValue !== false;
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
// Default disabled: check === true
|
|
114
|
+
return flagValue === true;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Get a registered assessor by ID.
|
|
119
|
+
* Returns undefined if not registered or disabled.
|
|
120
|
+
*/
|
|
121
|
+
getAssessor(id) {
|
|
122
|
+
const registered = this.assessors.get(id);
|
|
123
|
+
return registered?.instance;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Get all registered assessors for a specific phase.
|
|
127
|
+
*/
|
|
128
|
+
getByPhase(phase) {
|
|
129
|
+
return Array.from(this.assessors.values()).filter((r) => r.definition.phase === phase);
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Set Claude bridge for all supporting assessors.
|
|
133
|
+
* Called when Claude Code is enabled.
|
|
134
|
+
*/
|
|
135
|
+
setClaudeBridge(bridge) {
|
|
136
|
+
this.claudeBridge = bridge;
|
|
137
|
+
for (const registered of this.assessors.values()) {
|
|
138
|
+
if (registered.definition.supportsClaudeBridge) {
|
|
139
|
+
this.wireClaudeBridge(registered.instance);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Wire Claude bridge to a single assessor.
|
|
145
|
+
*/
|
|
146
|
+
wireClaudeBridge(assessor) {
|
|
147
|
+
if (this.claudeBridge && supportsClaudeBridge(assessor)) {
|
|
148
|
+
assessor.setClaudeBridge(this.claudeBridge);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Execute all assessors in phase order.
|
|
153
|
+
* Phase 0 (PRE) always runs first and sequentially.
|
|
154
|
+
* Other phases run based on parallelTesting config.
|
|
155
|
+
*
|
|
156
|
+
* @returns Partial MCPDirectoryAssessment with results from all assessors
|
|
157
|
+
*/
|
|
158
|
+
async executeAll(context) {
|
|
159
|
+
const results = {};
|
|
160
|
+
for (const phase of getOrderedPhases()) {
|
|
161
|
+
const phaseAssessors = this.getByPhase(phase);
|
|
162
|
+
if (phaseAssessors.length === 0) {
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
// Phase 0 (PRE) always runs sequentially for baseline capture
|
|
166
|
+
// Other phases respect parallelTesting config
|
|
167
|
+
const useParallel = phase !== AssessmentPhase.PRE && this.config.parallelTesting === true;
|
|
168
|
+
const phaseResults = await this.executePhase(phase, phaseAssessors, context, useParallel);
|
|
169
|
+
// Merge phase results into main results
|
|
170
|
+
for (const result of phaseResults) {
|
|
171
|
+
results[result.resultField] =
|
|
172
|
+
result.result;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return results;
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Execute all assessors in a single phase.
|
|
179
|
+
*/
|
|
180
|
+
async executePhase(phase, assessors, context, parallel) {
|
|
181
|
+
const phaseName = AssessmentPhase[phase];
|
|
182
|
+
this.logger.debug(`Executing phase ${phaseName} with ${assessors.length} assessors (parallel: ${parallel})`);
|
|
183
|
+
const toolCount = this.getToolCountForContext(context);
|
|
184
|
+
if (parallel) {
|
|
185
|
+
return this.executeParallel(assessors, context, toolCount);
|
|
186
|
+
}
|
|
187
|
+
else {
|
|
188
|
+
return this.executeSequential(assessors, context, toolCount);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Execute assessors in parallel with graceful degradation.
|
|
193
|
+
* Uses Promise.allSettled to continue execution even if some assessors fail.
|
|
194
|
+
*/
|
|
195
|
+
async executeParallel(assessors, context, toolCount) {
|
|
196
|
+
const promises = assessors.map(async (registered) => {
|
|
197
|
+
const { definition, instance } = registered;
|
|
198
|
+
// Emit start event (writes to stderr)
|
|
199
|
+
const estimatedTests = definition.estimateTests(context, this.config);
|
|
200
|
+
emitModuleStartedEvent(definition.displayName, estimatedTests, toolCount);
|
|
201
|
+
// Execute
|
|
202
|
+
const startTime = Date.now();
|
|
203
|
+
const result = await instance.assess(context);
|
|
204
|
+
const executionTime = Date.now() - startTime;
|
|
205
|
+
// Emit progress event (writes to stderr)
|
|
206
|
+
const status = this.extractStatus(result);
|
|
207
|
+
emitModuleProgress(definition.displayName, status, result, instance.getTestCount());
|
|
208
|
+
return {
|
|
209
|
+
id: definition.id,
|
|
210
|
+
resultField: definition.resultField,
|
|
211
|
+
result,
|
|
212
|
+
executionTime,
|
|
213
|
+
};
|
|
214
|
+
});
|
|
215
|
+
// Use Promise.allSettled for graceful degradation (P1 fix)
|
|
216
|
+
const settledResults = await Promise.allSettled(promises);
|
|
217
|
+
const successResults = [];
|
|
218
|
+
for (let i = 0; i < settledResults.length; i++) {
|
|
219
|
+
const settledResult = settledResults[i];
|
|
220
|
+
if (settledResult.status === "fulfilled") {
|
|
221
|
+
successResults.push(settledResult.value);
|
|
222
|
+
}
|
|
223
|
+
else {
|
|
224
|
+
const definition = assessors[i].definition;
|
|
225
|
+
this.logger.error(`Assessor ${definition.id} failed during parallel execution`, {
|
|
226
|
+
error: settledResult.reason,
|
|
227
|
+
});
|
|
228
|
+
// Emit failure progress event
|
|
229
|
+
emitModuleProgress(definition.displayName, "ERROR", null, 0);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
if (successResults.length < assessors.length) {
|
|
233
|
+
this.logger.warn(`${assessors.length - successResults.length} assessor(s) failed during parallel execution`);
|
|
234
|
+
}
|
|
235
|
+
return successResults;
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Execute assessors sequentially.
|
|
239
|
+
*/
|
|
240
|
+
async executeSequential(assessors, context, toolCount) {
|
|
241
|
+
const results = [];
|
|
242
|
+
for (const registered of assessors) {
|
|
243
|
+
const { definition, instance } = registered;
|
|
244
|
+
// Emit start event (writes to stderr)
|
|
245
|
+
const estimatedTests = definition.estimateTests(context, this.config);
|
|
246
|
+
emitModuleStartedEvent(definition.displayName, estimatedTests, toolCount);
|
|
247
|
+
// Execute
|
|
248
|
+
const startTime = Date.now();
|
|
249
|
+
try {
|
|
250
|
+
const result = await instance.assess(context);
|
|
251
|
+
const executionTime = Date.now() - startTime;
|
|
252
|
+
// Emit progress event (writes to stderr)
|
|
253
|
+
// Result should have a 'status' property
|
|
254
|
+
const status = this.extractStatus(result);
|
|
255
|
+
emitModuleProgress(definition.displayName, status, result, instance.getTestCount());
|
|
256
|
+
results.push({
|
|
257
|
+
id: definition.id,
|
|
258
|
+
resultField: definition.resultField,
|
|
259
|
+
result,
|
|
260
|
+
executionTime,
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
catch (error) {
|
|
264
|
+
this.logger.error(`Assessor ${definition.id} failed`, { error });
|
|
265
|
+
throw error;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
return results;
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Extract status string from assessment result.
|
|
272
|
+
* Most results have a 'status' property.
|
|
273
|
+
*/
|
|
274
|
+
extractStatus(result) {
|
|
275
|
+
if (result && typeof result === "object" && "status" in result) {
|
|
276
|
+
return String(result.status);
|
|
277
|
+
}
|
|
278
|
+
return "UNKNOWN";
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* Get tool count from context (respects selectedToolsForTesting config).
|
|
282
|
+
*/
|
|
283
|
+
getToolCountForContext(context) {
|
|
284
|
+
const tools = context.tools ?? [];
|
|
285
|
+
if (this.config.selectedToolsForTesting !== undefined) {
|
|
286
|
+
const selectedNames = new Set(this.config.selectedToolsForTesting);
|
|
287
|
+
return tools.filter((tool) => selectedNames.has(tool.name)).length;
|
|
288
|
+
}
|
|
289
|
+
return tools.length;
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Get total test count from all registered assessors.
|
|
293
|
+
*/
|
|
294
|
+
getTotalTestCount() {
|
|
295
|
+
let total = 0;
|
|
296
|
+
for (const registered of this.assessors.values()) {
|
|
297
|
+
total += registered.instance.getTestCount();
|
|
298
|
+
}
|
|
299
|
+
return total;
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* Get test count for a specific assessor.
|
|
303
|
+
*/
|
|
304
|
+
getTestCount(id) {
|
|
305
|
+
const registered = this.assessors.get(id);
|
|
306
|
+
return registered?.instance.getTestCount() ?? 0;
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Get all registered assessor IDs.
|
|
310
|
+
*/
|
|
311
|
+
getRegisteredIds() {
|
|
312
|
+
return Array.from(this.assessors.keys());
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* Check if a specific assessor is registered.
|
|
316
|
+
*/
|
|
317
|
+
isRegistered(id) {
|
|
318
|
+
return this.assessors.has(id);
|
|
319
|
+
}
|
|
320
|
+
/**
|
|
321
|
+
* Get the count of registered assessors.
|
|
322
|
+
*/
|
|
323
|
+
get size() {
|
|
324
|
+
return this.assessors.size;
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* Update configuration for future operations.
|
|
328
|
+
*
|
|
329
|
+
* **Important**: This does NOT re-register assessors. Assessors are registered
|
|
330
|
+
* once during construction based on the initial config. To change which
|
|
331
|
+
* assessors are enabled, create a new AssessorRegistry instance.
|
|
332
|
+
*
|
|
333
|
+
* @param config - New configuration to use
|
|
334
|
+
*/
|
|
335
|
+
updateConfig(config) {
|
|
336
|
+
this.config = config;
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* Reset test counts for all registered assessors.
|
|
340
|
+
* Called at the start of each assessment run.
|
|
341
|
+
*/
|
|
342
|
+
resetAllTestCounts() {
|
|
343
|
+
for (const registered of this.assessors.values()) {
|
|
344
|
+
registered.instance.resetTestCount();
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Get list of assessors that failed to register.
|
|
349
|
+
* Useful for reporting partial assessment results.
|
|
350
|
+
*
|
|
351
|
+
* @returns Array of failed registration info (id and error message)
|
|
352
|
+
*/
|
|
353
|
+
getFailedRegistrations() {
|
|
354
|
+
return [...this.failedRegistrations];
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* Check if any assessors failed to register.
|
|
358
|
+
*
|
|
359
|
+
* @returns true if at least one assessor failed registration
|
|
360
|
+
*/
|
|
361
|
+
hasFailedRegistrations() {
|
|
362
|
+
return this.failedRegistrations.length > 0;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test Count Estimators for AssessorRegistry
|
|
3
|
+
*
|
|
4
|
+
* Functions that estimate the number of tests each assessor will run.
|
|
5
|
+
* Used for progress event emission (emitModuleStartedEvent).
|
|
6
|
+
* Extracted from AssessmentOrchestrator for reuse in registry pattern.
|
|
7
|
+
*
|
|
8
|
+
* @module assessment/registry/estimators
|
|
9
|
+
* @see GitHub Issue #91
|
|
10
|
+
*/
|
|
11
|
+
import type { TestEstimatorFn } from "./types.js";
|
|
12
|
+
/**
|
|
13
|
+
* Temporal assessor: toolCount × temporalInvocations (default 25)
|
|
14
|
+
*/
|
|
15
|
+
export declare const estimateTemporalTests: TestEstimatorFn;
|
|
16
|
+
/**
|
|
17
|
+
* Functionality assessor: toolCount × 10 scenarios
|
|
18
|
+
*/
|
|
19
|
+
export declare const estimateFunctionalityTests: TestEstimatorFn;
|
|
20
|
+
/**
|
|
21
|
+
* Security assessor: toolCount × securityPatternsToTest (default 8)
|
|
22
|
+
*/
|
|
23
|
+
export declare const estimateSecurityTests: TestEstimatorFn;
|
|
24
|
+
/**
|
|
25
|
+
* Documentation assessor: fixed 5 checks
|
|
26
|
+
*/
|
|
27
|
+
export declare const estimateDocumentationTests: TestEstimatorFn;
|
|
28
|
+
/**
|
|
29
|
+
* Error handling assessor: toolCount × 5 error scenarios
|
|
30
|
+
*/
|
|
31
|
+
export declare const estimateErrorHandlingTests: TestEstimatorFn;
|
|
32
|
+
/**
|
|
33
|
+
* Usability assessor: fixed 10 checks
|
|
34
|
+
*/
|
|
35
|
+
export declare const estimateUsabilityTests: TestEstimatorFn;
|
|
36
|
+
/**
|
|
37
|
+
* Protocol compliance assessor: fixed 10 protocol checks
|
|
38
|
+
*/
|
|
39
|
+
export declare const estimateProtocolComplianceTests: TestEstimatorFn;
|
|
40
|
+
/**
|
|
41
|
+
* AUP compliance assessor: fixed 20 checks (14 AUP categories + violations)
|
|
42
|
+
*/
|
|
43
|
+
export declare const estimateAUPComplianceTests: TestEstimatorFn;
|
|
44
|
+
/**
|
|
45
|
+
* Tool annotations assessor: 1 check per tool
|
|
46
|
+
*/
|
|
47
|
+
export declare const estimateToolAnnotationTests: TestEstimatorFn;
|
|
48
|
+
/**
|
|
49
|
+
* Prohibited libraries assessor: fixed 5 library checks
|
|
50
|
+
*/
|
|
51
|
+
export declare const estimateProhibitedLibrariesTests: TestEstimatorFn;
|
|
52
|
+
/**
|
|
53
|
+
* Manifest validation assessor: fixed 10 manifest checks
|
|
54
|
+
*/
|
|
55
|
+
export declare const estimateManifestValidationTests: TestEstimatorFn;
|
|
56
|
+
/**
|
|
57
|
+
* Portability assessor: fixed 10 portability checks
|
|
58
|
+
*/
|
|
59
|
+
export declare const estimatePortabilityTests: TestEstimatorFn;
|
|
60
|
+
/**
|
|
61
|
+
* External API scanner assessor: fixed 10 API checks
|
|
62
|
+
*/
|
|
63
|
+
export declare const estimateExternalAPIScannerTests: TestEstimatorFn;
|
|
64
|
+
/**
|
|
65
|
+
* Authentication assessor: toolCount × 3 auth scenarios
|
|
66
|
+
*/
|
|
67
|
+
export declare const estimateAuthenticationTests: TestEstimatorFn;
|
|
68
|
+
/**
|
|
69
|
+
* Resource assessor: resourceCount × 5 resource tests
|
|
70
|
+
*/
|
|
71
|
+
export declare const estimateResourceTests: TestEstimatorFn;
|
|
72
|
+
/**
|
|
73
|
+
* Prompt assessor: promptCount × 10 prompt tests
|
|
74
|
+
*/
|
|
75
|
+
export declare const estimatePromptTests: TestEstimatorFn;
|
|
76
|
+
/**
|
|
77
|
+
* Cross-capability assessor: sum of capabilities × 5
|
|
78
|
+
*/
|
|
79
|
+
export declare const estimateCrossCapabilityTests: TestEstimatorFn;
|
|
80
|
+
/**
|
|
81
|
+
* File modularization assessor: 1 check per source file (or fixed 10 if no source)
|
|
82
|
+
*/
|
|
83
|
+
export declare const estimateFileModularizationTests: TestEstimatorFn;
|
|
84
|
+
/**
|
|
85
|
+
* Conformance assessor: fixed 7 conformance tests
|
|
86
|
+
*/
|
|
87
|
+
export declare const estimateConformanceTests: TestEstimatorFn;
|
|
88
|
+
/**
|
|
89
|
+
* Map of assessor ID to test estimator function.
|
|
90
|
+
* Used by AssessorRegistry for dynamic test count estimation.
|
|
91
|
+
*/
|
|
92
|
+
export declare const ESTIMATOR_MAP: Record<string, TestEstimatorFn>;
|
|
93
|
+
//# sourceMappingURL=estimators.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"estimators.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/registry/estimators.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAmC/C;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,eAInC,CAAC;AAMF;;GAEG;AACH,eAAO,MAAM,0BAA0B,EAAE,eACL,CAAC;AAErC;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,eAInC,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,0BAA0B,EAAE,eAAyB,CAAC;AAEnE;;GAEG;AACH,eAAO,MAAM,0BAA0B,EAAE,eACN,CAAC;AAEpC;;GAEG;AACH,eAAO,MAAM,sBAAsB,EAAE,eAA0B,CAAC;AAMhE;;GAEG;AACH,eAAO,MAAM,+BAA+B,EAAE,eAA0B,CAAC;AAMzE;;GAEG;AACH,eAAO,MAAM,0BAA0B,EAAE,eAA0B,CAAC;AAEpE;;GAEG;AACH,eAAO,MAAM,2BAA2B,EAAE,eACX,CAAC;AAEhC;;GAEG;AACH,eAAO,MAAM,gCAAgC,EAAE,eAAyB,CAAC;AAEzE;;GAEG;AACH,eAAO,MAAM,+BAA+B,EAAE,eAA0B,CAAC;AAEzE;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,eAA0B,CAAC;AAElE;;GAEG;AACH,eAAO,MAAM,+BAA+B,EAAE,eAA0B,CAAC;AAEzE;;GAEG;AACH,eAAO,MAAM,2BAA2B,EAAE,eACP,CAAC;AAMpC;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,eACL,CAAC;AAEhC;;GAEG;AACH,eAAO,MAAM,mBAAmB,EAAE,eACJ,CAAC;AAE/B;;GAEG;AACH,eAAO,MAAM,4BAA4B,EAAE,eAI1C,CAAC;AAMF;;GAEG;AACH,eAAO,MAAM,+BAA+B,EAAE,eAG7C,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,eAAyB,CAAC;AAMjE;;;GAGG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CA+BzD,CAAC"}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test Count Estimators for AssessorRegistry
|
|
3
|
+
*
|
|
4
|
+
* Functions that estimate the number of tests each assessor will run.
|
|
5
|
+
* Used for progress event emission (emitModuleStartedEvent).
|
|
6
|
+
* Extracted from AssessmentOrchestrator for reuse in registry pattern.
|
|
7
|
+
*
|
|
8
|
+
* @module assessment/registry/estimators
|
|
9
|
+
* @see GitHub Issue #91
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* Helper to get filtered tool count based on selectedToolsForTesting config.
|
|
13
|
+
*/
|
|
14
|
+
function getToolCount(context, config) {
|
|
15
|
+
const tools = context.tools ?? [];
|
|
16
|
+
if (config.selectedToolsForTesting !== undefined) {
|
|
17
|
+
const selectedNames = new Set(config.selectedToolsForTesting);
|
|
18
|
+
return tools.filter((tool) => selectedNames.has(tool.name)).length;
|
|
19
|
+
}
|
|
20
|
+
return tools.length;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Get resource count from context (for ResourceAssessor).
|
|
24
|
+
*/
|
|
25
|
+
function getResourceCount(context) {
|
|
26
|
+
return context.resources?.length ?? 0;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Get prompt count from context (for PromptAssessor).
|
|
30
|
+
*/
|
|
31
|
+
function getPromptCount(context) {
|
|
32
|
+
return context.prompts?.length ?? 0;
|
|
33
|
+
}
|
|
34
|
+
// ============================================================================
|
|
35
|
+
// Phase 0: PRE (Temporal)
|
|
36
|
+
// ============================================================================
|
|
37
|
+
/**
|
|
38
|
+
* Temporal assessor: toolCount × temporalInvocations (default 25)
|
|
39
|
+
*/
|
|
40
|
+
export const estimateTemporalTests = (context, config) => {
|
|
41
|
+
const toolCount = getToolCount(context, config);
|
|
42
|
+
const invocationsPerTool = config.temporalInvocations ?? 25;
|
|
43
|
+
return toolCount * invocationsPerTool;
|
|
44
|
+
};
|
|
45
|
+
// ============================================================================
|
|
46
|
+
// Phase 1: CORE (Functionality, Security, Documentation, ErrorHandling, Usability)
|
|
47
|
+
// ============================================================================
|
|
48
|
+
/**
|
|
49
|
+
* Functionality assessor: toolCount × 10 scenarios
|
|
50
|
+
*/
|
|
51
|
+
export const estimateFunctionalityTests = (context, config) => getToolCount(context, config) * 10;
|
|
52
|
+
/**
|
|
53
|
+
* Security assessor: toolCount × securityPatternsToTest (default 8)
|
|
54
|
+
*/
|
|
55
|
+
export const estimateSecurityTests = (context, config) => {
|
|
56
|
+
const toolCount = getToolCount(context, config);
|
|
57
|
+
const patternsToTest = config.securityPatternsToTest ?? 8;
|
|
58
|
+
return toolCount * patternsToTest;
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* Documentation assessor: fixed 5 checks
|
|
62
|
+
*/
|
|
63
|
+
export const estimateDocumentationTests = () => 5;
|
|
64
|
+
/**
|
|
65
|
+
* Error handling assessor: toolCount × 5 error scenarios
|
|
66
|
+
*/
|
|
67
|
+
export const estimateErrorHandlingTests = (context, config) => getToolCount(context, config) * 5;
|
|
68
|
+
/**
|
|
69
|
+
* Usability assessor: fixed 10 checks
|
|
70
|
+
*/
|
|
71
|
+
export const estimateUsabilityTests = () => 10;
|
|
72
|
+
// ============================================================================
|
|
73
|
+
// Phase 2: PROTOCOL (Protocol Compliance)
|
|
74
|
+
// ============================================================================
|
|
75
|
+
/**
|
|
76
|
+
* Protocol compliance assessor: fixed 10 protocol checks
|
|
77
|
+
*/
|
|
78
|
+
export const estimateProtocolComplianceTests = () => 10;
|
|
79
|
+
// ============================================================================
|
|
80
|
+
// Phase 3: COMPLIANCE (AUP, Annotations, Libraries, Manifest, Portability, APIs, Auth)
|
|
81
|
+
// ============================================================================
|
|
82
|
+
/**
|
|
83
|
+
* AUP compliance assessor: fixed 20 checks (14 AUP categories + violations)
|
|
84
|
+
*/
|
|
85
|
+
export const estimateAUPComplianceTests = () => 20;
|
|
86
|
+
/**
|
|
87
|
+
* Tool annotations assessor: 1 check per tool
|
|
88
|
+
*/
|
|
89
|
+
export const estimateToolAnnotationTests = (context, config) => getToolCount(context, config);
|
|
90
|
+
/**
|
|
91
|
+
* Prohibited libraries assessor: fixed 5 library checks
|
|
92
|
+
*/
|
|
93
|
+
export const estimateProhibitedLibrariesTests = () => 5;
|
|
94
|
+
/**
|
|
95
|
+
* Manifest validation assessor: fixed 10 manifest checks
|
|
96
|
+
*/
|
|
97
|
+
export const estimateManifestValidationTests = () => 10;
|
|
98
|
+
/**
|
|
99
|
+
* Portability assessor: fixed 10 portability checks
|
|
100
|
+
*/
|
|
101
|
+
export const estimatePortabilityTests = () => 10;
|
|
102
|
+
/**
|
|
103
|
+
* External API scanner assessor: fixed 10 API checks
|
|
104
|
+
*/
|
|
105
|
+
export const estimateExternalAPIScannerTests = () => 10;
|
|
106
|
+
/**
|
|
107
|
+
* Authentication assessor: toolCount × 3 auth scenarios
|
|
108
|
+
*/
|
|
109
|
+
export const estimateAuthenticationTests = (context, config) => getToolCount(context, config) * 3;
|
|
110
|
+
// ============================================================================
|
|
111
|
+
// Phase 4: CAPABILITY (Resources, Prompts, CrossCapability)
|
|
112
|
+
// ============================================================================
|
|
113
|
+
/**
|
|
114
|
+
* Resource assessor: resourceCount × 5 resource tests
|
|
115
|
+
*/
|
|
116
|
+
export const estimateResourceTests = (context) => getResourceCount(context) * 5;
|
|
117
|
+
/**
|
|
118
|
+
* Prompt assessor: promptCount × 10 prompt tests
|
|
119
|
+
*/
|
|
120
|
+
export const estimatePromptTests = (context) => getPromptCount(context) * 10;
|
|
121
|
+
/**
|
|
122
|
+
* Cross-capability assessor: sum of capabilities × 5
|
|
123
|
+
*/
|
|
124
|
+
export const estimateCrossCapabilityTests = (context) => {
|
|
125
|
+
const resourceCount = getResourceCount(context);
|
|
126
|
+
const promptCount = getPromptCount(context);
|
|
127
|
+
return (resourceCount + promptCount) * 5;
|
|
128
|
+
};
|
|
129
|
+
// ============================================================================
|
|
130
|
+
// Phase 5: QUALITY (FileModularization, Conformance)
|
|
131
|
+
// ============================================================================
|
|
132
|
+
/**
|
|
133
|
+
* File modularization assessor: 1 check per source file (or fixed 10 if no source)
|
|
134
|
+
*/
|
|
135
|
+
export const estimateFileModularizationTests = (context) => {
|
|
136
|
+
const sourceFiles = context.sourceCodeFiles?.size ?? 0;
|
|
137
|
+
return sourceFiles > 0 ? sourceFiles : 10;
|
|
138
|
+
};
|
|
139
|
+
/**
|
|
140
|
+
* Conformance assessor: fixed 7 conformance tests
|
|
141
|
+
*/
|
|
142
|
+
export const estimateConformanceTests = () => 7;
|
|
143
|
+
// ============================================================================
|
|
144
|
+
// Estimator Map (for dynamic lookup by assessor ID)
|
|
145
|
+
// ============================================================================
|
|
146
|
+
/**
|
|
147
|
+
* Map of assessor ID to test estimator function.
|
|
148
|
+
* Used by AssessorRegistry for dynamic test count estimation.
|
|
149
|
+
*/
|
|
150
|
+
export const ESTIMATOR_MAP = {
|
|
151
|
+
// Phase 0: PRE
|
|
152
|
+
temporal: estimateTemporalTests,
|
|
153
|
+
// Phase 1: CORE
|
|
154
|
+
functionality: estimateFunctionalityTests,
|
|
155
|
+
security: estimateSecurityTests,
|
|
156
|
+
documentation: estimateDocumentationTests,
|
|
157
|
+
errorHandling: estimateErrorHandlingTests,
|
|
158
|
+
usability: estimateUsabilityTests,
|
|
159
|
+
// Phase 2: PROTOCOL
|
|
160
|
+
protocolCompliance: estimateProtocolComplianceTests,
|
|
161
|
+
// Phase 3: COMPLIANCE
|
|
162
|
+
aupCompliance: estimateAUPComplianceTests,
|
|
163
|
+
toolAnnotations: estimateToolAnnotationTests,
|
|
164
|
+
prohibitedLibraries: estimateProhibitedLibrariesTests,
|
|
165
|
+
manifestValidation: estimateManifestValidationTests,
|
|
166
|
+
portability: estimatePortabilityTests,
|
|
167
|
+
externalAPIScanner: estimateExternalAPIScannerTests,
|
|
168
|
+
authentication: estimateAuthenticationTests,
|
|
169
|
+
// Phase 4: CAPABILITY
|
|
170
|
+
resources: estimateResourceTests,
|
|
171
|
+
prompts: estimatePromptTests,
|
|
172
|
+
crossCapability: estimateCrossCapabilityTests,
|
|
173
|
+
// Phase 5: QUALITY
|
|
174
|
+
fileModularization: estimateFileModularizationTests,
|
|
175
|
+
conformance: estimateConformanceTests,
|
|
176
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assessment Registry Module
|
|
3
|
+
*
|
|
4
|
+
* Exports all registry components for use by AssessmentOrchestrator.
|
|
5
|
+
*
|
|
6
|
+
* @module assessment/registry
|
|
7
|
+
* @see GitHub Issue #91
|
|
8
|
+
*/
|
|
9
|
+
export { AssessmentPhase, type AssessorDefinition, type AssessorConfigFlags, type AssessorConstructor, type AssessorSetupFn, type TestEstimatorFn, type RegisteredAssessor, type AssessorExecutionResult, type ClaudeBridgeCapable, supportsClaudeBridge, } from "./types.js";
|
|
10
|
+
export { ASSESSOR_DEFINITIONS, ASSESSOR_DEFINITION_MAP, getDefinitionsByPhase, getOrderedPhases, } from "./AssessorDefinitions.js";
|
|
11
|
+
export { AssessorRegistry, type FailedRegistration, } from "./AssessorRegistry.js";
|
|
12
|
+
export { estimateTemporalTests, estimateFunctionalityTests, estimateSecurityTests, estimateDocumentationTests, estimateErrorHandlingTests, estimateUsabilityTests, estimateProtocolComplianceTests, estimateAUPComplianceTests, estimateToolAnnotationTests, estimateProhibitedLibrariesTests, estimateManifestValidationTests, estimatePortabilityTests, estimateExternalAPIScannerTests, estimateAuthenticationTests, estimateResourceTests, estimatePromptTests, estimateCrossCapabilityTests, estimateFileModularizationTests, estimateConformanceTests, ESTIMATOR_MAP, } from "./estimators.js";
|
|
13
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/registry/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EACL,eAAe,EACf,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,kBAAkB,EACvB,KAAK,uBAAuB,EAC5B,KAAK,mBAAmB,EACxB,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAGjB,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,qBAAqB,EACrB,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EACL,gBAAgB,EAChB,KAAK,kBAAkB,GACxB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EACL,qBAAqB,EACrB,0BAA0B,EAC1B,qBAAqB,EACrB,0BAA0B,EAC1B,0BAA0B,EAC1B,sBAAsB,EACtB,+BAA+B,EAC/B,0BAA0B,EAC1B,2BAA2B,EAC3B,gCAAgC,EAChC,+BAA+B,EAC/B,wBAAwB,EACxB,+BAA+B,EAC/B,2BAA2B,EAC3B,qBAAqB,EACrB,mBAAmB,EACnB,4BAA4B,EAC5B,+BAA+B,EAC/B,wBAAwB,EACxB,aAAa,GACd,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assessment Registry Module
|
|
3
|
+
*
|
|
4
|
+
* Exports all registry components for use by AssessmentOrchestrator.
|
|
5
|
+
*
|
|
6
|
+
* @module assessment/registry
|
|
7
|
+
* @see GitHub Issue #91
|
|
8
|
+
*/
|
|
9
|
+
// Types
|
|
10
|
+
export { AssessmentPhase, supportsClaudeBridge, } from "./types.js";
|
|
11
|
+
// Definitions
|
|
12
|
+
export { ASSESSOR_DEFINITIONS, ASSESSOR_DEFINITION_MAP, getDefinitionsByPhase, getOrderedPhases, } from "./AssessorDefinitions.js";
|
|
13
|
+
// Registry
|
|
14
|
+
export { AssessorRegistry, } from "./AssessorRegistry.js";
|
|
15
|
+
// Estimators
|
|
16
|
+
export { estimateTemporalTests, estimateFunctionalityTests, estimateSecurityTests, estimateDocumentationTests, estimateErrorHandlingTests, estimateUsabilityTests, estimateProtocolComplianceTests, estimateAUPComplianceTests, estimateToolAnnotationTests, estimateProhibitedLibrariesTests, estimateManifestValidationTests, estimatePortabilityTests, estimateExternalAPIScannerTests, estimateAuthenticationTests, estimateResourceTests, estimatePromptTests, estimateCrossCapabilityTests, estimateFileModularizationTests, estimateConformanceTests, ESTIMATOR_MAP, } from "./estimators.js";
|