@bryan-thompson/inspector-assessment-client 1.32.0 → 1.32.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-Cl2ANLYP.js → OAuthCallback-CU-2TL5Q.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-Ws62T4Ns.js → OAuthDebugCallback-SiYZU1Sc.js} +1 -1
- package/dist/assets/{index-DaPIdOcS.js → index-DaMgI3Jj.js} +4 -4
- package/dist/index.html +1 -1
- package/lib/lib/assessment/resultTypes.d.ts +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +0 -28
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +13 -489
- package/lib/services/assessment/registry/AssessorDefinitions.d.ts.map +1 -1
- package/lib/services/assessment/registry/AssessorRegistry.d.ts +35 -2
- package/lib/services/assessment/registry/AssessorRegistry.d.ts.map +1 -1
- package/lib/services/assessment/registry/AssessorRegistry.js +82 -33
- package/lib/services/assessment/registry/index.d.ts +1 -1
- package/lib/services/assessment/registry/index.d.ts.map +1 -1
- package/lib/services/assessment/responseValidatorSchemas.d.ts +4 -4
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { u as useToast, r as reactExports, j as jsxRuntimeExports, p as parseOAuthCallbackParams, g as generateOAuthErrorDescription, S as SESSION_KEYS, I as InspectorOAuthClientProvider, a as auth } from "./index-
|
|
1
|
+
import { u as useToast, r as reactExports, j as jsxRuntimeExports, p as parseOAuthCallbackParams, g as generateOAuthErrorDescription, S as SESSION_KEYS, I as InspectorOAuthClientProvider, a as auth } from "./index-DaMgI3Jj.js";
|
|
2
2
|
const OAuthCallback = ({ onConnect }) => {
|
|
3
3
|
const { toast } = useToast();
|
|
4
4
|
const hasProcessedRef = reactExports.useRef(false);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { r as reactExports, S as SESSION_KEYS, p as parseOAuthCallbackParams, j as jsxRuntimeExports, g as generateOAuthErrorDescription } from "./index-
|
|
1
|
+
import { r as reactExports, S as SESSION_KEYS, p as parseOAuthCallbackParams, j as jsxRuntimeExports, g as generateOAuthErrorDescription } from "./index-DaMgI3Jj.js";
|
|
2
2
|
const OAuthDebugCallback = ({ onConnect }) => {
|
|
3
3
|
reactExports.useEffect(() => {
|
|
4
4
|
let isProcessed = false;
|
|
@@ -16373,7 +16373,7 @@ object({
|
|
|
16373
16373
|
token_type_hint: string().optional()
|
|
16374
16374
|
}).strip();
|
|
16375
16375
|
const name = "@bryan-thompson/inspector-assessment-client";
|
|
16376
|
-
const version$1 = "1.
|
|
16376
|
+
const version$1 = "1.32.2";
|
|
16377
16377
|
const packageJson = {
|
|
16378
16378
|
name,
|
|
16379
16379
|
version: version$1
|
|
@@ -48919,7 +48919,7 @@ const useTheme = () => {
|
|
|
48919
48919
|
[theme, setThemeWithSideEffect]
|
|
48920
48920
|
);
|
|
48921
48921
|
};
|
|
48922
|
-
const version = "1.
|
|
48922
|
+
const version = "1.32.2";
|
|
48923
48923
|
var [createTooltipContext] = createContextScope("Tooltip", [
|
|
48924
48924
|
createPopperScope
|
|
48925
48925
|
]);
|
|
@@ -52514,13 +52514,13 @@ const App = () => {
|
|
|
52514
52514
|
) });
|
|
52515
52515
|
if (window.location.pathname === "/oauth/callback") {
|
|
52516
52516
|
const OAuthCallback = React.lazy(
|
|
52517
|
-
() => __vitePreload(() => import("./OAuthCallback-
|
|
52517
|
+
() => __vitePreload(() => import("./OAuthCallback-CU-2TL5Q.js"), true ? [] : void 0)
|
|
52518
52518
|
);
|
|
52519
52519
|
return /* @__PURE__ */ jsxRuntimeExports.jsx(reactExports.Suspense, { fallback: /* @__PURE__ */ jsxRuntimeExports.jsx("div", { children: "Loading..." }), children: /* @__PURE__ */ jsxRuntimeExports.jsx(OAuthCallback, { onConnect: onOAuthConnect }) });
|
|
52520
52520
|
}
|
|
52521
52521
|
if (window.location.pathname === "/oauth/callback/debug") {
|
|
52522
52522
|
const OAuthDebugCallback = React.lazy(
|
|
52523
|
-
() => __vitePreload(() => import("./OAuthDebugCallback-
|
|
52523
|
+
() => __vitePreload(() => import("./OAuthDebugCallback-SiYZU1Sc.js"), true ? [] : void 0)
|
|
52524
52524
|
);
|
|
52525
52525
|
return /* @__PURE__ */ jsxRuntimeExports.jsx(reactExports.Suspense, { fallback: /* @__PURE__ */ jsxRuntimeExports.jsx("div", { children: "Loading..." }), children: /* @__PURE__ */ jsxRuntimeExports.jsx(OAuthDebugCallback, { onConnect: onOAuthDebugConnect }) });
|
|
52526
52526
|
}
|
package/dist/index.html
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
<link rel="icon" type="image/svg+xml" href="/mcp.svg" />
|
|
6
6
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
7
7
|
<title>MCP Inspector</title>
|
|
8
|
-
<script type="module" crossorigin src="/assets/index-
|
|
8
|
+
<script type="module" crossorigin src="/assets/index-DaMgI3Jj.js"></script>
|
|
9
9
|
<link rel="stylesheet" crossorigin href="/assets/index-BoUA5OL1.css">
|
|
10
10
|
</head>
|
|
11
11
|
<body>
|
|
@@ -688,7 +688,7 @@ export interface MCPDirectoryAssessment {
|
|
|
688
688
|
crossCapability?: CrossCapabilitySecurityAssessment;
|
|
689
689
|
/**
|
|
690
690
|
* MCP protocol-level compliance (error format, content types, initialization handshake)
|
|
691
|
-
* @deprecated
|
|
691
|
+
* @deprecated Use `protocolCompliance` instead. Merged with mcpSpecCompliance. Will be removed in v2.0.0.
|
|
692
692
|
*/
|
|
693
693
|
protocolConformance?: ProtocolConformanceAssessment;
|
|
694
694
|
/** File modularization assessment - detects large monolithic tool files (Issue #104) */
|
|
@@ -117,31 +117,7 @@ export declare class AssessmentOrchestrator {
|
|
|
117
117
|
private claudeBridge?;
|
|
118
118
|
private claudeEnabled;
|
|
119
119
|
private registry;
|
|
120
|
-
private get functionalityAssessor();
|
|
121
|
-
private get securityAssessor();
|
|
122
|
-
private get documentationAssessor();
|
|
123
|
-
private get errorHandlingAssessor();
|
|
124
|
-
private get usabilityAssessor();
|
|
125
|
-
private get protocolComplianceAssessor();
|
|
126
|
-
private get aupComplianceAssessor();
|
|
127
|
-
private get toolAnnotationAssessor();
|
|
128
|
-
private get prohibitedLibrariesAssessor();
|
|
129
|
-
private get manifestValidationAssessor();
|
|
130
|
-
private get portabilityAssessor();
|
|
131
|
-
private get externalAPIScannerAssessor();
|
|
132
|
-
private get temporalAssessor();
|
|
133
|
-
private get authenticationAssessor();
|
|
134
|
-
private get resourceAssessor();
|
|
135
|
-
private get promptAssessor();
|
|
136
|
-
private get crossCapabilityAssessor();
|
|
137
|
-
private get fileModularizationAssessor();
|
|
138
|
-
private get conformanceAssessor();
|
|
139
120
|
constructor(config?: Partial<AssessmentConfiguration>);
|
|
140
|
-
/**
|
|
141
|
-
* Get the count of tools that will actually be tested based on selectedToolsForTesting config.
|
|
142
|
-
* Used for accurate progress estimation in emitModuleStartedEvent calls.
|
|
143
|
-
*/
|
|
144
|
-
private getToolCountForTesting;
|
|
145
121
|
/**
|
|
146
122
|
* Initialize Claude Code Bridge for intelligent analysis
|
|
147
123
|
* This enables semantic AUP violation analysis, behavior inference, and intelligent test generation
|
|
@@ -163,10 +139,6 @@ export declare class AssessmentOrchestrator {
|
|
|
163
139
|
* @public
|
|
164
140
|
*/
|
|
165
141
|
getClaudeBridge(): ClaudeCodeBridge | undefined;
|
|
166
|
-
/**
|
|
167
|
-
* Reset test counts for all assessors
|
|
168
|
-
*/
|
|
169
|
-
private resetAllTestCounts;
|
|
170
142
|
/**
|
|
171
143
|
* Run a complete assessment on an MCP server
|
|
172
144
|
* @public
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AssessmentOrchestrator.d.ts","sourceRoot":"","sources":["../../../src/services/assessment/AssessmentOrchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,sBAAsB,EACtB,uBAAuB,EAEvB,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,EACX,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,IAAI,EACJ,2BAA2B,EAC5B,MAAM,oCAAoC,CAAC;
|
|
1
|
+
{"version":3,"file":"AssessmentOrchestrator.d.ts","sourceRoot":"","sources":["../../../src/services/assessment/AssessmentOrchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,sBAAsB,EACtB,uBAAuB,EAEvB,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,EACX,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,IAAI,EACJ,2BAA2B,EAC5B,MAAM,oCAAoC,CAAC;AAK5C,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EAEvB,MAAM,wBAAwB,CAAC;AAsBhC;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,KAAK,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC,CAAC;CACJ;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,KAAK,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IAClC,SAAS,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IAC3D,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC;AAED;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC5B,OAAO,CAAC,2BAA2B,CAAC,CAAC;IAC1C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,uBAAuB,CAAC;IAChC,UAAU,CAAC,EAAE,UAAU,CAAC;IAIxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAGtC,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAClC,WAAW,CAAC,EAAE,MAAM,CAAC;IAIrB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAG9B,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;IAC1B,iBAAiB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC1C,OAAO,CAAC,EAAE,SAAS,EAAE,CAAC;IACtB,kBAAkB,CAAC,EAAE,qBAAqB,CAAC;IAG3C,YAAY,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAChD,SAAS,CAAC,EAAE,CACV,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KACzB,OAAO,CAAC;QAAE,QAAQ,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC,CAAC;IAGrE,eAAe,CAAC,EAAE;QAChB,IAAI,EAAE,OAAO,GAAG,KAAK,GAAG,iBAAiB,CAAC;QAC1C,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,CAAC;IAIF,SAAS,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;CACnC;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,aAAa,CAAa;IAGlC,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,aAAa,CAAkB;IAIvC,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,MAAM,GAAE,OAAO,CAAC,uBAAuB,CAAM;IAiDzD;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAe9B;;;;OAIG;IACH,gBAAgB,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,sBAAsB,CAAC,GAAG,IAAI;IAgBhE;;;OAGG;IACH,eAAe,IAAI,OAAO;IAI1B;;;OAGG;IACH,eAAe,IAAI,gBAAgB,GAAG,SAAS;IAI/C;;;OAGG;IACG,iBAAiB,CACrB,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,sBAAsB,CAAC;IAiHlC;;;;OAIG;IACG,MAAM,CACV,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,IAAI,EAAE,EACb,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC5B,OAAO,CAAC,2BAA2B,CAAC,EACzC,UAAU,CAAC,EAAE,UAAU,EACvB,aAAa,CAAC,EAAE,MAAM,EACtB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACpC,OAAO,CAAC,sBAAsB,CAAC;IAclC,OAAO,CAAC,qBAAqB;IAO7B;;;OAGG;IACH,SAAS,IAAI,uBAAuB;IAIpC;;;OAGG;IACH,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,uBAAuB,CAAC,GAAG,IAAI;CAG7D"}
|
|
@@ -6,18 +6,16 @@
|
|
|
6
6
|
* @module AssessmentOrchestrator
|
|
7
7
|
*/
|
|
8
8
|
import { DEFAULT_ASSESSMENT_CONFIG, } from "../../lib/assessmentTypes.js";
|
|
9
|
-
// Note:
|
|
10
|
-
// Pattern configuration for tool annotation assessment now handled by registry (Issue #91)
|
|
11
|
-
// See AssessorDefinitions.ts customSetup for ToolAnnotationAssessor
|
|
9
|
+
// Note: All assessor module imports now handled by registry/AssessorDefinitions.ts (Issue #91)
|
|
12
10
|
// Claude Code integration for intelligent analysis
|
|
13
11
|
import { ClaudeCodeBridge, FULL_CLAUDE_CODE_CONFIG, } from "./lib/claudeCodeBridge.js";
|
|
14
12
|
import { TestDataGenerator } from "./TestDataGenerator.js";
|
|
15
13
|
// Structured logging
|
|
16
14
|
import { createLogger, DEFAULT_LOGGING_CONFIG } from "./lib/logger.js";
|
|
17
15
|
// Extracted helpers for testability
|
|
18
|
-
import {
|
|
16
|
+
import { determineOverallStatus, generateSummary, generateRecommendations, } from "./orchestratorHelpers.js";
|
|
19
17
|
// Registry pattern for assessor management (Issue #91)
|
|
20
|
-
import { AssessorRegistry, ASSESSOR_DEFINITIONS
|
|
18
|
+
import { AssessorRegistry, ASSESSOR_DEFINITIONS } from "./registry/index.js";
|
|
21
19
|
// Module scoring for dual-key output (Issue #124)
|
|
22
20
|
import { calculateModuleScore } from "../../lib/moduleScoring.js";
|
|
23
21
|
/**
|
|
@@ -43,69 +41,6 @@ export class AssessmentOrchestrator {
|
|
|
43
41
|
// Registry for assessor management (Issue #91)
|
|
44
42
|
// Delegates construction, test count aggregation, and Claude bridge wiring
|
|
45
43
|
registry;
|
|
46
|
-
// ============================================================================
|
|
47
|
-
// Private getters for backward compatibility with tests
|
|
48
|
-
// Tests access these via type assertions: (orchestrator as any).assessorName
|
|
49
|
-
// These delegate to the registry to maintain a single source of truth
|
|
50
|
-
// ============================================================================
|
|
51
|
-
get functionalityAssessor() {
|
|
52
|
-
return this.registry.getAssessor("functionality");
|
|
53
|
-
}
|
|
54
|
-
get securityAssessor() {
|
|
55
|
-
return this.registry.getAssessor("security");
|
|
56
|
-
}
|
|
57
|
-
get documentationAssessor() {
|
|
58
|
-
return this.registry.getAssessor("documentation");
|
|
59
|
-
}
|
|
60
|
-
get errorHandlingAssessor() {
|
|
61
|
-
return this.registry.getAssessor("errorHandling");
|
|
62
|
-
}
|
|
63
|
-
get usabilityAssessor() {
|
|
64
|
-
return this.registry.getAssessor("usability");
|
|
65
|
-
}
|
|
66
|
-
get protocolComplianceAssessor() {
|
|
67
|
-
return this.registry.getAssessor("protocolCompliance");
|
|
68
|
-
}
|
|
69
|
-
get aupComplianceAssessor() {
|
|
70
|
-
return this.registry.getAssessor("aupCompliance");
|
|
71
|
-
}
|
|
72
|
-
get toolAnnotationAssessor() {
|
|
73
|
-
return this.registry.getAssessor("toolAnnotations");
|
|
74
|
-
}
|
|
75
|
-
get prohibitedLibrariesAssessor() {
|
|
76
|
-
return this.registry.getAssessor("prohibitedLibraries");
|
|
77
|
-
}
|
|
78
|
-
get manifestValidationAssessor() {
|
|
79
|
-
return this.registry.getAssessor("manifestValidation");
|
|
80
|
-
}
|
|
81
|
-
get portabilityAssessor() {
|
|
82
|
-
return this.registry.getAssessor("portability");
|
|
83
|
-
}
|
|
84
|
-
get externalAPIScannerAssessor() {
|
|
85
|
-
return this.registry.getAssessor("externalAPIScanner");
|
|
86
|
-
}
|
|
87
|
-
get temporalAssessor() {
|
|
88
|
-
return this.registry.getAssessor("temporal");
|
|
89
|
-
}
|
|
90
|
-
get authenticationAssessor() {
|
|
91
|
-
return this.registry.getAssessor("authentication");
|
|
92
|
-
}
|
|
93
|
-
get resourceAssessor() {
|
|
94
|
-
return this.registry.getAssessor("resources");
|
|
95
|
-
}
|
|
96
|
-
get promptAssessor() {
|
|
97
|
-
return this.registry.getAssessor("prompts");
|
|
98
|
-
}
|
|
99
|
-
get crossCapabilityAssessor() {
|
|
100
|
-
return this.registry.getAssessor("crossCapability");
|
|
101
|
-
}
|
|
102
|
-
get fileModularizationAssessor() {
|
|
103
|
-
return this.registry.getAssessor("fileModularization");
|
|
104
|
-
}
|
|
105
|
-
get conformanceAssessor() {
|
|
106
|
-
return this.registry.getAssessor("conformance");
|
|
107
|
-
}
|
|
108
|
-
// Note: protocolConformanceAssessor merged into protocolComplianceAssessor (v1.25.2)
|
|
109
44
|
constructor(config = {}) {
|
|
110
45
|
this.config = { ...DEFAULT_ASSESSMENT_CONFIG, ...config };
|
|
111
46
|
// Initialize logger
|
|
@@ -139,17 +74,6 @@ export class AssessmentOrchestrator {
|
|
|
139
74
|
// Set logger for TestDataGenerator diagnostic output
|
|
140
75
|
TestDataGenerator.setLogger(this.logger);
|
|
141
76
|
}
|
|
142
|
-
/**
|
|
143
|
-
* Get the count of tools that will actually be tested based on selectedToolsForTesting config.
|
|
144
|
-
* Used for accurate progress estimation in emitModuleStartedEvent calls.
|
|
145
|
-
*/
|
|
146
|
-
getToolCountForTesting(tools) {
|
|
147
|
-
if (this.config.selectedToolsForTesting !== undefined) {
|
|
148
|
-
const selectedNames = new Set(this.config.selectedToolsForTesting);
|
|
149
|
-
return tools.filter((tool) => selectedNames.has(tool.name)).length;
|
|
150
|
-
}
|
|
151
|
-
return tools.length;
|
|
152
|
-
}
|
|
153
77
|
/**
|
|
154
78
|
* Initialize Claude Code Bridge for intelligent analysis
|
|
155
79
|
* This enables semantic AUP violation analysis, behavior inference, and intelligent test generation
|
|
@@ -201,55 +125,6 @@ export class AssessmentOrchestrator {
|
|
|
201
125
|
getClaudeBridge() {
|
|
202
126
|
return this.claudeBridge;
|
|
203
127
|
}
|
|
204
|
-
/**
|
|
205
|
-
* Reset test counts for all assessors
|
|
206
|
-
*/
|
|
207
|
-
resetAllTestCounts() {
|
|
208
|
-
this.functionalityAssessor?.resetTestCount();
|
|
209
|
-
this.securityAssessor?.resetTestCount();
|
|
210
|
-
this.documentationAssessor?.resetTestCount();
|
|
211
|
-
this.errorHandlingAssessor?.resetTestCount();
|
|
212
|
-
this.usabilityAssessor?.resetTestCount();
|
|
213
|
-
if (this.protocolComplianceAssessor) {
|
|
214
|
-
this.protocolComplianceAssessor.resetTestCount();
|
|
215
|
-
}
|
|
216
|
-
// Reset new assessors
|
|
217
|
-
if (this.aupComplianceAssessor) {
|
|
218
|
-
this.aupComplianceAssessor.resetTestCount();
|
|
219
|
-
}
|
|
220
|
-
if (this.toolAnnotationAssessor) {
|
|
221
|
-
this.toolAnnotationAssessor.resetTestCount();
|
|
222
|
-
}
|
|
223
|
-
if (this.prohibitedLibrariesAssessor) {
|
|
224
|
-
this.prohibitedLibrariesAssessor.resetTestCount();
|
|
225
|
-
}
|
|
226
|
-
if (this.manifestValidationAssessor) {
|
|
227
|
-
this.manifestValidationAssessor.resetTestCount();
|
|
228
|
-
}
|
|
229
|
-
if (this.portabilityAssessor) {
|
|
230
|
-
this.portabilityAssessor.resetTestCount();
|
|
231
|
-
}
|
|
232
|
-
if (this.authenticationAssessor) {
|
|
233
|
-
this.authenticationAssessor.resetTestCount();
|
|
234
|
-
}
|
|
235
|
-
// Reset new capability assessors
|
|
236
|
-
if (this.resourceAssessor) {
|
|
237
|
-
this.resourceAssessor.resetTestCount();
|
|
238
|
-
}
|
|
239
|
-
if (this.promptAssessor) {
|
|
240
|
-
this.promptAssessor.resetTestCount();
|
|
241
|
-
}
|
|
242
|
-
if (this.crossCapabilityAssessor) {
|
|
243
|
-
this.crossCapabilityAssessor.resetTestCount();
|
|
244
|
-
}
|
|
245
|
-
if (this.fileModularizationAssessor) {
|
|
246
|
-
this.fileModularizationAssessor.resetTestCount();
|
|
247
|
-
}
|
|
248
|
-
// Reset official conformance assessor
|
|
249
|
-
if (this.conformanceAssessor) {
|
|
250
|
-
this.conformanceAssessor.resetTestCount();
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
128
|
/**
|
|
254
129
|
* Run a complete assessment on an MCP server
|
|
255
130
|
* @public
|
|
@@ -257,298 +132,14 @@ export class AssessmentOrchestrator {
|
|
|
257
132
|
async runFullAssessment(context) {
|
|
258
133
|
this.startTime = Date.now();
|
|
259
134
|
this.totalTestsRun = 0;
|
|
260
|
-
this.resetAllTestCounts();
|
|
261
|
-
//
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
//
|
|
265
|
-
//
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
const invocationsPerTool = this.config.temporalInvocations ?? 25;
|
|
269
|
-
emitModuleStartedEvent("Temporal", toolCount * invocationsPerTool, toolCount);
|
|
270
|
-
assessmentResults.temporal = await this.temporalAssessor.assess(context);
|
|
271
|
-
emitModuleProgress("Temporal", assessmentResults.temporal.status, assessmentResults.temporal, this.temporalAssessor.getTestCount());
|
|
272
|
-
}
|
|
273
|
-
if (this.config.parallelTesting) {
|
|
274
|
-
// Calculate estimates for module_started events
|
|
275
|
-
const toolCount = this.getToolCountForTesting(context.tools);
|
|
276
|
-
const securityPatterns = this.config.securityPatternsToTest || 17;
|
|
277
|
-
// Core assessments - only emit and run if not skipped
|
|
278
|
-
if (this.functionalityAssessor) {
|
|
279
|
-
emitModuleStartedEvent("Functionality", toolCount * 10, toolCount);
|
|
280
|
-
assessmentPromises.push(this.functionalityAssessor.assess(context).then((r) => {
|
|
281
|
-
emitModuleProgress("Functionality", r.status, r, this.functionalityAssessor.getTestCount());
|
|
282
|
-
return (assessmentResults.functionality = r);
|
|
283
|
-
}));
|
|
284
|
-
}
|
|
285
|
-
if (this.securityAssessor) {
|
|
286
|
-
emitModuleStartedEvent("Security", securityPatterns * toolCount, toolCount);
|
|
287
|
-
assessmentPromises.push(this.securityAssessor.assess(context).then((r) => {
|
|
288
|
-
emitModuleProgress("Security", r.status, r, this.securityAssessor.getTestCount());
|
|
289
|
-
return (assessmentResults.security = r);
|
|
290
|
-
}));
|
|
291
|
-
}
|
|
292
|
-
if (this.documentationAssessor) {
|
|
293
|
-
emitModuleStartedEvent("Documentation", 5, toolCount);
|
|
294
|
-
assessmentPromises.push(this.documentationAssessor.assess(context).then((r) => {
|
|
295
|
-
emitModuleProgress("Documentation", r.status, r, this.documentationAssessor.getTestCount());
|
|
296
|
-
return (assessmentResults.documentation = r);
|
|
297
|
-
}));
|
|
298
|
-
}
|
|
299
|
-
if (this.errorHandlingAssessor) {
|
|
300
|
-
emitModuleStartedEvent("Error Handling", toolCount * 5, toolCount);
|
|
301
|
-
assessmentPromises.push(this.errorHandlingAssessor.assess(context).then((r) => {
|
|
302
|
-
emitModuleProgress("Error Handling", r.status, r, this.errorHandlingAssessor.getTestCount());
|
|
303
|
-
return (assessmentResults.errorHandling = r);
|
|
304
|
-
}));
|
|
305
|
-
}
|
|
306
|
-
if (this.usabilityAssessor) {
|
|
307
|
-
emitModuleStartedEvent("Usability", 10, toolCount);
|
|
308
|
-
assessmentPromises.push(this.usabilityAssessor.assess(context).then((r) => {
|
|
309
|
-
emitModuleProgress("Usability", r.status, r, this.usabilityAssessor.getTestCount());
|
|
310
|
-
return (assessmentResults.usability = r);
|
|
311
|
-
}));
|
|
312
|
-
}
|
|
313
|
-
// Extended assessments - unified protocol compliance
|
|
314
|
-
if (this.protocolComplianceAssessor) {
|
|
315
|
-
emitModuleStartedEvent("Protocol Compliance", 10, toolCount);
|
|
316
|
-
assessmentPromises.push(this.protocolComplianceAssessor.assess(context).then((r) => {
|
|
317
|
-
emitModuleProgress("Protocol Compliance", r.status, r, this.protocolComplianceAssessor.getTestCount());
|
|
318
|
-
return (assessmentResults.mcpSpecCompliance = r);
|
|
319
|
-
}));
|
|
320
|
-
}
|
|
321
|
-
// New MCP Directory Compliance Gap assessments
|
|
322
|
-
if (this.aupComplianceAssessor) {
|
|
323
|
-
emitModuleStartedEvent("AUP", 20, toolCount);
|
|
324
|
-
assessmentPromises.push(this.aupComplianceAssessor.assess(context).then((r) => {
|
|
325
|
-
emitModuleProgress("AUP", r.status, r, this.aupComplianceAssessor.getTestCount());
|
|
326
|
-
return (assessmentResults.aupCompliance = r);
|
|
327
|
-
}));
|
|
328
|
-
}
|
|
329
|
-
if (this.toolAnnotationAssessor) {
|
|
330
|
-
emitModuleStartedEvent("Annotations", toolCount, toolCount);
|
|
331
|
-
assessmentPromises.push(this.toolAnnotationAssessor.assess(context).then((r) => {
|
|
332
|
-
emitModuleProgress("Annotations", r.status, r, this.toolAnnotationAssessor.getTestCount());
|
|
333
|
-
return (assessmentResults.toolAnnotations = r);
|
|
334
|
-
}));
|
|
335
|
-
}
|
|
336
|
-
if (this.prohibitedLibrariesAssessor) {
|
|
337
|
-
emitModuleStartedEvent("Libraries", 5, toolCount);
|
|
338
|
-
assessmentPromises.push(this.prohibitedLibrariesAssessor.assess(context).then((r) => {
|
|
339
|
-
emitModuleProgress("Libraries", r.status, r, this.prohibitedLibrariesAssessor.getTestCount());
|
|
340
|
-
return (assessmentResults.prohibitedLibraries = r);
|
|
341
|
-
}));
|
|
342
|
-
}
|
|
343
|
-
if (this.manifestValidationAssessor) {
|
|
344
|
-
emitModuleStartedEvent("Manifest", 10, toolCount);
|
|
345
|
-
assessmentPromises.push(this.manifestValidationAssessor.assess(context).then((r) => {
|
|
346
|
-
emitModuleProgress("Manifest", r.status, r, this.manifestValidationAssessor.getTestCount());
|
|
347
|
-
return (assessmentResults.manifestValidation = r);
|
|
348
|
-
}));
|
|
349
|
-
}
|
|
350
|
-
if (this.portabilityAssessor) {
|
|
351
|
-
emitModuleStartedEvent("Portability", 10, toolCount);
|
|
352
|
-
assessmentPromises.push(this.portabilityAssessor.assess(context).then((r) => {
|
|
353
|
-
emitModuleProgress("Portability", r.status, r, this.portabilityAssessor.getTestCount());
|
|
354
|
-
return (assessmentResults.portability = r);
|
|
355
|
-
}));
|
|
356
|
-
}
|
|
357
|
-
if (this.externalAPIScannerAssessor) {
|
|
358
|
-
emitModuleStartedEvent("External APIs", 10, toolCount);
|
|
359
|
-
assessmentPromises.push(this.externalAPIScannerAssessor.assess(context).then((r) => {
|
|
360
|
-
emitModuleProgress("External APIs", r.status, r, this.externalAPIScannerAssessor.getTestCount());
|
|
361
|
-
return (assessmentResults.externalAPIScanner = r);
|
|
362
|
-
}));
|
|
363
|
-
}
|
|
364
|
-
if (this.authenticationAssessor) {
|
|
365
|
-
const sourceFileCount = context.sourceCodeFiles?.size || 0;
|
|
366
|
-
emitModuleStartedEvent("Authentication", sourceFileCount, sourceFileCount);
|
|
367
|
-
assessmentPromises.push(this.authenticationAssessor.assess(context).then((r) => {
|
|
368
|
-
emitModuleProgress("Authentication", r.status, r, this.authenticationAssessor.getTestCount());
|
|
369
|
-
return (assessmentResults.authentication = r);
|
|
370
|
-
}));
|
|
371
|
-
}
|
|
372
|
-
// NOTE: Temporal runs in PHASE 0 above, not in parallel with other modules
|
|
373
|
-
// New capability assessors
|
|
374
|
-
if (this.resourceAssessor) {
|
|
375
|
-
const resourceCount = (context.resources?.length || 0) +
|
|
376
|
-
(context.resourceTemplates?.length || 0);
|
|
377
|
-
emitModuleStartedEvent("Resources", resourceCount * 5, resourceCount);
|
|
378
|
-
assessmentPromises.push(this.resourceAssessor.assess(context).then((r) => {
|
|
379
|
-
emitModuleProgress("Resources", r.status, r, this.resourceAssessor.getTestCount());
|
|
380
|
-
return (assessmentResults.resources = r);
|
|
381
|
-
}));
|
|
382
|
-
}
|
|
383
|
-
if (this.promptAssessor) {
|
|
384
|
-
const promptCount = context.prompts?.length || 0;
|
|
385
|
-
emitModuleStartedEvent("Prompts", promptCount * 10, promptCount);
|
|
386
|
-
assessmentPromises.push(this.promptAssessor.assess(context).then((r) => {
|
|
387
|
-
emitModuleProgress("Prompts", r.status, r, this.promptAssessor.getTestCount());
|
|
388
|
-
return (assessmentResults.prompts = r);
|
|
389
|
-
}));
|
|
390
|
-
}
|
|
391
|
-
if (this.crossCapabilityAssessor) {
|
|
392
|
-
const capabilityCount = toolCount +
|
|
393
|
-
(context.resources?.length || 0) +
|
|
394
|
-
(context.prompts?.length || 0);
|
|
395
|
-
emitModuleStartedEvent("Cross-Capability", capabilityCount * 3, capabilityCount);
|
|
396
|
-
assessmentPromises.push(this.crossCapabilityAssessor.assess(context).then((r) => {
|
|
397
|
-
emitModuleProgress("Cross-Capability", r.status, r, this.crossCapabilityAssessor.getTestCount());
|
|
398
|
-
return (assessmentResults.crossCapability = r);
|
|
399
|
-
}));
|
|
400
|
-
}
|
|
401
|
-
if (this.fileModularizationAssessor) {
|
|
402
|
-
const sourceFileCount = context.sourceCodeFiles?.size || 0;
|
|
403
|
-
emitModuleStartedEvent("File Modularization", sourceFileCount, sourceFileCount);
|
|
404
|
-
assessmentPromises.push(this.fileModularizationAssessor.assess(context).then((r) => {
|
|
405
|
-
emitModuleProgress("File Modularization", r.status, r, this.fileModularizationAssessor.getTestCount());
|
|
406
|
-
return (assessmentResults.fileModularization = r);
|
|
407
|
-
}));
|
|
408
|
-
}
|
|
409
|
-
// Official MCP conformance testing (opt-in, requires HTTP/SSE transport)
|
|
410
|
-
if (this.conformanceAssessor) {
|
|
411
|
-
// Conformance tests ~7 server scenarios
|
|
412
|
-
emitModuleStartedEvent("Conformance", 7, toolCount);
|
|
413
|
-
assessmentPromises.push(this.conformanceAssessor.assess(context).then((r) => {
|
|
414
|
-
emitModuleProgress("Conformance", r.status, r, this.conformanceAssessor.getTestCount());
|
|
415
|
-
return (assessmentResults.conformance = r);
|
|
416
|
-
}));
|
|
417
|
-
}
|
|
418
|
-
// Note: Protocol Conformance now handled by unified ProtocolComplianceAssessor above
|
|
419
|
-
await Promise.all(assessmentPromises);
|
|
420
|
-
}
|
|
421
|
-
else {
|
|
422
|
-
// Sequential execution with module_started events
|
|
423
|
-
const toolCount = this.getToolCountForTesting(context.tools);
|
|
424
|
-
const securityPatterns = this.config.securityPatternsToTest || 17;
|
|
425
|
-
// NOTE: Temporal runs in PHASE 0 above, before sequential/parallel phases
|
|
426
|
-
// Core assessments - only emit and run if not skipped
|
|
427
|
-
if (this.functionalityAssessor) {
|
|
428
|
-
// Functionality: ~10 scenarios per tool
|
|
429
|
-
emitModuleStartedEvent("Functionality", toolCount * 10, toolCount);
|
|
430
|
-
assessmentResults.functionality =
|
|
431
|
-
await this.functionalityAssessor.assess(context);
|
|
432
|
-
emitModuleProgress("Functionality", assessmentResults.functionality.status, assessmentResults.functionality, this.functionalityAssessor.getTestCount());
|
|
433
|
-
}
|
|
434
|
-
if (this.securityAssessor) {
|
|
435
|
-
// Security: patterns × tools
|
|
436
|
-
emitModuleStartedEvent("Security", securityPatterns * toolCount, toolCount);
|
|
437
|
-
assessmentResults.security =
|
|
438
|
-
await this.securityAssessor.assess(context);
|
|
439
|
-
emitModuleProgress("Security", assessmentResults.security.status, assessmentResults.security, this.securityAssessor.getTestCount());
|
|
440
|
-
}
|
|
441
|
-
if (this.documentationAssessor) {
|
|
442
|
-
// Documentation: ~5 static tests
|
|
443
|
-
emitModuleStartedEvent("Documentation", 5, toolCount);
|
|
444
|
-
assessmentResults.documentation =
|
|
445
|
-
await this.documentationAssessor.assess(context);
|
|
446
|
-
emitModuleProgress("Documentation", assessmentResults.documentation.status, assessmentResults.documentation, this.documentationAssessor.getTestCount());
|
|
447
|
-
}
|
|
448
|
-
if (this.errorHandlingAssessor) {
|
|
449
|
-
// Error Handling: ~5 tests per tool
|
|
450
|
-
emitModuleStartedEvent("Error Handling", toolCount * 5, toolCount);
|
|
451
|
-
assessmentResults.errorHandling =
|
|
452
|
-
await this.errorHandlingAssessor.assess(context);
|
|
453
|
-
emitModuleProgress("Error Handling", assessmentResults.errorHandling.status, assessmentResults.errorHandling, this.errorHandlingAssessor.getTestCount());
|
|
454
|
-
}
|
|
455
|
-
if (this.usabilityAssessor) {
|
|
456
|
-
// Usability: ~10 static tests
|
|
457
|
-
emitModuleStartedEvent("Usability", 10, toolCount);
|
|
458
|
-
assessmentResults.usability =
|
|
459
|
-
await this.usabilityAssessor.assess(context);
|
|
460
|
-
emitModuleProgress("Usability", assessmentResults.usability.status, assessmentResults.usability, this.usabilityAssessor.getTestCount());
|
|
461
|
-
}
|
|
462
|
-
if (this.protocolComplianceAssessor) {
|
|
463
|
-
emitModuleStartedEvent("Protocol Compliance", 10, toolCount);
|
|
464
|
-
assessmentResults.mcpSpecCompliance =
|
|
465
|
-
await this.protocolComplianceAssessor.assess(context);
|
|
466
|
-
emitModuleProgress("Protocol Compliance", assessmentResults.mcpSpecCompliance.status, assessmentResults.mcpSpecCompliance, this.protocolComplianceAssessor.getTestCount());
|
|
467
|
-
}
|
|
468
|
-
// New MCP Directory Compliance Gap assessments (sequential)
|
|
469
|
-
if (this.aupComplianceAssessor) {
|
|
470
|
-
emitModuleStartedEvent("AUP", 20, toolCount);
|
|
471
|
-
assessmentResults.aupCompliance =
|
|
472
|
-
await this.aupComplianceAssessor.assess(context);
|
|
473
|
-
emitModuleProgress("AUP", assessmentResults.aupCompliance.status, assessmentResults.aupCompliance, this.aupComplianceAssessor.getTestCount());
|
|
474
|
-
}
|
|
475
|
-
if (this.toolAnnotationAssessor) {
|
|
476
|
-
emitModuleStartedEvent("Annotations", toolCount, toolCount);
|
|
477
|
-
assessmentResults.toolAnnotations =
|
|
478
|
-
await this.toolAnnotationAssessor.assess(context);
|
|
479
|
-
emitModuleProgress("Annotations", assessmentResults.toolAnnotations.status, assessmentResults.toolAnnotations, this.toolAnnotationAssessor.getTestCount());
|
|
480
|
-
}
|
|
481
|
-
if (this.prohibitedLibrariesAssessor) {
|
|
482
|
-
emitModuleStartedEvent("Libraries", 5, toolCount);
|
|
483
|
-
assessmentResults.prohibitedLibraries =
|
|
484
|
-
await this.prohibitedLibrariesAssessor.assess(context);
|
|
485
|
-
emitModuleProgress("Libraries", assessmentResults.prohibitedLibraries.status, assessmentResults.prohibitedLibraries, this.prohibitedLibrariesAssessor.getTestCount());
|
|
486
|
-
}
|
|
487
|
-
if (this.manifestValidationAssessor) {
|
|
488
|
-
emitModuleStartedEvent("Manifest", 10, toolCount);
|
|
489
|
-
assessmentResults.manifestValidation =
|
|
490
|
-
await this.manifestValidationAssessor.assess(context);
|
|
491
|
-
emitModuleProgress("Manifest", assessmentResults.manifestValidation.status, assessmentResults.manifestValidation, this.manifestValidationAssessor.getTestCount());
|
|
492
|
-
}
|
|
493
|
-
if (this.portabilityAssessor) {
|
|
494
|
-
emitModuleStartedEvent("Portability", 10, toolCount);
|
|
495
|
-
assessmentResults.portability =
|
|
496
|
-
await this.portabilityAssessor.assess(context);
|
|
497
|
-
emitModuleProgress("Portability", assessmentResults.portability.status, assessmentResults.portability, this.portabilityAssessor.getTestCount());
|
|
498
|
-
}
|
|
499
|
-
if (this.externalAPIScannerAssessor) {
|
|
500
|
-
emitModuleStartedEvent("External APIs", 10, toolCount);
|
|
501
|
-
assessmentResults.externalAPIScanner =
|
|
502
|
-
await this.externalAPIScannerAssessor.assess(context);
|
|
503
|
-
emitModuleProgress("External APIs", assessmentResults.externalAPIScanner.status, assessmentResults.externalAPIScanner, this.externalAPIScannerAssessor.getTestCount());
|
|
504
|
-
}
|
|
505
|
-
if (this.authenticationAssessor) {
|
|
506
|
-
const sourceFileCount = context.sourceCodeFiles?.size || 0;
|
|
507
|
-
emitModuleStartedEvent("Authentication", sourceFileCount, sourceFileCount);
|
|
508
|
-
assessmentResults.authentication =
|
|
509
|
-
await this.authenticationAssessor.assess(context);
|
|
510
|
-
emitModuleProgress("Authentication", assessmentResults.authentication.status, assessmentResults.authentication, this.authenticationAssessor.getTestCount());
|
|
511
|
-
}
|
|
512
|
-
// New capability assessors (sequential)
|
|
513
|
-
if (this.resourceAssessor) {
|
|
514
|
-
const resourceCount = (context.resources?.length || 0) +
|
|
515
|
-
(context.resourceTemplates?.length || 0);
|
|
516
|
-
emitModuleStartedEvent("Resources", resourceCount * 5, resourceCount);
|
|
517
|
-
assessmentResults.resources =
|
|
518
|
-
await this.resourceAssessor.assess(context);
|
|
519
|
-
emitModuleProgress("Resources", assessmentResults.resources.status, assessmentResults.resources, this.resourceAssessor.getTestCount());
|
|
520
|
-
}
|
|
521
|
-
if (this.promptAssessor) {
|
|
522
|
-
const promptCount = context.prompts?.length || 0;
|
|
523
|
-
emitModuleStartedEvent("Prompts", promptCount * 10, promptCount);
|
|
524
|
-
assessmentResults.prompts = await this.promptAssessor.assess(context);
|
|
525
|
-
emitModuleProgress("Prompts", assessmentResults.prompts.status, assessmentResults.prompts, this.promptAssessor.getTestCount());
|
|
526
|
-
}
|
|
527
|
-
if (this.crossCapabilityAssessor) {
|
|
528
|
-
const capabilityCount = toolCount +
|
|
529
|
-
(context.resources?.length || 0) +
|
|
530
|
-
(context.prompts?.length || 0);
|
|
531
|
-
emitModuleStartedEvent("Cross-Capability", capabilityCount * 3, capabilityCount);
|
|
532
|
-
assessmentResults.crossCapability =
|
|
533
|
-
await this.crossCapabilityAssessor.assess(context);
|
|
534
|
-
emitModuleProgress("Cross-Capability", assessmentResults.crossCapability.status, assessmentResults.crossCapability, this.crossCapabilityAssessor.getTestCount());
|
|
535
|
-
}
|
|
536
|
-
if (this.fileModularizationAssessor) {
|
|
537
|
-
const sourceFileCount = context.sourceCodeFiles?.size || 0;
|
|
538
|
-
emitModuleStartedEvent("File Modularization", sourceFileCount, sourceFileCount);
|
|
539
|
-
assessmentResults.fileModularization =
|
|
540
|
-
await this.fileModularizationAssessor.assess(context);
|
|
541
|
-
emitModuleProgress("File Modularization", assessmentResults.fileModularization.status, assessmentResults.fileModularization, this.fileModularizationAssessor.getTestCount());
|
|
542
|
-
}
|
|
543
|
-
// Official MCP conformance testing (sequential, opt-in)
|
|
544
|
-
if (this.conformanceAssessor) {
|
|
545
|
-
emitModuleStartedEvent("Conformance", 7, toolCount);
|
|
546
|
-
assessmentResults.conformance =
|
|
547
|
-
await this.conformanceAssessor.assess(context);
|
|
548
|
-
emitModuleProgress("Conformance", assessmentResults.conformance.status, assessmentResults.conformance, this.conformanceAssessor.getTestCount());
|
|
549
|
-
}
|
|
550
|
-
// Note: Protocol Conformance now handled by unified ProtocolComplianceAssessor above
|
|
551
|
-
}
|
|
135
|
+
this.registry.resetAllTestCounts();
|
|
136
|
+
// Execute all assessors via registry (Issue #91)
|
|
137
|
+
// Registry handles:
|
|
138
|
+
// - Phase-ordered execution (Phase 0/PRE always runs first and sequentially)
|
|
139
|
+
// - Parallel vs sequential based on config.parallelTesting
|
|
140
|
+
// - JSONL events (module_started, module_progress)
|
|
141
|
+
// - Test count tracking
|
|
142
|
+
const assessmentResults = await this.registry.executeAll(context);
|
|
552
143
|
// Integrate temporal findings into security.vulnerabilities for unified view
|
|
553
144
|
if (assessmentResults.temporal?.rugPullsDetected &&
|
|
554
145
|
assessmentResults.temporal.rugPullsDetected > 0 &&
|
|
@@ -640,75 +231,8 @@ export class AssessmentOrchestrator {
|
|
|
640
231
|
return this.runFullAssessment(context);
|
|
641
232
|
}
|
|
642
233
|
collectTotalTestCount() {
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
const functionalityCount = this.functionalityAssessor?.getTestCount() || 0;
|
|
646
|
-
const securityCount = this.securityAssessor?.getTestCount() || 0;
|
|
647
|
-
const documentationCount = this.documentationAssessor?.getTestCount() || 0;
|
|
648
|
-
const errorHandlingCount = this.errorHandlingAssessor?.getTestCount() || 0;
|
|
649
|
-
const usabilityCount = this.usabilityAssessor?.getTestCount() || 0;
|
|
650
|
-
const mcpSpecCount = this.protocolComplianceAssessor?.getTestCount() || 0;
|
|
651
|
-
// New assessor counts
|
|
652
|
-
const aupCount = this.aupComplianceAssessor?.getTestCount() || 0;
|
|
653
|
-
const annotationCount = this.toolAnnotationAssessor?.getTestCount() || 0;
|
|
654
|
-
const librariesCount = this.prohibitedLibrariesAssessor?.getTestCount() || 0;
|
|
655
|
-
const manifestCount = this.manifestValidationAssessor?.getTestCount() || 0;
|
|
656
|
-
const portabilityCount = this.portabilityAssessor?.getTestCount() || 0;
|
|
657
|
-
const authenticationCount = this.authenticationAssessor?.getTestCount() || 0;
|
|
658
|
-
const externalAPICount = this.externalAPIScannerAssessor?.getTestCount() || 0;
|
|
659
|
-
const temporalCount = this.temporalAssessor?.getTestCount() || 0;
|
|
660
|
-
// New capability assessor counts
|
|
661
|
-
const resourcesCount = this.resourceAssessor?.getTestCount() || 0;
|
|
662
|
-
const promptsCount = this.promptAssessor?.getTestCount() || 0;
|
|
663
|
-
const crossCapabilityCount = this.crossCapabilityAssessor?.getTestCount() || 0;
|
|
664
|
-
// Code quality assessor counts
|
|
665
|
-
const fileModularizationCount = this.fileModularizationAssessor?.getTestCount() || 0;
|
|
666
|
-
// Official MCP conformance test count
|
|
667
|
-
const conformanceCount = this.conformanceAssessor?.getTestCount() || 0;
|
|
668
|
-
// Note: Protocol conformance now included in mcpSpecCount (unified ProtocolComplianceAssessor)
|
|
669
|
-
this.logger.debug("Test counts by assessor", {
|
|
670
|
-
functionality: functionalityCount,
|
|
671
|
-
security: securityCount,
|
|
672
|
-
documentation: documentationCount,
|
|
673
|
-
errorHandling: errorHandlingCount,
|
|
674
|
-
usability: usabilityCount,
|
|
675
|
-
mcpSpec: mcpSpecCount,
|
|
676
|
-
aupCompliance: aupCount,
|
|
677
|
-
toolAnnotations: annotationCount,
|
|
678
|
-
prohibitedLibraries: librariesCount,
|
|
679
|
-
manifestValidation: manifestCount,
|
|
680
|
-
portability: portabilityCount,
|
|
681
|
-
authentication: authenticationCount,
|
|
682
|
-
externalAPIScanner: externalAPICount,
|
|
683
|
-
temporal: temporalCount,
|
|
684
|
-
resources: resourcesCount,
|
|
685
|
-
prompts: promptsCount,
|
|
686
|
-
crossCapability: crossCapabilityCount,
|
|
687
|
-
fileModularization: fileModularizationCount,
|
|
688
|
-
conformance: conformanceCount,
|
|
689
|
-
// Note: protocolConformance now included in mcpSpec (unified)
|
|
690
|
-
});
|
|
691
|
-
total =
|
|
692
|
-
functionalityCount +
|
|
693
|
-
securityCount +
|
|
694
|
-
documentationCount +
|
|
695
|
-
errorHandlingCount +
|
|
696
|
-
usabilityCount +
|
|
697
|
-
mcpSpecCount +
|
|
698
|
-
aupCount +
|
|
699
|
-
annotationCount +
|
|
700
|
-
librariesCount +
|
|
701
|
-
manifestCount +
|
|
702
|
-
portabilityCount +
|
|
703
|
-
authenticationCount +
|
|
704
|
-
externalAPICount +
|
|
705
|
-
temporalCount +
|
|
706
|
-
resourcesCount +
|
|
707
|
-
promptsCount +
|
|
708
|
-
crossCapabilityCount +
|
|
709
|
-
fileModularizationCount +
|
|
710
|
-
conformanceCount;
|
|
711
|
-
// Note: protocolConformance now included in mcpSpecCount (unified)
|
|
234
|
+
// Delegate to registry for centralized test count aggregation (Issue #91)
|
|
235
|
+
const total = this.registry.getTotalTestCount();
|
|
712
236
|
this.logger.debug("Total test count", { total });
|
|
713
237
|
return total;
|
|
714
238
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AssessorDefinitions.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/registry/AssessorDefinitions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AA4D1C;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,oBAAoB,EAAE,kBAAkB,
|
|
1
|
+
{"version":3,"file":"AssessorDefinitions.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/registry/AssessorDefinitions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AA4D1C;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,oBAAoB,EAAE,kBAAkB,EA+SpD,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,uBAAuB,EAAE,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAEnE,CAAC;AAEF;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,eAAe,GACrB,kBAAkB,EAAE,CAEtB;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,eAAe,EAAE,CASpD"}
|
|
@@ -23,11 +23,19 @@ import { type AssessorDefinition, type RegisteredAssessor, AssessmentPhase } fro
|
|
|
23
23
|
* 4. Test count aggregation from all assessors
|
|
24
24
|
* 5. Backward-compatible property access via getAssessor()
|
|
25
25
|
*/
|
|
26
|
+
/**
|
|
27
|
+
* Information about a failed assessor registration.
|
|
28
|
+
*/
|
|
29
|
+
export interface FailedRegistration {
|
|
30
|
+
id: string;
|
|
31
|
+
error: string;
|
|
32
|
+
}
|
|
26
33
|
export declare class AssessorRegistry {
|
|
27
34
|
private config;
|
|
28
35
|
private logger;
|
|
29
36
|
private assessors;
|
|
30
37
|
private claudeBridge?;
|
|
38
|
+
private failedRegistrations;
|
|
31
39
|
constructor(config: AssessmentConfiguration);
|
|
32
40
|
/**
|
|
33
41
|
* Register all enabled assessors based on configuration.
|
|
@@ -80,7 +88,8 @@ export declare class AssessorRegistry {
|
|
|
80
88
|
*/
|
|
81
89
|
private executePhase;
|
|
82
90
|
/**
|
|
83
|
-
* Execute assessors in parallel.
|
|
91
|
+
* Execute assessors in parallel with graceful degradation.
|
|
92
|
+
* Uses Promise.allSettled to continue execution even if some assessors fail.
|
|
84
93
|
*/
|
|
85
94
|
private executeParallel;
|
|
86
95
|
/**
|
|
@@ -117,8 +126,32 @@ export declare class AssessorRegistry {
|
|
|
117
126
|
*/
|
|
118
127
|
get size(): number;
|
|
119
128
|
/**
|
|
120
|
-
* Update configuration
|
|
129
|
+
* Update configuration for future operations.
|
|
130
|
+
*
|
|
131
|
+
* **Important**: This does NOT re-register assessors. Assessors are registered
|
|
132
|
+
* once during construction based on the initial config. To change which
|
|
133
|
+
* assessors are enabled, create a new AssessorRegistry instance.
|
|
134
|
+
*
|
|
135
|
+
* @param config - New configuration to use
|
|
121
136
|
*/
|
|
122
137
|
updateConfig(config: AssessmentConfiguration): void;
|
|
138
|
+
/**
|
|
139
|
+
* Reset test counts for all registered assessors.
|
|
140
|
+
* Called at the start of each assessment run.
|
|
141
|
+
*/
|
|
142
|
+
resetAllTestCounts(): void;
|
|
143
|
+
/**
|
|
144
|
+
* Get list of assessors that failed to register.
|
|
145
|
+
* Useful for reporting partial assessment results.
|
|
146
|
+
*
|
|
147
|
+
* @returns Array of failed registration info (id and error message)
|
|
148
|
+
*/
|
|
149
|
+
getFailedRegistrations(): FailedRegistration[];
|
|
150
|
+
/**
|
|
151
|
+
* Check if any assessors failed to register.
|
|
152
|
+
*
|
|
153
|
+
* @returns true if at least one assessor failed registration
|
|
154
|
+
*/
|
|
155
|
+
hasFailedRegistrations(): boolean;
|
|
123
156
|
}
|
|
124
157
|
//# sourceMappingURL=AssessorRegistry.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AssessorRegistry.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/registry/AssessorRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAC5E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,8BAA8B,CAAC;AAC3E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAEhE,OAAO,EACL,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EAEvB,eAAe,EAEhB,MAAM,SAAS,CAAC;AAOjB;;;;;;;;;GASG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAA8C;IAC/D,OAAO,CAAC,YAAY,CAAC,CAAmB;
|
|
1
|
+
{"version":3,"file":"AssessorRegistry.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/registry/AssessorRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAC5E,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,8BAA8B,CAAC;AAC3E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAEhE,OAAO,EACL,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EAEvB,eAAe,EAEhB,MAAM,SAAS,CAAC;AAOjB;;;;;;;;;GASG;AACH;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAA8C;IAC/D,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,mBAAmB,CAA4B;gBAE3C,MAAM,EAAE,uBAAuB;IAQ3C;;;OAGG;IACH,WAAW,CAAC,WAAW,GAAE,kBAAkB,EAAyB,GAAG,IAAI;IAc3E;;OAEG;IACH,OAAO,CAAC,QAAQ;IAmChB;;;OAGG;IACH,SAAS,CAAC,UAAU,EAAE,kBAAkB,GAAG,OAAO;IAmClD;;;;OAIG;IACH,OAAO,CAAC,SAAS;IAajB;;;OAGG;IACH,WAAW,CAAC,CAAC,SAAS,YAAY,GAAG,YAAY,EAC/C,EAAE,EAAE,MAAM,GACT,CAAC,GAAG,SAAS;IAKhB;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,eAAe,GAAG,kBAAkB,EAAE;IAMxD;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAU/C;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;;;;;OAMG;IACG,UAAU,CACd,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;IA+B3C;;OAEG;YACW,YAAY;IAoB1B;;;OAGG;YACW,eAAe;IAgE7B;;OAEG;YACW,iBAAiB;IAyD/B;;;OAGG;IACH,OAAO,CAAC,aAAa;IAOrB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAS9B;;OAEG;IACH,iBAAiB,IAAI,MAAM;IAQ3B;;OAEG;IACH,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAKhC;;OAEG;IACH,gBAAgB,IAAI,MAAM,EAAE;IAI5B;;OAEG;IACH,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAIjC;;OAEG;IACH,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;;;;OAQG;IACH,YAAY,CAAC,MAAM,EAAE,uBAAuB,GAAG,IAAI;IAInD;;;OAGG;IACH,kBAAkB,IAAI,IAAI;IAM1B;;;;;OAKG;IACH,sBAAsB,IAAI,kBAAkB,EAAE;IAI9C;;;;OAIG;IACH,sBAAsB,IAAI,OAAO;CAGlC"}
|
|
@@ -11,21 +11,12 @@ import { createLogger, DEFAULT_LOGGING_CONFIG } from "../lib/logger.js";
|
|
|
11
11
|
import { AssessmentPhase, supportsClaudeBridge, } from "./types.js";
|
|
12
12
|
import { ASSESSOR_DEFINITIONS, getOrderedPhases } from "./AssessorDefinitions.js";
|
|
13
13
|
import { emitModuleStartedEvent, emitModuleProgress, } from "../orchestratorHelpers.js";
|
|
14
|
-
/**
|
|
15
|
-
* AssessorRegistry manages assessor instances and their execution.
|
|
16
|
-
*
|
|
17
|
-
* Key responsibilities:
|
|
18
|
-
* 1. Lazy instantiation based on configuration flags
|
|
19
|
-
* 2. Phase-ordered execution with Phase 0 always first and sequential
|
|
20
|
-
* 3. Claude bridge wiring to supporting assessors
|
|
21
|
-
* 4. Test count aggregation from all assessors
|
|
22
|
-
* 5. Backward-compatible property access via getAssessor()
|
|
23
|
-
*/
|
|
24
14
|
export class AssessorRegistry {
|
|
25
15
|
config;
|
|
26
16
|
logger;
|
|
27
17
|
assessors = new Map();
|
|
28
18
|
claudeBridge;
|
|
19
|
+
failedRegistrations = [];
|
|
29
20
|
constructor(config) {
|
|
30
21
|
this.config = config;
|
|
31
22
|
this.logger = createLogger("AssessorRegistry", config.logging ?? DEFAULT_LOGGING_CONFIG);
|
|
@@ -73,6 +64,11 @@ export class AssessorRegistry {
|
|
|
73
64
|
this.logger.error(`Failed to register assessor: ${definition.id}`, {
|
|
74
65
|
error,
|
|
75
66
|
});
|
|
67
|
+
// Track failed registrations for summary reporting (P1 fix)
|
|
68
|
+
this.failedRegistrations.push({
|
|
69
|
+
id: definition.id,
|
|
70
|
+
error: error instanceof Error ? error.message : String(error),
|
|
71
|
+
});
|
|
76
72
|
}
|
|
77
73
|
}
|
|
78
74
|
/**
|
|
@@ -172,7 +168,8 @@ export class AssessorRegistry {
|
|
|
172
168
|
const phaseResults = await this.executePhase(phase, phaseAssessors, context, useParallel);
|
|
173
169
|
// Merge phase results into main results
|
|
174
170
|
for (const result of phaseResults) {
|
|
175
|
-
results[result.resultField] =
|
|
171
|
+
results[result.resultField] =
|
|
172
|
+
result.result;
|
|
176
173
|
}
|
|
177
174
|
}
|
|
178
175
|
return results;
|
|
@@ -192,7 +189,8 @@ export class AssessorRegistry {
|
|
|
192
189
|
}
|
|
193
190
|
}
|
|
194
191
|
/**
|
|
195
|
-
* Execute assessors in parallel.
|
|
192
|
+
* Execute assessors in parallel with graceful degradation.
|
|
193
|
+
* Uses Promise.allSettled to continue execution even if some assessors fail.
|
|
196
194
|
*/
|
|
197
195
|
async executeParallel(assessors, context, toolCount) {
|
|
198
196
|
const promises = assessors.map(async (registered) => {
|
|
@@ -202,26 +200,39 @@ export class AssessorRegistry {
|
|
|
202
200
|
emitModuleStartedEvent(definition.displayName, estimatedTests, toolCount);
|
|
203
201
|
// Execute
|
|
204
202
|
const startTime = Date.now();
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
203
|
+
const result = await instance.assess(context);
|
|
204
|
+
const executionTime = Date.now() - startTime;
|
|
205
|
+
// Emit progress event (writes to stderr)
|
|
206
|
+
const status = this.extractStatus(result);
|
|
207
|
+
emitModuleProgress(definition.displayName, status, result, instance.getTestCount());
|
|
208
|
+
return {
|
|
209
|
+
id: definition.id,
|
|
210
|
+
resultField: definition.resultField,
|
|
211
|
+
result,
|
|
212
|
+
executionTime,
|
|
213
|
+
};
|
|
214
|
+
});
|
|
215
|
+
// Use Promise.allSettled for graceful degradation (P1 fix)
|
|
216
|
+
const settledResults = await Promise.allSettled(promises);
|
|
217
|
+
const successResults = [];
|
|
218
|
+
for (let i = 0; i < settledResults.length; i++) {
|
|
219
|
+
const settledResult = settledResults[i];
|
|
220
|
+
if (settledResult.status === "fulfilled") {
|
|
221
|
+
successResults.push(settledResult.value);
|
|
218
222
|
}
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
223
|
+
else {
|
|
224
|
+
const definition = assessors[i].definition;
|
|
225
|
+
this.logger.error(`Assessor ${definition.id} failed during parallel execution`, {
|
|
226
|
+
error: settledResult.reason,
|
|
227
|
+
});
|
|
228
|
+
// Emit failure progress event
|
|
229
|
+
emitModuleProgress(definition.displayName, "ERROR", null, 0);
|
|
222
230
|
}
|
|
223
|
-
}
|
|
224
|
-
|
|
231
|
+
}
|
|
232
|
+
if (successResults.length < assessors.length) {
|
|
233
|
+
this.logger.warn(`${assessors.length - successResults.length} assessor(s) failed during parallel execution`);
|
|
234
|
+
}
|
|
235
|
+
return successResults;
|
|
225
236
|
}
|
|
226
237
|
/**
|
|
227
238
|
* Execute assessors sequentially.
|
|
@@ -250,10 +261,16 @@ export class AssessorRegistry {
|
|
|
250
261
|
});
|
|
251
262
|
}
|
|
252
263
|
catch (error) {
|
|
253
|
-
|
|
254
|
-
|
|
264
|
+
const executionTime = Date.now() - startTime;
|
|
265
|
+
this.logger.error(`Assessor ${definition.id} failed during sequential execution`, { error, executionTime });
|
|
266
|
+
// Emit failure progress event (consistent with parallel execution)
|
|
267
|
+
emitModuleProgress(definition.displayName, "ERROR", null, 0);
|
|
268
|
+
// Continue with remaining assessors (graceful degradation)
|
|
255
269
|
}
|
|
256
270
|
}
|
|
271
|
+
if (results.length < assessors.length) {
|
|
272
|
+
this.logger.warn(`${assessors.length - results.length} assessor(s) failed during sequential execution`);
|
|
273
|
+
}
|
|
257
274
|
return results;
|
|
258
275
|
}
|
|
259
276
|
/**
|
|
@@ -313,9 +330,41 @@ export class AssessorRegistry {
|
|
|
313
330
|
return this.assessors.size;
|
|
314
331
|
}
|
|
315
332
|
/**
|
|
316
|
-
* Update configuration
|
|
333
|
+
* Update configuration for future operations.
|
|
334
|
+
*
|
|
335
|
+
* **Important**: This does NOT re-register assessors. Assessors are registered
|
|
336
|
+
* once during construction based on the initial config. To change which
|
|
337
|
+
* assessors are enabled, create a new AssessorRegistry instance.
|
|
338
|
+
*
|
|
339
|
+
* @param config - New configuration to use
|
|
317
340
|
*/
|
|
318
341
|
updateConfig(config) {
|
|
319
342
|
this.config = config;
|
|
320
343
|
}
|
|
344
|
+
/**
|
|
345
|
+
* Reset test counts for all registered assessors.
|
|
346
|
+
* Called at the start of each assessment run.
|
|
347
|
+
*/
|
|
348
|
+
resetAllTestCounts() {
|
|
349
|
+
for (const registered of this.assessors.values()) {
|
|
350
|
+
registered.instance.resetTestCount();
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* Get list of assessors that failed to register.
|
|
355
|
+
* Useful for reporting partial assessment results.
|
|
356
|
+
*
|
|
357
|
+
* @returns Array of failed registration info (id and error message)
|
|
358
|
+
*/
|
|
359
|
+
getFailedRegistrations() {
|
|
360
|
+
return [...this.failedRegistrations];
|
|
361
|
+
}
|
|
362
|
+
/**
|
|
363
|
+
* Check if any assessors failed to register.
|
|
364
|
+
*
|
|
365
|
+
* @returns true if at least one assessor failed registration
|
|
366
|
+
*/
|
|
367
|
+
hasFailedRegistrations() {
|
|
368
|
+
return this.failedRegistrations.length > 0;
|
|
369
|
+
}
|
|
321
370
|
}
|
|
@@ -8,6 +8,6 @@
|
|
|
8
8
|
*/
|
|
9
9
|
export { AssessmentPhase, type AssessorDefinition, type AssessorConfigFlags, type AssessorConstructor, type AssessorSetupFn, type TestEstimatorFn, type RegisteredAssessor, type AssessorExecutionResult, type ClaudeBridgeCapable, supportsClaudeBridge, } from "./types.js";
|
|
10
10
|
export { ASSESSOR_DEFINITIONS, ASSESSOR_DEFINITION_MAP, getDefinitionsByPhase, getOrderedPhases, } from "./AssessorDefinitions.js";
|
|
11
|
-
export { AssessorRegistry } from "./AssessorRegistry.js";
|
|
11
|
+
export { AssessorRegistry, type FailedRegistration } from "./AssessorRegistry.js";
|
|
12
12
|
export { estimateTemporalTests, estimateFunctionalityTests, estimateSecurityTests, estimateDocumentationTests, estimateErrorHandlingTests, estimateUsabilityTests, estimateProtocolComplianceTests, estimateAUPComplianceTests, estimateToolAnnotationTests, estimateProhibitedLibrariesTests, estimateManifestValidationTests, estimatePortabilityTests, estimateExternalAPIScannerTests, estimateAuthenticationTests, estimateResourceTests, estimatePromptTests, estimateCrossCapabilityTests, estimateFileModularizationTests, estimateConformanceTests, ESTIMATOR_MAP, } from "./estimators.js";
|
|
13
13
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/registry/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EACL,eAAe,EACf,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,kBAAkB,EACvB,KAAK,uBAAuB,EAC5B,KAAK,mBAAmB,EACxB,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAGjB,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,qBAAqB,EACrB,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/registry/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EACL,eAAe,EACf,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,kBAAkB,EACvB,KAAK,uBAAuB,EAC5B,KAAK,mBAAmB,EACxB,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAGjB,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,qBAAqB,EACrB,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAAE,gBAAgB,EAAE,KAAK,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG/E,OAAO,EACL,qBAAqB,EACrB,0BAA0B,EAC1B,qBAAqB,EACrB,0BAA0B,EAC1B,0BAA0B,EAC1B,sBAAsB,EACtB,+BAA+B,EAC/B,0BAA0B,EAC1B,2BAA2B,EAC3B,gCAAgC,EAChC,+BAA+B,EAC/B,wBAAwB,EACxB,+BAA+B,EAC/B,2BAA2B,EAC3B,qBAAqB,EACrB,mBAAmB,EACnB,4BAA4B,EAC5B,+BAA+B,EAC/B,wBAAwB,EACxB,aAAa,GACd,MAAM,cAAc,CAAC"}
|
|
@@ -480,6 +480,7 @@ export declare const MCPToolCallResultSchema: z.ZodObject<{
|
|
|
480
480
|
structuredContent: z.ZodOptional<z.ZodUnknown>;
|
|
481
481
|
_meta: z.ZodOptional<z.ZodUnknown>;
|
|
482
482
|
}, "strip", z.ZodTypeAny, {
|
|
483
|
+
_meta?: unknown;
|
|
483
484
|
content?: ({
|
|
484
485
|
text?: string;
|
|
485
486
|
type?: "text";
|
|
@@ -503,9 +504,9 @@ export declare const MCPToolCallResultSchema: z.ZodObject<{
|
|
|
503
504
|
uri?: string;
|
|
504
505
|
})[];
|
|
505
506
|
isError?: boolean;
|
|
506
|
-
_meta?: unknown;
|
|
507
507
|
structuredContent?: unknown;
|
|
508
508
|
}, {
|
|
509
|
+
_meta?: unknown;
|
|
509
510
|
content?: ({
|
|
510
511
|
text?: string;
|
|
511
512
|
type?: "text";
|
|
@@ -529,7 +530,6 @@ export declare const MCPToolCallResultSchema: z.ZodObject<{
|
|
|
529
530
|
uri?: string;
|
|
530
531
|
})[];
|
|
531
532
|
isError?: boolean;
|
|
532
|
-
_meta?: unknown;
|
|
533
533
|
structuredContent?: unknown;
|
|
534
534
|
}>;
|
|
535
535
|
export type MCPToolCallResultParsed = z.infer<typeof MCPToolCallResultSchema>;
|
|
@@ -596,6 +596,7 @@ export declare function safeParseContentArray(content: unknown): z.SafeParseRetu
|
|
|
596
596
|
* @returns SafeParseResult with success status and data/error
|
|
597
597
|
*/
|
|
598
598
|
export declare function safeParseMCPToolCallResult(result: unknown): z.SafeParseReturnType<{
|
|
599
|
+
_meta?: unknown;
|
|
599
600
|
content?: ({
|
|
600
601
|
text?: string;
|
|
601
602
|
type?: "text";
|
|
@@ -619,9 +620,9 @@ export declare function safeParseMCPToolCallResult(result: unknown): z.SafeParse
|
|
|
619
620
|
uri?: string;
|
|
620
621
|
})[];
|
|
621
622
|
isError?: boolean;
|
|
622
|
-
_meta?: unknown;
|
|
623
623
|
structuredContent?: unknown;
|
|
624
624
|
}, {
|
|
625
|
+
_meta?: unknown;
|
|
625
626
|
content?: ({
|
|
626
627
|
text?: string;
|
|
627
628
|
type?: "text";
|
|
@@ -645,7 +646,6 @@ export declare function safeParseMCPToolCallResult(result: unknown): z.SafeParse
|
|
|
645
646
|
uri?: string;
|
|
646
647
|
})[];
|
|
647
648
|
isError?: boolean;
|
|
648
|
-
_meta?: unknown;
|
|
649
649
|
structuredContent?: unknown;
|
|
650
650
|
}>;
|
|
651
651
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bryan-thompson/inspector-assessment-client",
|
|
3
|
-
"version": "1.32.
|
|
3
|
+
"version": "1.32.2",
|
|
4
4
|
"description": "Client-side application for the Enhanced MCP Inspector with assessment capabilities",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Bryan Thompson <bryan@triepod.ai>",
|