@bryan-thompson/inspector-assessment-client 1.30.1 → 1.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-BbE88qbF.js → OAuthCallback-Cl2ANLYP.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-CfRYq1JG.js → OAuthDebugCallback-Ws62T4Ns.js} +1 -1
- package/dist/assets/{index-cHhcEXbr.css → index-BoUA5OL1.css} +3 -0
- package/dist/assets/{index-CsUB73MT.js → index-DaPIdOcS.js} +3746 -115
- package/dist/index.html +2 -2
- package/lib/lib/assessment/configTypes.d.ts +6 -0
- package/lib/lib/assessment/configTypes.d.ts.map +1 -1
- package/lib/lib/assessment/extendedTypes.d.ts +95 -0
- package/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
- package/lib/lib/assessment/resultTypes.d.ts +14 -2
- package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/lib/lib/assessment/sharedSchemas.d.ts +140 -0
- package/lib/lib/assessment/sharedSchemas.d.ts.map +1 -0
- package/lib/lib/assessment/sharedSchemas.js +113 -0
- package/lib/lib/moduleScoring.d.ts.map +1 -1
- package/lib/lib/moduleScoring.js +5 -0
- package/lib/lib/securityPatterns.d.ts.map +1 -1
- package/lib/lib/securityPatterns.js +2 -2
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +20 -18
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +143 -144
- package/lib/services/assessment/ResponseValidator.d.ts +10 -0
- package/lib/services/assessment/ResponseValidator.d.ts.map +1 -1
- package/lib/services/assessment/ResponseValidator.js +30 -6
- package/lib/services/assessment/config/performanceConfig.d.ts +2 -0
- package/lib/services/assessment/config/performanceConfig.d.ts.map +1 -1
- package/lib/services/assessment/config/performanceConfig.js +5 -33
- package/lib/services/assessment/config/performanceConfigSchemas.d.ts +111 -0
- package/lib/services/assessment/config/performanceConfigSchemas.d.ts.map +1 -0
- package/lib/services/assessment/config/performanceConfigSchemas.js +123 -0
- package/lib/services/assessment/modules/ConformanceAssessor.d.ts +64 -0
- package/lib/services/assessment/modules/ConformanceAssessor.d.ts.map +1 -0
- package/lib/services/assessment/modules/ConformanceAssessor.js +329 -0
- package/lib/services/assessment/modules/ResourceAssessor.d.ts +14 -0
- package/lib/services/assessment/modules/ResourceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ResourceAssessor.js +226 -0
- package/lib/services/assessment/modules/TemporalAssessor.d.ts +14 -0
- package/lib/services/assessment/modules/TemporalAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/TemporalAssessor.js +29 -1
- package/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts +9 -0
- package/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts.map +1 -1
- package/lib/services/assessment/modules/annotations/AlignmentChecker.js +97 -5
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts +6 -4
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts.map +1 -1
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.js +58 -0
- package/lib/services/assessment/modules/annotations/index.d.ts +1 -1
- package/lib/services/assessment/modules/annotations/index.d.ts.map +1 -1
- package/lib/services/assessment/modules/annotations/index.js +2 -1
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.js +3 -3
- package/lib/services/assessment/registry/AssessorDefinitions.d.ts +38 -0
- package/lib/services/assessment/registry/AssessorDefinitions.d.ts.map +1 -0
- package/lib/services/assessment/registry/AssessorDefinitions.js +370 -0
- package/lib/services/assessment/registry/AssessorRegistry.d.ts +124 -0
- package/lib/services/assessment/registry/AssessorRegistry.d.ts.map +1 -0
- package/lib/services/assessment/registry/AssessorRegistry.js +321 -0
- package/lib/services/assessment/registry/estimators.d.ts +93 -0
- package/lib/services/assessment/registry/estimators.d.ts.map +1 -0
- package/lib/services/assessment/registry/estimators.js +176 -0
- package/lib/services/assessment/registry/index.d.ts +13 -0
- package/lib/services/assessment/registry/index.d.ts.map +1 -0
- package/lib/services/assessment/registry/index.js +16 -0
- package/lib/services/assessment/registry/types.d.ts +180 -0
- package/lib/services/assessment/registry/types.d.ts.map +1 -0
- package/lib/services/assessment/registry/types.js +35 -0
- package/lib/services/assessment/responseValidatorSchemas.d.ts +751 -0
- package/lib/services/assessment/responseValidatorSchemas.d.ts.map +1 -0
- package/lib/services/assessment/responseValidatorSchemas.js +244 -0
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"securityPatterns.d.ts","sourceRoot":"","sources":["../../src/lib/securityPatterns.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,iBAAiB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,eAAe,EAAE,CAAC;CAC7B;AAED;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB,EAAE,aAAa,
|
|
1
|
+
{"version":3,"file":"securityPatterns.d.ts","sourceRoot":"","sources":["../../src/lib/securityPatterns.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,iBAAiB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,eAAe,EAAE,CAAC;CAC7B;AAED;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB,EAAE,aAAa,EAo7DnD,CAAC;AAEF;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,UAAU,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,MAAM,GACb,eAAe,EAAE,CAQnB;AAED;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,aAAa,EAAE,CAEtD;AAED;;GAEG;AACH,wBAAgB,oBAAoB;;;;;;;;EA8BnC"}
|
|
@@ -1798,10 +1798,10 @@ export const SECURITY_ATTACK_PATTERNS = [
|
|
|
1798
1798
|
payloadType: "crypto",
|
|
1799
1799
|
parameterTypes: ["action", "operation", "type"],
|
|
1800
1800
|
},
|
|
1801
|
-
// CWE-326: Inadequate encryption key length
|
|
1801
|
+
// CWE-326: Inadequate encryption key length (< 16 bytes)
|
|
1802
1802
|
{
|
|
1803
1803
|
payload: "sign",
|
|
1804
|
-
evidence: /"key_length"\s*:\s*[1-9](?!\d)|"key_secure"\s*:\s*false|"cwe_ids"\s*:\s*\[.*"CWE-326"/i,
|
|
1804
|
+
evidence: /"key_length"\s*:\s*(?:[1-9]|1[0-5])(?!\d)|"key_secure"\s*:\s*false|"cwe_ids"\s*:\s*\[.*"CWE-326"/i,
|
|
1805
1805
|
riskLevel: "HIGH",
|
|
1806
1806
|
description: "Weak HMAC key length easily brute-forceable (CWE-326)",
|
|
1807
1807
|
payloadType: "crypto",
|
|
@@ -116,24 +116,26 @@ export declare class AssessmentOrchestrator {
|
|
|
116
116
|
private totalTestsRun;
|
|
117
117
|
private claudeBridge?;
|
|
118
118
|
private claudeEnabled;
|
|
119
|
-
private
|
|
120
|
-
private
|
|
121
|
-
private
|
|
122
|
-
private
|
|
123
|
-
private
|
|
124
|
-
private
|
|
125
|
-
private
|
|
126
|
-
private
|
|
127
|
-
private
|
|
128
|
-
private
|
|
129
|
-
private
|
|
130
|
-
private
|
|
131
|
-
private
|
|
132
|
-
private
|
|
133
|
-
private
|
|
134
|
-
private
|
|
135
|
-
private
|
|
136
|
-
private
|
|
119
|
+
private registry;
|
|
120
|
+
private get functionalityAssessor();
|
|
121
|
+
private get securityAssessor();
|
|
122
|
+
private get documentationAssessor();
|
|
123
|
+
private get errorHandlingAssessor();
|
|
124
|
+
private get usabilityAssessor();
|
|
125
|
+
private get protocolComplianceAssessor();
|
|
126
|
+
private get aupComplianceAssessor();
|
|
127
|
+
private get toolAnnotationAssessor();
|
|
128
|
+
private get prohibitedLibrariesAssessor();
|
|
129
|
+
private get manifestValidationAssessor();
|
|
130
|
+
private get portabilityAssessor();
|
|
131
|
+
private get externalAPIScannerAssessor();
|
|
132
|
+
private get temporalAssessor();
|
|
133
|
+
private get authenticationAssessor();
|
|
134
|
+
private get resourceAssessor();
|
|
135
|
+
private get promptAssessor();
|
|
136
|
+
private get crossCapabilityAssessor();
|
|
137
|
+
private get fileModularizationAssessor();
|
|
138
|
+
private get conformanceAssessor();
|
|
137
139
|
constructor(config?: Partial<AssessmentConfiguration>);
|
|
138
140
|
/**
|
|
139
141
|
* Get the count of tools that will actually be tested based on selectedToolsForTesting config.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AssessmentOrchestrator.d.ts","sourceRoot":"","sources":["../../../src/services/assessment/AssessmentOrchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,sBAAsB,EACtB,uBAAuB,EAEvB,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,EACX,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,IAAI,EACJ,2BAA2B,EAC5B,MAAM,oCAAoC,CAAC;AAuC5C,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EAEvB,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"AssessmentOrchestrator.d.ts","sourceRoot":"","sources":["../../../src/services/assessment/AssessmentOrchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,sBAAsB,EACtB,uBAAuB,EAEvB,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,EACX,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,IAAI,EACJ,2BAA2B,EAC5B,MAAM,oCAAoC,CAAC;AAuC5C,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EAEvB,MAAM,wBAAwB,CAAC;AA2BhC;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,KAAK,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC,CAAC;CACJ;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,KAAK,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IAClC,SAAS,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IAC3D,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC;AAED;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC5B,OAAO,CAAC,2BAA2B,CAAC,CAAC;IAC1C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,uBAAuB,CAAC;IAChC,UAAU,CAAC,EAAE,UAAU,CAAC;IAIxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAGtC,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAClC,WAAW,CAAC,EAAE,MAAM,CAAC;IAIrB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAG9B,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;IAC1B,iBAAiB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC1C,OAAO,CAAC,EAAE,SAAS,EAAE,CAAC;IACtB,kBAAkB,CAAC,EAAE,qBAAqB,CAAC;IAG3C,YAAY,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAChD,SAAS,CAAC,EAAE,CACV,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KACzB,OAAO,CAAC;QAAE,QAAQ,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC,CAAC;IAGrE,eAAe,CAAC,EAAE;QAChB,IAAI,EAAE,OAAO,GAAG,KAAK,GAAG,iBAAiB,CAAC;QAC1C,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,CAAC;IAIF,SAAS,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;CACnC;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,aAAa,CAAa;IAGlC,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,aAAa,CAAkB;IAIvC,OAAO,CAAC,QAAQ,CAAmB;IAQnC,OAAO,KAAK,qBAAqB,GAEhC;IAED,OAAO,KAAK,gBAAgB,GAE3B;IAED,OAAO,KAAK,qBAAqB,GAEhC;IAED,OAAO,KAAK,qBAAqB,GAEhC;IAED,OAAO,KAAK,iBAAiB,GAE5B;IAED,OAAO,KAAK,0BAA0B,GAErC;IAED,OAAO,KAAK,qBAAqB,GAEhC;IAED,OAAO,KAAK,sBAAsB,GAEjC;IAED,OAAO,KAAK,2BAA2B,GAEtC;IAED,OAAO,KAAK,0BAA0B,GAErC;IAED,OAAO,KAAK,mBAAmB,GAE9B;IAED,OAAO,KAAK,0BAA0B,GAErC;IAED,OAAO,KAAK,gBAAgB,GAE3B;IAED,OAAO,KAAK,sBAAsB,GAEjC;IAED,OAAO,KAAK,gBAAgB,GAE3B;IAED,OAAO,KAAK,cAAc,GAEzB;IAED,OAAO,KAAK,uBAAuB,GAElC;IAED,OAAO,KAAK,0BAA0B,GAErC;IAED,OAAO,KAAK,mBAAmB,GAE9B;gBAIW,MAAM,GAAE,OAAO,CAAC,uBAAuB,CAAM;IAiDzD;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAQ9B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAe9B;;;;OAIG;IACH,gBAAgB,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,sBAAsB,CAAC,GAAG,IAAI;IAgBhE;;;OAGG;IACH,eAAe,IAAI,OAAO;IAI1B;;;OAGG;IACH,eAAe,IAAI,gBAAgB,GAAG,SAAS;IAI/C;;OAEG;IACH,OAAO,CAAC,kBAAkB;IA+C1B;;;OAGG;IACG,iBAAiB,CACrB,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,sBAAsB,CAAC;IAqqBlC;;;;OAIG;IACG,MAAM,CACV,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,IAAI,EAAE,EACb,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC5B,OAAO,CAAC,2BAA2B,CAAC,EACzC,UAAU,CAAC,EAAE,UAAU,EACvB,aAAa,CAAC,EAAE,MAAM,EACtB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACpC,OAAO,CAAC,sBAAsB,CAAC;IAclC,OAAO,CAAC,qBAAqB;IAyF7B;;;OAGG;IACH,SAAS,IAAI,uBAAuB;IAIpC;;;OAGG;IACH,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,uBAAuB,CAAC,GAAG,IAAI;CAG7D"}
|
|
@@ -6,32 +6,9 @@
|
|
|
6
6
|
* @module AssessmentOrchestrator
|
|
7
7
|
*/
|
|
8
8
|
import { DEFAULT_ASSESSMENT_CONFIG, } from "../../lib/assessmentTypes.js";
|
|
9
|
-
// Core assessment modules
|
|
10
|
-
import { FunctionalityAssessor } from "./modules/FunctionalityAssessor.js";
|
|
11
|
-
import { SecurityAssessor } from "./modules/SecurityAssessor.js";
|
|
12
|
-
import { DocumentationAssessor } from "./modules/DocumentationAssessor.js";
|
|
13
|
-
import { ErrorHandlingAssessor } from "./modules/ErrorHandlingAssessor.js";
|
|
14
|
-
import { UsabilityAssessor } from "./modules/UsabilityAssessor.js";
|
|
15
|
-
// Extended assessment modules - unified protocol compliance
|
|
16
|
-
import { ProtocolComplianceAssessor } from "./modules/ProtocolComplianceAssessor.js";
|
|
17
|
-
// New MCP Directory Compliance Gap assessors
|
|
18
|
-
import { AUPComplianceAssessor } from "./modules/AUPComplianceAssessor.js";
|
|
19
|
-
import { ToolAnnotationAssessor } from "./modules/ToolAnnotationAssessor.js";
|
|
20
|
-
import { ProhibitedLibrariesAssessor } from "./modules/ProhibitedLibrariesAssessor.js";
|
|
21
|
-
import { ManifestValidationAssessor } from "./modules/ManifestValidationAssessor.js";
|
|
22
|
-
import { PortabilityAssessor } from "./modules/PortabilityAssessor.js";
|
|
23
|
-
import { ExternalAPIScannerAssessor } from "./modules/ExternalAPIScannerAssessor.js";
|
|
24
|
-
import { TemporalAssessor } from "./modules/TemporalAssessor.js";
|
|
25
|
-
import { AuthenticationAssessor } from "./modules/AuthenticationAssessor.js";
|
|
26
|
-
// New capability assessors
|
|
27
|
-
import { ResourceAssessor } from "./modules/ResourceAssessor.js";
|
|
28
|
-
import { PromptAssessor } from "./modules/PromptAssessor.js";
|
|
29
|
-
import { CrossCapabilitySecurityAssessor } from "./modules/CrossCapabilitySecurityAssessor.js";
|
|
30
|
-
// Code quality assessors
|
|
31
|
-
import { FileModularizationAssessor } from "./modules/FileModularizationAssessor.js";
|
|
32
9
|
// Note: ProtocolConformanceAssessor merged into ProtocolComplianceAssessor (v1.25.2)
|
|
33
|
-
// Pattern configuration for tool annotation assessment
|
|
34
|
-
|
|
10
|
+
// Pattern configuration for tool annotation assessment now handled by registry (Issue #91)
|
|
11
|
+
// See AssessorDefinitions.ts customSetup for ToolAnnotationAssessor
|
|
35
12
|
// Claude Code integration for intelligent analysis
|
|
36
13
|
import { ClaudeCodeBridge, FULL_CLAUDE_CODE_CONFIG, } from "./lib/claudeCodeBridge.js";
|
|
37
14
|
import { TestDataGenerator } from "./TestDataGenerator.js";
|
|
@@ -39,6 +16,10 @@ import { TestDataGenerator } from "./TestDataGenerator.js";
|
|
|
39
16
|
import { createLogger, DEFAULT_LOGGING_CONFIG } from "./lib/logger.js";
|
|
40
17
|
// Extracted helpers for testability
|
|
41
18
|
import { emitModuleStartedEvent, emitModuleProgress, determineOverallStatus, generateSummary, generateRecommendations, } from "./orchestratorHelpers.js";
|
|
19
|
+
// Registry pattern for assessor management (Issue #91)
|
|
20
|
+
import { AssessorRegistry, ASSESSOR_DEFINITIONS, } from "./registry/index.js";
|
|
21
|
+
// Module scoring for dual-key output (Issue #124)
|
|
22
|
+
import { calculateModuleScore } from "../../lib/moduleScoring.js";
|
|
42
23
|
/**
|
|
43
24
|
* Main orchestrator class for running MCP server assessments
|
|
44
25
|
*
|
|
@@ -59,29 +40,71 @@ export class AssessmentOrchestrator {
|
|
|
59
40
|
// Claude Code Bridge for intelligent analysis
|
|
60
41
|
claudeBridge;
|
|
61
42
|
claudeEnabled = false;
|
|
62
|
-
//
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
//
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
43
|
+
// Registry for assessor management (Issue #91)
|
|
44
|
+
// Delegates construction, test count aggregation, and Claude bridge wiring
|
|
45
|
+
registry;
|
|
46
|
+
// ============================================================================
|
|
47
|
+
// Private getters for backward compatibility with tests
|
|
48
|
+
// Tests access these via type assertions: (orchestrator as any).assessorName
|
|
49
|
+
// These delegate to the registry to maintain a single source of truth
|
|
50
|
+
// ============================================================================
|
|
51
|
+
get functionalityAssessor() {
|
|
52
|
+
return this.registry.getAssessor("functionality");
|
|
53
|
+
}
|
|
54
|
+
get securityAssessor() {
|
|
55
|
+
return this.registry.getAssessor("security");
|
|
56
|
+
}
|
|
57
|
+
get documentationAssessor() {
|
|
58
|
+
return this.registry.getAssessor("documentation");
|
|
59
|
+
}
|
|
60
|
+
get errorHandlingAssessor() {
|
|
61
|
+
return this.registry.getAssessor("errorHandling");
|
|
62
|
+
}
|
|
63
|
+
get usabilityAssessor() {
|
|
64
|
+
return this.registry.getAssessor("usability");
|
|
65
|
+
}
|
|
66
|
+
get protocolComplianceAssessor() {
|
|
67
|
+
return this.registry.getAssessor("protocolCompliance");
|
|
68
|
+
}
|
|
69
|
+
get aupComplianceAssessor() {
|
|
70
|
+
return this.registry.getAssessor("aupCompliance");
|
|
71
|
+
}
|
|
72
|
+
get toolAnnotationAssessor() {
|
|
73
|
+
return this.registry.getAssessor("toolAnnotations");
|
|
74
|
+
}
|
|
75
|
+
get prohibitedLibrariesAssessor() {
|
|
76
|
+
return this.registry.getAssessor("prohibitedLibraries");
|
|
77
|
+
}
|
|
78
|
+
get manifestValidationAssessor() {
|
|
79
|
+
return this.registry.getAssessor("manifestValidation");
|
|
80
|
+
}
|
|
81
|
+
get portabilityAssessor() {
|
|
82
|
+
return this.registry.getAssessor("portability");
|
|
83
|
+
}
|
|
84
|
+
get externalAPIScannerAssessor() {
|
|
85
|
+
return this.registry.getAssessor("externalAPIScanner");
|
|
86
|
+
}
|
|
87
|
+
get temporalAssessor() {
|
|
88
|
+
return this.registry.getAssessor("temporal");
|
|
89
|
+
}
|
|
90
|
+
get authenticationAssessor() {
|
|
91
|
+
return this.registry.getAssessor("authentication");
|
|
92
|
+
}
|
|
93
|
+
get resourceAssessor() {
|
|
94
|
+
return this.registry.getAssessor("resources");
|
|
95
|
+
}
|
|
96
|
+
get promptAssessor() {
|
|
97
|
+
return this.registry.getAssessor("prompts");
|
|
98
|
+
}
|
|
99
|
+
get crossCapabilityAssessor() {
|
|
100
|
+
return this.registry.getAssessor("crossCapability");
|
|
101
|
+
}
|
|
102
|
+
get fileModularizationAssessor() {
|
|
103
|
+
return this.registry.getAssessor("fileModularization");
|
|
104
|
+
}
|
|
105
|
+
get conformanceAssessor() {
|
|
106
|
+
return this.registry.getAssessor("conformance");
|
|
107
|
+
}
|
|
85
108
|
// Note: protocolConformanceAssessor merged into protocolComplianceAssessor (v1.25.2)
|
|
86
109
|
constructor(config = {}) {
|
|
87
110
|
this.config = { ...DEFAULT_ASSESSMENT_CONFIG, ...config };
|
|
@@ -100,92 +123,17 @@ export class AssessmentOrchestrator {
|
|
|
100
123
|
if (this.config.claudeCode?.enabled) {
|
|
101
124
|
this.initializeClaudeBridge(this.config.claudeCode);
|
|
102
125
|
}
|
|
103
|
-
// Initialize
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
if (this.config.assessmentCategories?.documentation !== false) {
|
|
115
|
-
this.documentationAssessor = new DocumentationAssessor(this.config);
|
|
116
|
-
}
|
|
117
|
-
if (this.config.assessmentCategories?.errorHandling !== false) {
|
|
118
|
-
this.errorHandlingAssessor = new ErrorHandlingAssessor(this.config);
|
|
119
|
-
}
|
|
120
|
-
if (this.config.assessmentCategories?.usability !== false) {
|
|
121
|
-
this.usabilityAssessor = new UsabilityAssessor(this.config);
|
|
122
|
-
}
|
|
123
|
-
// Initialize extended assessors if enabled
|
|
124
|
-
if (this.config.enableExtendedAssessment) {
|
|
125
|
-
// Initialize unified protocol compliance assessor
|
|
126
|
-
// Supports new protocolCompliance flag and deprecated mcpSpecCompliance/protocolConformance
|
|
127
|
-
if (this.config.assessmentCategories?.protocolCompliance ||
|
|
128
|
-
this.config.assessmentCategories?.mcpSpecCompliance ||
|
|
129
|
-
this.config.assessmentCategories?.protocolConformance) {
|
|
130
|
-
this.protocolComplianceAssessor = new ProtocolComplianceAssessor(this.config);
|
|
131
|
-
}
|
|
132
|
-
// Initialize new MCP Directory Compliance Gap assessors
|
|
133
|
-
if (this.config.assessmentCategories?.aupCompliance) {
|
|
134
|
-
this.aupComplianceAssessor = new AUPComplianceAssessor(this.config);
|
|
135
|
-
// Wire up Claude bridge for semantic analysis
|
|
136
|
-
if (this.claudeBridge) {
|
|
137
|
-
this.aupComplianceAssessor.setClaudeBridge(this.claudeBridge);
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
if (this.config.assessmentCategories?.toolAnnotations) {
|
|
141
|
-
this.toolAnnotationAssessor = new ToolAnnotationAssessor(this.config);
|
|
142
|
-
// Wire up Claude bridge for behavior inference
|
|
143
|
-
if (this.claudeBridge) {
|
|
144
|
-
this.toolAnnotationAssessor.setClaudeBridge(this.claudeBridge);
|
|
145
|
-
}
|
|
146
|
-
// Load custom pattern configuration if provided
|
|
147
|
-
if (this.config.patternConfigPath) {
|
|
148
|
-
const patternConfig = loadPatternConfig(this.config.patternConfigPath, this.logger);
|
|
149
|
-
const compiledPatterns = compilePatterns(patternConfig);
|
|
150
|
-
this.toolAnnotationAssessor.setPatterns(compiledPatterns);
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
if (this.config.assessmentCategories?.prohibitedLibraries) {
|
|
154
|
-
this.prohibitedLibrariesAssessor = new ProhibitedLibrariesAssessor(this.config);
|
|
155
|
-
}
|
|
156
|
-
if (this.config.assessmentCategories?.manifestValidation) {
|
|
157
|
-
this.manifestValidationAssessor = new ManifestValidationAssessor(this.config);
|
|
158
|
-
}
|
|
159
|
-
if (this.config.assessmentCategories?.portability) {
|
|
160
|
-
this.portabilityAssessor = new PortabilityAssessor(this.config);
|
|
161
|
-
}
|
|
162
|
-
if (this.config.assessmentCategories?.externalAPIScanner) {
|
|
163
|
-
this.externalAPIScannerAssessor = new ExternalAPIScannerAssessor(this.config);
|
|
164
|
-
}
|
|
165
|
-
if (this.config.assessmentCategories?.temporal) {
|
|
166
|
-
this.temporalAssessor = new TemporalAssessor(this.config);
|
|
167
|
-
}
|
|
168
|
-
if (this.config.assessmentCategories?.authentication) {
|
|
169
|
-
this.authenticationAssessor = new AuthenticationAssessor(this.config);
|
|
170
|
-
}
|
|
171
|
-
// Initialize new capability assessors
|
|
172
|
-
if (this.config.assessmentCategories?.resources) {
|
|
173
|
-
this.resourceAssessor = new ResourceAssessor(this.config);
|
|
174
|
-
}
|
|
175
|
-
if (this.config.assessmentCategories?.prompts) {
|
|
176
|
-
this.promptAssessor = new PromptAssessor(this.config);
|
|
177
|
-
}
|
|
178
|
-
if (this.config.assessmentCategories?.crossCapability) {
|
|
179
|
-
this.crossCapabilityAssessor = new CrossCapabilitySecurityAssessor(this.config);
|
|
180
|
-
}
|
|
181
|
-
// Initialize code quality assessors
|
|
182
|
-
if (this.config.assessmentCategories?.fileModularization) {
|
|
183
|
-
this.fileModularizationAssessor = new FileModularizationAssessor(this.config);
|
|
184
|
-
}
|
|
185
|
-
// Note: Protocol conformance now handled by unified ProtocolComplianceAssessor above
|
|
186
|
-
}
|
|
187
|
-
// Wire up Claude bridge to TestDataGenerator for intelligent test generation
|
|
126
|
+
// Initialize registry and register all enabled assessors (Issue #91)
|
|
127
|
+
// The registry handles:
|
|
128
|
+
// - Conditional instantiation based on config flags
|
|
129
|
+
// - Deprecated flag OR logic (e.g., protocolCompliance supports 3 flags)
|
|
130
|
+
// - Custom setup (e.g., ToolAnnotationAssessor pattern config)
|
|
131
|
+
// - Claude bridge wiring for supporting assessors
|
|
132
|
+
this.registry = new AssessorRegistry(this.config);
|
|
133
|
+
this.registry.registerAll(ASSESSOR_DEFINITIONS);
|
|
134
|
+
// Wire up Claude bridge to registry (handles all supporting assessors)
|
|
188
135
|
if (this.claudeBridge) {
|
|
136
|
+
this.registry.setClaudeBridge(this.claudeBridge);
|
|
189
137
|
TestDataGenerator.setClaudeBridge(this.claudeBridge);
|
|
190
138
|
}
|
|
191
139
|
// Set logger for TestDataGenerator diagnostic output
|
|
@@ -233,17 +181,9 @@ export class AssessmentOrchestrator {
|
|
|
233
181
|
enabled: true,
|
|
234
182
|
};
|
|
235
183
|
this.initializeClaudeBridge(bridgeConfig);
|
|
236
|
-
// Wire up to
|
|
184
|
+
// Wire up to all supporting assessors via registry
|
|
237
185
|
if (this.claudeBridge) {
|
|
238
|
-
|
|
239
|
-
this.aupComplianceAssessor.setClaudeBridge(this.claudeBridge);
|
|
240
|
-
}
|
|
241
|
-
if (this.toolAnnotationAssessor) {
|
|
242
|
-
this.toolAnnotationAssessor.setClaudeBridge(this.claudeBridge);
|
|
243
|
-
}
|
|
244
|
-
if (this.securityAssessor) {
|
|
245
|
-
this.securityAssessor.setClaudeBridge(this.claudeBridge);
|
|
246
|
-
}
|
|
186
|
+
this.registry.setClaudeBridge(this.claudeBridge);
|
|
247
187
|
TestDataGenerator.setClaudeBridge(this.claudeBridge);
|
|
248
188
|
}
|
|
249
189
|
}
|
|
@@ -305,6 +245,10 @@ export class AssessmentOrchestrator {
|
|
|
305
245
|
if (this.fileModularizationAssessor) {
|
|
306
246
|
this.fileModularizationAssessor.resetTestCount();
|
|
307
247
|
}
|
|
248
|
+
// Reset official conformance assessor
|
|
249
|
+
if (this.conformanceAssessor) {
|
|
250
|
+
this.conformanceAssessor.resetTestCount();
|
|
251
|
+
}
|
|
308
252
|
}
|
|
309
253
|
/**
|
|
310
254
|
* Run a complete assessment on an MCP server
|
|
@@ -462,6 +406,15 @@ export class AssessmentOrchestrator {
|
|
|
462
406
|
return (assessmentResults.fileModularization = r);
|
|
463
407
|
}));
|
|
464
408
|
}
|
|
409
|
+
// Official MCP conformance testing (opt-in, requires HTTP/SSE transport)
|
|
410
|
+
if (this.conformanceAssessor) {
|
|
411
|
+
// Conformance tests ~7 server scenarios
|
|
412
|
+
emitModuleStartedEvent("Conformance", 7, toolCount);
|
|
413
|
+
assessmentPromises.push(this.conformanceAssessor.assess(context).then((r) => {
|
|
414
|
+
emitModuleProgress("Conformance", r.status, r, this.conformanceAssessor.getTestCount());
|
|
415
|
+
return (assessmentResults.conformance = r);
|
|
416
|
+
}));
|
|
417
|
+
}
|
|
465
418
|
// Note: Protocol Conformance now handled by unified ProtocolComplianceAssessor above
|
|
466
419
|
await Promise.all(assessmentPromises);
|
|
467
420
|
}
|
|
@@ -587,6 +540,13 @@ export class AssessmentOrchestrator {
|
|
|
587
540
|
await this.fileModularizationAssessor.assess(context);
|
|
588
541
|
emitModuleProgress("File Modularization", assessmentResults.fileModularization.status, assessmentResults.fileModularization, this.fileModularizationAssessor.getTestCount());
|
|
589
542
|
}
|
|
543
|
+
// Official MCP conformance testing (sequential, opt-in)
|
|
544
|
+
if (this.conformanceAssessor) {
|
|
545
|
+
emitModuleStartedEvent("Conformance", 7, toolCount);
|
|
546
|
+
assessmentResults.conformance =
|
|
547
|
+
await this.conformanceAssessor.assess(context);
|
|
548
|
+
emitModuleProgress("Conformance", assessmentResults.conformance.status, assessmentResults.conformance, this.conformanceAssessor.getTestCount());
|
|
549
|
+
}
|
|
590
550
|
// Note: Protocol Conformance now handled by unified ProtocolComplianceAssessor above
|
|
591
551
|
}
|
|
592
552
|
// Integrate temporal findings into security.vulnerabilities for unified view
|
|
@@ -597,6 +557,41 @@ export class AssessmentOrchestrator {
|
|
|
597
557
|
assessmentResults.security.vulnerabilities.push(`RUG_PULL_TEMPORAL: ${detail.tool} - Tool behavior changed after invocation ${detail.firstDeviationAt}. Requires immediate manual review.`);
|
|
598
558
|
}
|
|
599
559
|
}
|
|
560
|
+
// Issue #124: Dual-key output for v2.0.0 transition
|
|
561
|
+
// Output BOTH old and new keys to maintain backward compatibility
|
|
562
|
+
// Old keys (documentation, usability, mcpSpecCompliance) will be removed in v2.0.0
|
|
563
|
+
// developerExperience (new) = documentation + usability (deprecated)
|
|
564
|
+
if (assessmentResults.documentation && assessmentResults.usability) {
|
|
565
|
+
const docScore = calculateModuleScore(assessmentResults.documentation) ?? 50;
|
|
566
|
+
const usabilityScore = calculateModuleScore(assessmentResults.usability) ?? 50;
|
|
567
|
+
const combinedStatus = determineOverallStatus({
|
|
568
|
+
documentation: assessmentResults.documentation,
|
|
569
|
+
usability: assessmentResults.usability,
|
|
570
|
+
});
|
|
571
|
+
assessmentResults.developerExperience = {
|
|
572
|
+
documentation: assessmentResults.documentation,
|
|
573
|
+
usability: assessmentResults.usability,
|
|
574
|
+
status: combinedStatus,
|
|
575
|
+
score: Math.round((docScore + usabilityScore) / 2),
|
|
576
|
+
};
|
|
577
|
+
// Emit deprecation warning for old keys
|
|
578
|
+
this.logger.warn("Output keys 'documentation' and 'usability' are deprecated. " +
|
|
579
|
+
"Use 'developerExperience' instead. These keys will be removed in v2.0.0.", {
|
|
580
|
+
deprecated: ["documentation", "usability"],
|
|
581
|
+
replacement: "developerExperience",
|
|
582
|
+
});
|
|
583
|
+
}
|
|
584
|
+
// protocolCompliance (new) = mcpSpecCompliance (deprecated)
|
|
585
|
+
if (assessmentResults.mcpSpecCompliance) {
|
|
586
|
+
assessmentResults.protocolCompliance =
|
|
587
|
+
assessmentResults.mcpSpecCompliance;
|
|
588
|
+
// Emit deprecation warning for old key
|
|
589
|
+
this.logger.warn("Output key 'mcpSpecCompliance' is deprecated. " +
|
|
590
|
+
"Use 'protocolCompliance' instead. This key will be removed in v2.0.0.", {
|
|
591
|
+
deprecated: ["mcpSpecCompliance"],
|
|
592
|
+
replacement: "protocolCompliance",
|
|
593
|
+
});
|
|
594
|
+
}
|
|
600
595
|
// Collect test counts from all assessors
|
|
601
596
|
this.totalTestsRun = this.collectTotalTestCount();
|
|
602
597
|
// Determine overall status
|
|
@@ -668,6 +663,8 @@ export class AssessmentOrchestrator {
|
|
|
668
663
|
const crossCapabilityCount = this.crossCapabilityAssessor?.getTestCount() || 0;
|
|
669
664
|
// Code quality assessor counts
|
|
670
665
|
const fileModularizationCount = this.fileModularizationAssessor?.getTestCount() || 0;
|
|
666
|
+
// Official MCP conformance test count
|
|
667
|
+
const conformanceCount = this.conformanceAssessor?.getTestCount() || 0;
|
|
671
668
|
// Note: Protocol conformance now included in mcpSpecCount (unified ProtocolComplianceAssessor)
|
|
672
669
|
this.logger.debug("Test counts by assessor", {
|
|
673
670
|
functionality: functionalityCount,
|
|
@@ -688,6 +685,7 @@ export class AssessmentOrchestrator {
|
|
|
688
685
|
prompts: promptsCount,
|
|
689
686
|
crossCapability: crossCapabilityCount,
|
|
690
687
|
fileModularization: fileModularizationCount,
|
|
688
|
+
conformance: conformanceCount,
|
|
691
689
|
// Note: protocolConformance now included in mcpSpec (unified)
|
|
692
690
|
});
|
|
693
691
|
total =
|
|
@@ -708,7 +706,8 @@ export class AssessmentOrchestrator {
|
|
|
708
706
|
resourcesCount +
|
|
709
707
|
promptsCount +
|
|
710
708
|
crossCapabilityCount +
|
|
711
|
-
fileModularizationCount
|
|
709
|
+
fileModularizationCount +
|
|
710
|
+
conformanceCount;
|
|
712
711
|
// Note: protocolConformance now included in mcpSpecCount (unified)
|
|
713
712
|
this.logger.debug("Total test count", { total });
|
|
714
713
|
return total;
|
|
@@ -24,6 +24,16 @@ export interface ValidationContext {
|
|
|
24
24
|
scenarioCategory?: "happy_path" | "edge_case" | "boundary" | "error_case";
|
|
25
25
|
}
|
|
26
26
|
export declare class ResponseValidator {
|
|
27
|
+
/**
|
|
28
|
+
* Safely extract content array from response using Zod validation.
|
|
29
|
+
* Falls back to undefined if content is not a valid array.
|
|
30
|
+
*/
|
|
31
|
+
private static safeGetContentArray;
|
|
32
|
+
/**
|
|
33
|
+
* Safely parse MCP tool call result using Zod validation.
|
|
34
|
+
* Returns validated data or undefined if validation fails.
|
|
35
|
+
*/
|
|
36
|
+
private static safeGetMCPResponse;
|
|
27
37
|
/**
|
|
28
38
|
* Extract response metadata including content types, structuredContent, and _meta
|
|
29
39
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ResponseValidator.d.ts","sourceRoot":"","sources":["../../../src/services/assessment/ResponseValidator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,2BAA2B,EAC3B,IAAI,EACL,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"ResponseValidator.d.ts","sourceRoot":"","sources":["../../../src/services/assessment/ResponseValidator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,2BAA2B,EAC3B,IAAI,EACL,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAazD,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,cAAc,EACV,eAAe,GACf,mBAAmB,GACnB,mBAAmB,GACnB,QAAQ,GACR,OAAO,CAAC;IACZ,8EAA8E;IAC9E,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,QAAQ,EAAE,2BAA2B,CAAC;IACtC,gBAAgB,CAAC,EAAE,YAAY,GAAG,WAAW,GAAG,UAAU,GAAG,YAAY,CAAC;CAC3E;AAED,qBAAa,iBAAiB;IAC5B;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,mBAAmB;IAOlC;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAOjC;;OAEG;IACH,MAAM,CAAC,uBAAuB,CAAC,OAAO,EAAE,iBAAiB,GAAG,gBAAgB;IA2G5E;;OAEG;IACH,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,GAAG,gBAAgB;IAgHrE;;;OAGG;IACH,MAAM,CAAC,oBAAoB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO;IAgThE;;OAEG;IACH,MAAM,CAAC,0BAA0B,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,MAAM;CAsBvE"}
|
|
@@ -6,13 +6,31 @@
|
|
|
6
6
|
* @module assessment/ResponseValidator
|
|
7
7
|
*/
|
|
8
8
|
import { validateToolOutput, hasOutputSchema, tryExtractJsonFromContent, } from "../../utils/schemaUtils.js";
|
|
9
|
+
import { safeParseContentArray, safeParseMCPToolCallResult, } from "./responseValidatorSchemas.js";
|
|
9
10
|
export class ResponseValidator {
|
|
11
|
+
/**
|
|
12
|
+
* Safely extract content array from response using Zod validation.
|
|
13
|
+
* Falls back to undefined if content is not a valid array.
|
|
14
|
+
*/
|
|
15
|
+
static safeGetContentArray(response) {
|
|
16
|
+
const parseResult = safeParseContentArray(response.content);
|
|
17
|
+
return parseResult.success ? parseResult.data : undefined;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Safely parse MCP tool call result using Zod validation.
|
|
21
|
+
* Returns validated data or undefined if validation fails.
|
|
22
|
+
*/
|
|
23
|
+
static safeGetMCPResponse(response) {
|
|
24
|
+
const parseResult = safeParseMCPToolCallResult(response);
|
|
25
|
+
return parseResult.success ? parseResult.data : undefined;
|
|
26
|
+
}
|
|
10
27
|
/**
|
|
11
28
|
* Extract response metadata including content types, structuredContent, and _meta
|
|
12
29
|
*/
|
|
13
30
|
static extractResponseMetadata(context) {
|
|
14
|
-
|
|
15
|
-
const
|
|
31
|
+
// Use validated parsing for content array and full response
|
|
32
|
+
const content = this.safeGetContentArray(context.response);
|
|
33
|
+
const validatedResponse = this.safeGetMCPResponse(context.response);
|
|
16
34
|
// Track content types present
|
|
17
35
|
const contentTypes = [];
|
|
18
36
|
let textBlockCount = 0;
|
|
@@ -40,17 +58,23 @@ export class ResponseValidator {
|
|
|
40
58
|
}
|
|
41
59
|
}
|
|
42
60
|
// Check for structuredContent property (MCP 2024-11-05+)
|
|
43
|
-
|
|
44
|
-
|
|
61
|
+
// Use validated response data when available, fallback to raw response check
|
|
62
|
+
const hasStructuredContent = validatedResponse?.structuredContent !== undefined ||
|
|
63
|
+
("structuredContent" in context.response &&
|
|
64
|
+
context.response.structuredContent !== undefined);
|
|
45
65
|
// Check for _meta property
|
|
46
|
-
const hasMeta =
|
|
66
|
+
const hasMeta = validatedResponse?._meta !== undefined ||
|
|
67
|
+
("_meta" in context.response && context.response._meta !== undefined);
|
|
47
68
|
// Output schema validation
|
|
48
69
|
let outputSchemaValidation;
|
|
49
70
|
const toolHasOutputSchema = hasOutputSchema(context.tool.name);
|
|
50
71
|
if (toolHasOutputSchema) {
|
|
51
72
|
if (hasStructuredContent) {
|
|
52
73
|
// Primary path: validate structuredContent
|
|
53
|
-
|
|
74
|
+
// Prefer validated data, fallback to raw response
|
|
75
|
+
const structuredContent = validatedResponse?.structuredContent ??
|
|
76
|
+
context.response.structuredContent;
|
|
77
|
+
const validation = validateToolOutput(context.tool.name, structuredContent);
|
|
54
78
|
outputSchemaValidation = {
|
|
55
79
|
hasOutputSchema: true,
|
|
56
80
|
isValid: validation.isValid,
|
|
@@ -102,6 +102,8 @@ export declare const PERFORMANCE_PRESETS: {
|
|
|
102
102
|
* Validate a partial performance config.
|
|
103
103
|
* Ensures values are within reasonable bounds.
|
|
104
104
|
*
|
|
105
|
+
* Uses Zod schema validation under the hood (Issue #84).
|
|
106
|
+
*
|
|
105
107
|
* @public
|
|
106
108
|
* @param config - Partial config to validate
|
|
107
109
|
* @returns Array of validation error messages (empty if valid)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"performanceConfig.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/config/performanceConfig.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"performanceConfig.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/config/performanceConfig.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAG5C;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,oBAAoB,EAAE,MAAM,CAAC;IAE7B;;;;OAIG;IACH,sBAAsB,EAAE,MAAM,CAAC;IAE/B;;;OAGG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,aAAa,EAAE,MAAM,CAAC;IAEtB;;;;OAIG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAE9B;;;;;;;;;;OAUG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,wBAAwB,EAAE,MAAM,CAAC;CAClC;AAED;;;;GAIG;AACH,eAAO,MAAM,0BAA0B,EAAE,QAAQ,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CASzE,CAAC;AAEL;;;GAGG;AACH,eAAO,MAAM,mBAAmB;IAC9B,mDAAmD;;IAGnD,8CAA8C;;;;8BA1ExB,MAAM;uBAoBb,MAAM;+BAOE,MAAM;+BAaN,MAAM;kCAOH,MAAM;;IAkChC,kEAAkE;;;;;8BAjF5C,MAAM;uBAoBb,MAAM;+BAOE,MAAM;kCAoBH,MAAM;;CAyCxB,CAAC;AAEX;;;;;;;;;GASG;AACH,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,OAAO,CAAC,iBAAiB,CAAC,GACjC,MAAM,EAAE,CAGV;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAClC,QAAQ,CAAC,iBAAiB,CAAC,CAsB7B;AAED;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CACnC,UAAU,CAAC,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,MAAM,GACd,QAAQ,CAAC,iBAAiB,CAAC,CAyC7B"}
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* @see https://github.com/triepod-ai/inspector-assessment/issues/37
|
|
11
11
|
*/
|
|
12
12
|
import * as fs from "fs";
|
|
13
|
+
import { validatePerformanceConfigWithZod } from "./performanceConfigSchemas.js";
|
|
13
14
|
/**
|
|
14
15
|
* Default performance configuration.
|
|
15
16
|
* These values preserve existing behavior across all modules.
|
|
@@ -49,44 +50,15 @@ export const PERFORMANCE_PRESETS = {
|
|
|
49
50
|
* Validate a partial performance config.
|
|
50
51
|
* Ensures values are within reasonable bounds.
|
|
51
52
|
*
|
|
53
|
+
* Uses Zod schema validation under the hood (Issue #84).
|
|
54
|
+
*
|
|
52
55
|
* @public
|
|
53
56
|
* @param config - Partial config to validate
|
|
54
57
|
* @returns Array of validation error messages (empty if valid)
|
|
55
58
|
*/
|
|
56
59
|
export function validatePerformanceConfig(config) {
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
(config.batchFlushIntervalMs < 50 || config.batchFlushIntervalMs > 10000)) {
|
|
60
|
-
errors.push("batchFlushIntervalMs must be between 50 and 10000");
|
|
61
|
-
}
|
|
62
|
-
if (config.functionalityBatchSize !== undefined &&
|
|
63
|
-
(config.functionalityBatchSize < 1 || config.functionalityBatchSize > 100)) {
|
|
64
|
-
errors.push("functionalityBatchSize must be between 1 and 100");
|
|
65
|
-
}
|
|
66
|
-
if (config.securityBatchSize !== undefined &&
|
|
67
|
-
(config.securityBatchSize < 1 || config.securityBatchSize > 100)) {
|
|
68
|
-
errors.push("securityBatchSize must be between 1 and 100");
|
|
69
|
-
}
|
|
70
|
-
if (config.testTimeoutMs !== undefined &&
|
|
71
|
-
(config.testTimeoutMs < 100 || config.testTimeoutMs > 300000)) {
|
|
72
|
-
errors.push("testTimeoutMs must be between 100 and 300000");
|
|
73
|
-
}
|
|
74
|
-
if (config.securityTestTimeoutMs !== undefined &&
|
|
75
|
-
(config.securityTestTimeoutMs < 100 ||
|
|
76
|
-
config.securityTestTimeoutMs > 300000)) {
|
|
77
|
-
errors.push("securityTestTimeoutMs must be between 100 and 300000");
|
|
78
|
-
}
|
|
79
|
-
if (config.queueWarningThreshold !== undefined &&
|
|
80
|
-
(config.queueWarningThreshold < 100 ||
|
|
81
|
-
config.queueWarningThreshold > 1000000)) {
|
|
82
|
-
errors.push("queueWarningThreshold must be between 100 and 1000000");
|
|
83
|
-
}
|
|
84
|
-
if (config.eventEmitterMaxListeners !== undefined &&
|
|
85
|
-
(config.eventEmitterMaxListeners < 10 ||
|
|
86
|
-
config.eventEmitterMaxListeners > 1000)) {
|
|
87
|
-
errors.push("eventEmitterMaxListeners must be between 10 and 1000");
|
|
88
|
-
}
|
|
89
|
-
return errors;
|
|
60
|
+
// Delegate to Zod schema validation
|
|
61
|
+
return validatePerformanceConfigWithZod(config);
|
|
90
62
|
}
|
|
91
63
|
/**
|
|
92
64
|
* Merge a partial config with defaults.
|