@bryan-thompson/inspector-assessment-client 1.30.0 → 1.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-BbE88qbF.js → OAuthCallback-CXcl26vR.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-CfRYq1JG.js → OAuthDebugCallback-J9s4SF_c.js} +1 -1
- package/dist/assets/{index-cHhcEXbr.css → index-BoUA5OL1.css} +3 -0
- package/dist/assets/{index-CsUB73MT.js → index-_HAw2b2G.js} +3746 -115
- package/dist/index.html +2 -2
- package/lib/lib/assessment/configTypes.d.ts +6 -0
- package/lib/lib/assessment/configTypes.d.ts.map +1 -1
- package/lib/lib/assessment/extendedTypes.d.ts +74 -0
- package/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
- package/lib/lib/assessment/resultTypes.d.ts +3 -1
- package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/lib/lib/assessment/sharedSchemas.d.ts +140 -0
- package/lib/lib/assessment/sharedSchemas.d.ts.map +1 -0
- package/lib/lib/assessment/sharedSchemas.js +113 -0
- package/lib/lib/securityPatterns.d.ts.map +1 -1
- package/lib/lib/securityPatterns.js +2 -2
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +1 -0
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +34 -1
- package/lib/services/assessment/ResponseValidator.d.ts +10 -0
- package/lib/services/assessment/ResponseValidator.d.ts.map +1 -1
- package/lib/services/assessment/ResponseValidator.js +30 -6
- package/lib/services/assessment/config/performanceConfig.d.ts +2 -0
- package/lib/services/assessment/config/performanceConfig.d.ts.map +1 -1
- package/lib/services/assessment/config/performanceConfig.js +5 -33
- package/lib/services/assessment/config/performanceConfigSchemas.d.ts +111 -0
- package/lib/services/assessment/config/performanceConfigSchemas.d.ts.map +1 -0
- package/lib/services/assessment/config/performanceConfigSchemas.js +123 -0
- package/lib/services/assessment/modules/ConformanceAssessor.d.ts +60 -0
- package/lib/services/assessment/modules/ConformanceAssessor.d.ts.map +1 -0
- package/lib/services/assessment/modules/ConformanceAssessor.js +308 -0
- package/lib/services/assessment/modules/ResourceAssessor.d.ts +14 -0
- package/lib/services/assessment/modules/ResourceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ResourceAssessor.js +221 -0
- package/lib/services/assessment/modules/TemporalAssessor.d.ts +14 -0
- package/lib/services/assessment/modules/TemporalAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/TemporalAssessor.js +29 -1
- package/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts +9 -0
- package/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts.map +1 -1
- package/lib/services/assessment/modules/annotations/AlignmentChecker.js +97 -5
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts +6 -4
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts.map +1 -1
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.js +58 -0
- package/lib/services/assessment/modules/annotations/index.d.ts +1 -1
- package/lib/services/assessment/modules/annotations/index.d.ts.map +1 -1
- package/lib/services/assessment/modules/annotations/index.js +2 -1
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.js +3 -3
- package/lib/services/assessment/responseValidatorSchemas.d.ts +751 -0
- package/lib/services/assessment/responseValidatorSchemas.d.ts.map +1 -0
- package/lib/services/assessment/responseValidatorSchemas.js +244 -0
- package/package.json +1 -1
|
@@ -29,6 +29,8 @@ import { PromptAssessor } from "./modules/PromptAssessor.js";
|
|
|
29
29
|
import { CrossCapabilitySecurityAssessor } from "./modules/CrossCapabilitySecurityAssessor.js";
|
|
30
30
|
// Code quality assessors
|
|
31
31
|
import { FileModularizationAssessor } from "./modules/FileModularizationAssessor.js";
|
|
32
|
+
// Official MCP conformance testing
|
|
33
|
+
import { ConformanceAssessor } from "./modules/ConformanceAssessor.js";
|
|
32
34
|
// Note: ProtocolConformanceAssessor merged into ProtocolComplianceAssessor (v1.25.2)
|
|
33
35
|
// Pattern configuration for tool annotation assessment
|
|
34
36
|
import { loadPatternConfig, compilePatterns, } from "./config/annotationPatterns.js";
|
|
@@ -82,6 +84,8 @@ export class AssessmentOrchestrator {
|
|
|
82
84
|
crossCapabilityAssessor;
|
|
83
85
|
// Code quality assessors
|
|
84
86
|
fileModularizationAssessor;
|
|
87
|
+
// Official MCP conformance testing (opt-in via --conformance flag)
|
|
88
|
+
conformanceAssessor;
|
|
85
89
|
// Note: protocolConformanceAssessor merged into protocolComplianceAssessor (v1.25.2)
|
|
86
90
|
constructor(config = {}) {
|
|
87
91
|
this.config = { ...DEFAULT_ASSESSMENT_CONFIG, ...config };
|
|
@@ -182,6 +186,11 @@ export class AssessmentOrchestrator {
|
|
|
182
186
|
if (this.config.assessmentCategories?.fileModularization) {
|
|
183
187
|
this.fileModularizationAssessor = new FileModularizationAssessor(this.config);
|
|
184
188
|
}
|
|
189
|
+
// Initialize official MCP conformance testing (opt-in via --conformance flag)
|
|
190
|
+
// Requires HTTP/SSE transport with serverUrl available
|
|
191
|
+
if (this.config.assessmentCategories?.conformance) {
|
|
192
|
+
this.conformanceAssessor = new ConformanceAssessor(this.config);
|
|
193
|
+
}
|
|
185
194
|
// Note: Protocol conformance now handled by unified ProtocolComplianceAssessor above
|
|
186
195
|
}
|
|
187
196
|
// Wire up Claude bridge to TestDataGenerator for intelligent test generation
|
|
@@ -305,6 +314,10 @@ export class AssessmentOrchestrator {
|
|
|
305
314
|
if (this.fileModularizationAssessor) {
|
|
306
315
|
this.fileModularizationAssessor.resetTestCount();
|
|
307
316
|
}
|
|
317
|
+
// Reset official conformance assessor
|
|
318
|
+
if (this.conformanceAssessor) {
|
|
319
|
+
this.conformanceAssessor.resetTestCount();
|
|
320
|
+
}
|
|
308
321
|
}
|
|
309
322
|
/**
|
|
310
323
|
* Run a complete assessment on an MCP server
|
|
@@ -462,6 +475,15 @@ export class AssessmentOrchestrator {
|
|
|
462
475
|
return (assessmentResults.fileModularization = r);
|
|
463
476
|
}));
|
|
464
477
|
}
|
|
478
|
+
// Official MCP conformance testing (opt-in, requires HTTP/SSE transport)
|
|
479
|
+
if (this.conformanceAssessor) {
|
|
480
|
+
// Conformance tests ~7 server scenarios
|
|
481
|
+
emitModuleStartedEvent("Conformance", 7, toolCount);
|
|
482
|
+
assessmentPromises.push(this.conformanceAssessor.assess(context).then((r) => {
|
|
483
|
+
emitModuleProgress("Conformance", r.status, r, this.conformanceAssessor.getTestCount());
|
|
484
|
+
return (assessmentResults.conformance = r);
|
|
485
|
+
}));
|
|
486
|
+
}
|
|
465
487
|
// Note: Protocol Conformance now handled by unified ProtocolComplianceAssessor above
|
|
466
488
|
await Promise.all(assessmentPromises);
|
|
467
489
|
}
|
|
@@ -587,6 +609,13 @@ export class AssessmentOrchestrator {
|
|
|
587
609
|
await this.fileModularizationAssessor.assess(context);
|
|
588
610
|
emitModuleProgress("File Modularization", assessmentResults.fileModularization.status, assessmentResults.fileModularization, this.fileModularizationAssessor.getTestCount());
|
|
589
611
|
}
|
|
612
|
+
// Official MCP conformance testing (sequential, opt-in)
|
|
613
|
+
if (this.conformanceAssessor) {
|
|
614
|
+
emitModuleStartedEvent("Conformance", 7, toolCount);
|
|
615
|
+
assessmentResults.conformance =
|
|
616
|
+
await this.conformanceAssessor.assess(context);
|
|
617
|
+
emitModuleProgress("Conformance", assessmentResults.conformance.status, assessmentResults.conformance, this.conformanceAssessor.getTestCount());
|
|
618
|
+
}
|
|
590
619
|
// Note: Protocol Conformance now handled by unified ProtocolComplianceAssessor above
|
|
591
620
|
}
|
|
592
621
|
// Integrate temporal findings into security.vulnerabilities for unified view
|
|
@@ -668,6 +697,8 @@ export class AssessmentOrchestrator {
|
|
|
668
697
|
const crossCapabilityCount = this.crossCapabilityAssessor?.getTestCount() || 0;
|
|
669
698
|
// Code quality assessor counts
|
|
670
699
|
const fileModularizationCount = this.fileModularizationAssessor?.getTestCount() || 0;
|
|
700
|
+
// Official MCP conformance test count
|
|
701
|
+
const conformanceCount = this.conformanceAssessor?.getTestCount() || 0;
|
|
671
702
|
// Note: Protocol conformance now included in mcpSpecCount (unified ProtocolComplianceAssessor)
|
|
672
703
|
this.logger.debug("Test counts by assessor", {
|
|
673
704
|
functionality: functionalityCount,
|
|
@@ -688,6 +719,7 @@ export class AssessmentOrchestrator {
|
|
|
688
719
|
prompts: promptsCount,
|
|
689
720
|
crossCapability: crossCapabilityCount,
|
|
690
721
|
fileModularization: fileModularizationCount,
|
|
722
|
+
conformance: conformanceCount,
|
|
691
723
|
// Note: protocolConformance now included in mcpSpec (unified)
|
|
692
724
|
});
|
|
693
725
|
total =
|
|
@@ -708,7 +740,8 @@ export class AssessmentOrchestrator {
|
|
|
708
740
|
resourcesCount +
|
|
709
741
|
promptsCount +
|
|
710
742
|
crossCapabilityCount +
|
|
711
|
-
fileModularizationCount
|
|
743
|
+
fileModularizationCount +
|
|
744
|
+
conformanceCount;
|
|
712
745
|
// Note: protocolConformance now included in mcpSpecCount (unified)
|
|
713
746
|
this.logger.debug("Total test count", { total });
|
|
714
747
|
return total;
|
|
@@ -24,6 +24,16 @@ export interface ValidationContext {
|
|
|
24
24
|
scenarioCategory?: "happy_path" | "edge_case" | "boundary" | "error_case";
|
|
25
25
|
}
|
|
26
26
|
export declare class ResponseValidator {
|
|
27
|
+
/**
|
|
28
|
+
* Safely extract content array from response using Zod validation.
|
|
29
|
+
* Falls back to undefined if content is not a valid array.
|
|
30
|
+
*/
|
|
31
|
+
private static safeGetContentArray;
|
|
32
|
+
/**
|
|
33
|
+
* Safely parse MCP tool call result using Zod validation.
|
|
34
|
+
* Returns validated data or undefined if validation fails.
|
|
35
|
+
*/
|
|
36
|
+
private static safeGetMCPResponse;
|
|
27
37
|
/**
|
|
28
38
|
* Extract response metadata including content types, structuredContent, and _meta
|
|
29
39
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ResponseValidator.d.ts","sourceRoot":"","sources":["../../../src/services/assessment/ResponseValidator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,2BAA2B,EAC3B,IAAI,EACL,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"ResponseValidator.d.ts","sourceRoot":"","sources":["../../../src/services/assessment/ResponseValidator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,2BAA2B,EAC3B,IAAI,EACL,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAazD,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,cAAc,EACV,eAAe,GACf,mBAAmB,GACnB,mBAAmB,GACnB,QAAQ,GACR,OAAO,CAAC;IACZ,8EAA8E;IAC9E,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,QAAQ,EAAE,2BAA2B,CAAC;IACtC,gBAAgB,CAAC,EAAE,YAAY,GAAG,WAAW,GAAG,UAAU,GAAG,YAAY,CAAC;CAC3E;AAED,qBAAa,iBAAiB;IAC5B;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,mBAAmB;IAOlC;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAOjC;;OAEG;IACH,MAAM,CAAC,uBAAuB,CAAC,OAAO,EAAE,iBAAiB,GAAG,gBAAgB;IA2G5E;;OAEG;IACH,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,GAAG,gBAAgB;IAgHrE;;;OAGG;IACH,MAAM,CAAC,oBAAoB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO;IAgThE;;OAEG;IACH,MAAM,CAAC,0BAA0B,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,MAAM;CAsBvE"}
|
|
@@ -6,13 +6,31 @@
|
|
|
6
6
|
* @module assessment/ResponseValidator
|
|
7
7
|
*/
|
|
8
8
|
import { validateToolOutput, hasOutputSchema, tryExtractJsonFromContent, } from "../../utils/schemaUtils.js";
|
|
9
|
+
import { safeParseContentArray, safeParseMCPToolCallResult, } from "./responseValidatorSchemas.js";
|
|
9
10
|
export class ResponseValidator {
|
|
11
|
+
/**
|
|
12
|
+
* Safely extract content array from response using Zod validation.
|
|
13
|
+
* Falls back to undefined if content is not a valid array.
|
|
14
|
+
*/
|
|
15
|
+
static safeGetContentArray(response) {
|
|
16
|
+
const parseResult = safeParseContentArray(response.content);
|
|
17
|
+
return parseResult.success ? parseResult.data : undefined;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Safely parse MCP tool call result using Zod validation.
|
|
21
|
+
* Returns validated data or undefined if validation fails.
|
|
22
|
+
*/
|
|
23
|
+
static safeGetMCPResponse(response) {
|
|
24
|
+
const parseResult = safeParseMCPToolCallResult(response);
|
|
25
|
+
return parseResult.success ? parseResult.data : undefined;
|
|
26
|
+
}
|
|
10
27
|
/**
|
|
11
28
|
* Extract response metadata including content types, structuredContent, and _meta
|
|
12
29
|
*/
|
|
13
30
|
static extractResponseMetadata(context) {
|
|
14
|
-
|
|
15
|
-
const
|
|
31
|
+
// Use validated parsing for content array and full response
|
|
32
|
+
const content = this.safeGetContentArray(context.response);
|
|
33
|
+
const validatedResponse = this.safeGetMCPResponse(context.response);
|
|
16
34
|
// Track content types present
|
|
17
35
|
const contentTypes = [];
|
|
18
36
|
let textBlockCount = 0;
|
|
@@ -40,17 +58,23 @@ export class ResponseValidator {
|
|
|
40
58
|
}
|
|
41
59
|
}
|
|
42
60
|
// Check for structuredContent property (MCP 2024-11-05+)
|
|
43
|
-
|
|
44
|
-
|
|
61
|
+
// Use validated response data when available, fallback to raw response check
|
|
62
|
+
const hasStructuredContent = validatedResponse?.structuredContent !== undefined ||
|
|
63
|
+
("structuredContent" in context.response &&
|
|
64
|
+
context.response.structuredContent !== undefined);
|
|
45
65
|
// Check for _meta property
|
|
46
|
-
const hasMeta =
|
|
66
|
+
const hasMeta = validatedResponse?._meta !== undefined ||
|
|
67
|
+
("_meta" in context.response && context.response._meta !== undefined);
|
|
47
68
|
// Output schema validation
|
|
48
69
|
let outputSchemaValidation;
|
|
49
70
|
const toolHasOutputSchema = hasOutputSchema(context.tool.name);
|
|
50
71
|
if (toolHasOutputSchema) {
|
|
51
72
|
if (hasStructuredContent) {
|
|
52
73
|
// Primary path: validate structuredContent
|
|
53
|
-
|
|
74
|
+
// Prefer validated data, fallback to raw response
|
|
75
|
+
const structuredContent = validatedResponse?.structuredContent ??
|
|
76
|
+
context.response.structuredContent;
|
|
77
|
+
const validation = validateToolOutput(context.tool.name, structuredContent);
|
|
54
78
|
outputSchemaValidation = {
|
|
55
79
|
hasOutputSchema: true,
|
|
56
80
|
isValid: validation.isValid,
|
|
@@ -102,6 +102,8 @@ export declare const PERFORMANCE_PRESETS: {
|
|
|
102
102
|
* Validate a partial performance config.
|
|
103
103
|
* Ensures values are within reasonable bounds.
|
|
104
104
|
*
|
|
105
|
+
* Uses Zod schema validation under the hood (Issue #84).
|
|
106
|
+
*
|
|
105
107
|
* @public
|
|
106
108
|
* @param config - Partial config to validate
|
|
107
109
|
* @returns Array of validation error messages (empty if valid)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"performanceConfig.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/config/performanceConfig.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"performanceConfig.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/config/performanceConfig.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAG5C;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,oBAAoB,EAAE,MAAM,CAAC;IAE7B;;;;OAIG;IACH,sBAAsB,EAAE,MAAM,CAAC;IAE/B;;;OAGG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,aAAa,EAAE,MAAM,CAAC;IAEtB;;;;OAIG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAE9B;;;;;;;;;;OAUG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,wBAAwB,EAAE,MAAM,CAAC;CAClC;AAED;;;;GAIG;AACH,eAAO,MAAM,0BAA0B,EAAE,QAAQ,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CASzE,CAAC;AAEL;;;GAGG;AACH,eAAO,MAAM,mBAAmB;IAC9B,mDAAmD;;IAGnD,8CAA8C;;;;8BA1ExB,MAAM;uBAoBb,MAAM;+BAOE,MAAM;+BAaN,MAAM;kCAOH,MAAM;;IAkChC,kEAAkE;;;;;8BAjF5C,MAAM;uBAoBb,MAAM;+BAOE,MAAM;kCAoBH,MAAM;;CAyCxB,CAAC;AAEX;;;;;;;;;GASG;AACH,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,OAAO,CAAC,iBAAiB,CAAC,GACjC,MAAM,EAAE,CAGV;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAClC,QAAQ,CAAC,iBAAiB,CAAC,CAsB7B;AAED;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CACnC,UAAU,CAAC,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,MAAM,GACd,QAAQ,CAAC,iBAAiB,CAAC,CAyC7B"}
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* @see https://github.com/triepod-ai/inspector-assessment/issues/37
|
|
11
11
|
*/
|
|
12
12
|
import * as fs from "fs";
|
|
13
|
+
import { validatePerformanceConfigWithZod } from "./performanceConfigSchemas.js";
|
|
13
14
|
/**
|
|
14
15
|
* Default performance configuration.
|
|
15
16
|
* These values preserve existing behavior across all modules.
|
|
@@ -49,44 +50,15 @@ export const PERFORMANCE_PRESETS = {
|
|
|
49
50
|
* Validate a partial performance config.
|
|
50
51
|
* Ensures values are within reasonable bounds.
|
|
51
52
|
*
|
|
53
|
+
* Uses Zod schema validation under the hood (Issue #84).
|
|
54
|
+
*
|
|
52
55
|
* @public
|
|
53
56
|
* @param config - Partial config to validate
|
|
54
57
|
* @returns Array of validation error messages (empty if valid)
|
|
55
58
|
*/
|
|
56
59
|
export function validatePerformanceConfig(config) {
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
(config.batchFlushIntervalMs < 50 || config.batchFlushIntervalMs > 10000)) {
|
|
60
|
-
errors.push("batchFlushIntervalMs must be between 50 and 10000");
|
|
61
|
-
}
|
|
62
|
-
if (config.functionalityBatchSize !== undefined &&
|
|
63
|
-
(config.functionalityBatchSize < 1 || config.functionalityBatchSize > 100)) {
|
|
64
|
-
errors.push("functionalityBatchSize must be between 1 and 100");
|
|
65
|
-
}
|
|
66
|
-
if (config.securityBatchSize !== undefined &&
|
|
67
|
-
(config.securityBatchSize < 1 || config.securityBatchSize > 100)) {
|
|
68
|
-
errors.push("securityBatchSize must be between 1 and 100");
|
|
69
|
-
}
|
|
70
|
-
if (config.testTimeoutMs !== undefined &&
|
|
71
|
-
(config.testTimeoutMs < 100 || config.testTimeoutMs > 300000)) {
|
|
72
|
-
errors.push("testTimeoutMs must be between 100 and 300000");
|
|
73
|
-
}
|
|
74
|
-
if (config.securityTestTimeoutMs !== undefined &&
|
|
75
|
-
(config.securityTestTimeoutMs < 100 ||
|
|
76
|
-
config.securityTestTimeoutMs > 300000)) {
|
|
77
|
-
errors.push("securityTestTimeoutMs must be between 100 and 300000");
|
|
78
|
-
}
|
|
79
|
-
if (config.queueWarningThreshold !== undefined &&
|
|
80
|
-
(config.queueWarningThreshold < 100 ||
|
|
81
|
-
config.queueWarningThreshold > 1000000)) {
|
|
82
|
-
errors.push("queueWarningThreshold must be between 100 and 1000000");
|
|
83
|
-
}
|
|
84
|
-
if (config.eventEmitterMaxListeners !== undefined &&
|
|
85
|
-
(config.eventEmitterMaxListeners < 10 ||
|
|
86
|
-
config.eventEmitterMaxListeners > 1000)) {
|
|
87
|
-
errors.push("eventEmitterMaxListeners must be between 10 and 1000");
|
|
88
|
-
}
|
|
89
|
-
return errors;
|
|
60
|
+
// Delegate to Zod schema validation
|
|
61
|
+
return validatePerformanceConfigWithZod(config);
|
|
90
62
|
}
|
|
91
63
|
/**
|
|
92
64
|
* Merge a partial config with defaults.
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Zod Schemas for Performance Configuration
|
|
3
|
+
*
|
|
4
|
+
* Runtime validation schemas for performance configuration.
|
|
5
|
+
* Replaces manual validatePerformanceConfig() function with declarative schemas.
|
|
6
|
+
*
|
|
7
|
+
* @module assessment/config/performanceConfigSchemas
|
|
8
|
+
* @see performanceConfig.ts for the interface definitions
|
|
9
|
+
* @see sharedSchemas.ts for PERF_CONFIG_RANGES constants
|
|
10
|
+
*/
|
|
11
|
+
import { z } from "zod";
|
|
12
|
+
import { PERF_CONFIG_RANGES } from "../../../lib/assessment/sharedSchemas.js";
|
|
13
|
+
export { PERF_CONFIG_RANGES };
|
|
14
|
+
/**
|
|
15
|
+
* Schema for performance configuration fields.
|
|
16
|
+
* All fields are optional since partial configs are merged with defaults.
|
|
17
|
+
*
|
|
18
|
+
* Validation ranges are defined in PERF_CONFIG_RANGES (sharedSchemas.ts).
|
|
19
|
+
*/
|
|
20
|
+
export declare const PerformanceConfigSchema: z.ZodObject<{
|
|
21
|
+
/**
|
|
22
|
+
* Interval in milliseconds between progress batch flushes.
|
|
23
|
+
*/
|
|
24
|
+
batchFlushIntervalMs: z.ZodOptional<z.ZodNumber>;
|
|
25
|
+
/**
|
|
26
|
+
* Batch size for functionality assessment progress events.
|
|
27
|
+
*/
|
|
28
|
+
functionalityBatchSize: z.ZodOptional<z.ZodNumber>;
|
|
29
|
+
/**
|
|
30
|
+
* Batch size for security assessment progress events.
|
|
31
|
+
*/
|
|
32
|
+
securityBatchSize: z.ZodOptional<z.ZodNumber>;
|
|
33
|
+
/**
|
|
34
|
+
* Timeout for individual test scenario execution in milliseconds.
|
|
35
|
+
*/
|
|
36
|
+
testTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
37
|
+
/**
|
|
38
|
+
* Timeout for individual security payload tests in milliseconds.
|
|
39
|
+
*/
|
|
40
|
+
securityTestTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
41
|
+
/**
|
|
42
|
+
* Warning threshold for queue depth monitoring.
|
|
43
|
+
*/
|
|
44
|
+
queueWarningThreshold: z.ZodOptional<z.ZodNumber>;
|
|
45
|
+
/**
|
|
46
|
+
* Maximum EventEmitter listeners to prevent Node.js warnings.
|
|
47
|
+
*/
|
|
48
|
+
eventEmitterMaxListeners: z.ZodOptional<z.ZodNumber>;
|
|
49
|
+
}, "strip", z.ZodTypeAny, {
|
|
50
|
+
batchFlushIntervalMs?: number;
|
|
51
|
+
functionalityBatchSize?: number;
|
|
52
|
+
securityBatchSize?: number;
|
|
53
|
+
testTimeoutMs?: number;
|
|
54
|
+
securityTestTimeoutMs?: number;
|
|
55
|
+
queueWarningThreshold?: number;
|
|
56
|
+
eventEmitterMaxListeners?: number;
|
|
57
|
+
}, {
|
|
58
|
+
batchFlushIntervalMs?: number;
|
|
59
|
+
functionalityBatchSize?: number;
|
|
60
|
+
securityBatchSize?: number;
|
|
61
|
+
testTimeoutMs?: number;
|
|
62
|
+
securityTestTimeoutMs?: number;
|
|
63
|
+
queueWarningThreshold?: number;
|
|
64
|
+
eventEmitterMaxListeners?: number;
|
|
65
|
+
}>;
|
|
66
|
+
/**
|
|
67
|
+
* Type inferred from the schema.
|
|
68
|
+
* Equivalent to Partial<PerformanceConfig> from performanceConfig.ts
|
|
69
|
+
*/
|
|
70
|
+
export type PartialPerformanceConfig = z.infer<typeof PerformanceConfigSchema>;
|
|
71
|
+
/**
|
|
72
|
+
* Validate a partial performance config using Zod.
|
|
73
|
+
* Drop-in replacement for the manual validatePerformanceConfig() function.
|
|
74
|
+
*
|
|
75
|
+
* @param config - Partial config to validate
|
|
76
|
+
* @returns Array of validation error messages (empty if valid)
|
|
77
|
+
*/
|
|
78
|
+
export declare function validatePerformanceConfigWithZod(config: unknown): string[];
|
|
79
|
+
/**
|
|
80
|
+
* Parse and validate a performance config, returning the validated data.
|
|
81
|
+
* Throws ZodError if validation fails.
|
|
82
|
+
*
|
|
83
|
+
* @param config - Config to parse and validate
|
|
84
|
+
* @returns Validated partial config
|
|
85
|
+
* @throws ZodError if validation fails
|
|
86
|
+
*/
|
|
87
|
+
export declare function parsePerformanceConfig(config: unknown): PartialPerformanceConfig;
|
|
88
|
+
/**
|
|
89
|
+
* Safely parse a performance config without throwing.
|
|
90
|
+
*
|
|
91
|
+
* @param config - Config to parse and validate
|
|
92
|
+
* @returns SafeParseResult with success status and data/error
|
|
93
|
+
*/
|
|
94
|
+
export declare function safeParsePerformanceConfig(config: unknown): z.SafeParseReturnType<{
|
|
95
|
+
batchFlushIntervalMs?: number;
|
|
96
|
+
functionalityBatchSize?: number;
|
|
97
|
+
securityBatchSize?: number;
|
|
98
|
+
testTimeoutMs?: number;
|
|
99
|
+
securityTestTimeoutMs?: number;
|
|
100
|
+
queueWarningThreshold?: number;
|
|
101
|
+
eventEmitterMaxListeners?: number;
|
|
102
|
+
}, {
|
|
103
|
+
batchFlushIntervalMs?: number;
|
|
104
|
+
functionalityBatchSize?: number;
|
|
105
|
+
securityBatchSize?: number;
|
|
106
|
+
testTimeoutMs?: number;
|
|
107
|
+
securityTestTimeoutMs?: number;
|
|
108
|
+
queueWarningThreshold?: number;
|
|
109
|
+
eventEmitterMaxListeners?: number;
|
|
110
|
+
}>;
|
|
111
|
+
//# sourceMappingURL=performanceConfigSchemas.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"performanceConfigSchemas.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/config/performanceConfigSchemas.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAG3E,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB;IAClC;;OAEG;;IAcH;;OAEG;;IAcH;;OAEG;;IAcH;;OAEG;;IAcH;;OAEG;;IAcH;;OAEG;;IAcH;;OAEG;;;;;;;;;;;;;;;;;;EAaH,CAAC;AAEH;;;GAGG;AACH,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAE/E;;;;;;GAMG;AACH,wBAAgB,gCAAgC,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,EAAE,CAW1E;AAED;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,OAAO,GACd,wBAAwB,CAE1B;AAED;;;;;GAKG;AACH,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,OAAO;;;;;;;;;;;;;;;;GAEzD"}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Zod Schemas for Performance Configuration
|
|
3
|
+
*
|
|
4
|
+
* Runtime validation schemas for performance configuration.
|
|
5
|
+
* Replaces manual validatePerformanceConfig() function with declarative schemas.
|
|
6
|
+
*
|
|
7
|
+
* @module assessment/config/performanceConfigSchemas
|
|
8
|
+
* @see performanceConfig.ts for the interface definitions
|
|
9
|
+
* @see sharedSchemas.ts for PERF_CONFIG_RANGES constants
|
|
10
|
+
*/
|
|
11
|
+
import { z } from "zod";
|
|
12
|
+
// Import validation range constants from single source of truth
|
|
13
|
+
import { PERF_CONFIG_RANGES } from "../../../lib/assessment/sharedSchemas.js";
|
|
14
|
+
// Re-export for consumers who need the range constants
|
|
15
|
+
export { PERF_CONFIG_RANGES };
|
|
16
|
+
/**
|
|
17
|
+
* Schema for performance configuration fields.
|
|
18
|
+
* All fields are optional since partial configs are merged with defaults.
|
|
19
|
+
*
|
|
20
|
+
* Validation ranges are defined in PERF_CONFIG_RANGES (sharedSchemas.ts).
|
|
21
|
+
*/
|
|
22
|
+
export const PerformanceConfigSchema = z.object({
|
|
23
|
+
/**
|
|
24
|
+
* Interval in milliseconds between progress batch flushes.
|
|
25
|
+
*/
|
|
26
|
+
batchFlushIntervalMs: z
|
|
27
|
+
.number()
|
|
28
|
+
.int("batchFlushIntervalMs must be an integer")
|
|
29
|
+
.min(PERF_CONFIG_RANGES.batchFlushIntervalMs.min, `batchFlushIntervalMs must be >= ${PERF_CONFIG_RANGES.batchFlushIntervalMs.min}`)
|
|
30
|
+
.max(PERF_CONFIG_RANGES.batchFlushIntervalMs.max, `batchFlushIntervalMs must be <= ${PERF_CONFIG_RANGES.batchFlushIntervalMs.max}`)
|
|
31
|
+
.optional(),
|
|
32
|
+
/**
|
|
33
|
+
* Batch size for functionality assessment progress events.
|
|
34
|
+
*/
|
|
35
|
+
functionalityBatchSize: z
|
|
36
|
+
.number()
|
|
37
|
+
.int("functionalityBatchSize must be an integer")
|
|
38
|
+
.min(PERF_CONFIG_RANGES.functionalityBatchSize.min, `functionalityBatchSize must be >= ${PERF_CONFIG_RANGES.functionalityBatchSize.min}`)
|
|
39
|
+
.max(PERF_CONFIG_RANGES.functionalityBatchSize.max, `functionalityBatchSize must be <= ${PERF_CONFIG_RANGES.functionalityBatchSize.max}`)
|
|
40
|
+
.optional(),
|
|
41
|
+
/**
|
|
42
|
+
* Batch size for security assessment progress events.
|
|
43
|
+
*/
|
|
44
|
+
securityBatchSize: z
|
|
45
|
+
.number()
|
|
46
|
+
.int("securityBatchSize must be an integer")
|
|
47
|
+
.min(PERF_CONFIG_RANGES.securityBatchSize.min, `securityBatchSize must be >= ${PERF_CONFIG_RANGES.securityBatchSize.min}`)
|
|
48
|
+
.max(PERF_CONFIG_RANGES.securityBatchSize.max, `securityBatchSize must be <= ${PERF_CONFIG_RANGES.securityBatchSize.max}`)
|
|
49
|
+
.optional(),
|
|
50
|
+
/**
|
|
51
|
+
* Timeout for individual test scenario execution in milliseconds.
|
|
52
|
+
*/
|
|
53
|
+
testTimeoutMs: z
|
|
54
|
+
.number()
|
|
55
|
+
.int("testTimeoutMs must be an integer")
|
|
56
|
+
.min(PERF_CONFIG_RANGES.testTimeoutMs.min, `testTimeoutMs must be >= ${PERF_CONFIG_RANGES.testTimeoutMs.min}`)
|
|
57
|
+
.max(PERF_CONFIG_RANGES.testTimeoutMs.max, `testTimeoutMs must be <= ${PERF_CONFIG_RANGES.testTimeoutMs.max}`)
|
|
58
|
+
.optional(),
|
|
59
|
+
/**
|
|
60
|
+
* Timeout for individual security payload tests in milliseconds.
|
|
61
|
+
*/
|
|
62
|
+
securityTestTimeoutMs: z
|
|
63
|
+
.number()
|
|
64
|
+
.int("securityTestTimeoutMs must be an integer")
|
|
65
|
+
.min(PERF_CONFIG_RANGES.securityTestTimeoutMs.min, `securityTestTimeoutMs must be >= ${PERF_CONFIG_RANGES.securityTestTimeoutMs.min}`)
|
|
66
|
+
.max(PERF_CONFIG_RANGES.securityTestTimeoutMs.max, `securityTestTimeoutMs must be <= ${PERF_CONFIG_RANGES.securityTestTimeoutMs.max}`)
|
|
67
|
+
.optional(),
|
|
68
|
+
/**
|
|
69
|
+
* Warning threshold for queue depth monitoring.
|
|
70
|
+
*/
|
|
71
|
+
queueWarningThreshold: z
|
|
72
|
+
.number()
|
|
73
|
+
.int("queueWarningThreshold must be an integer")
|
|
74
|
+
.min(PERF_CONFIG_RANGES.queueWarningThreshold.min, `queueWarningThreshold must be >= ${PERF_CONFIG_RANGES.queueWarningThreshold.min}`)
|
|
75
|
+
.max(PERF_CONFIG_RANGES.queueWarningThreshold.max, `queueWarningThreshold must be <= ${PERF_CONFIG_RANGES.queueWarningThreshold.max}`)
|
|
76
|
+
.optional(),
|
|
77
|
+
/**
|
|
78
|
+
* Maximum EventEmitter listeners to prevent Node.js warnings.
|
|
79
|
+
*/
|
|
80
|
+
eventEmitterMaxListeners: z
|
|
81
|
+
.number()
|
|
82
|
+
.int("eventEmitterMaxListeners must be an integer")
|
|
83
|
+
.min(PERF_CONFIG_RANGES.eventEmitterMaxListeners.min, `eventEmitterMaxListeners must be >= ${PERF_CONFIG_RANGES.eventEmitterMaxListeners.min}`)
|
|
84
|
+
.max(PERF_CONFIG_RANGES.eventEmitterMaxListeners.max, `eventEmitterMaxListeners must be <= ${PERF_CONFIG_RANGES.eventEmitterMaxListeners.max}`)
|
|
85
|
+
.optional(),
|
|
86
|
+
});
|
|
87
|
+
/**
|
|
88
|
+
* Validate a partial performance config using Zod.
|
|
89
|
+
* Drop-in replacement for the manual validatePerformanceConfig() function.
|
|
90
|
+
*
|
|
91
|
+
* @param config - Partial config to validate
|
|
92
|
+
* @returns Array of validation error messages (empty if valid)
|
|
93
|
+
*/
|
|
94
|
+
export function validatePerformanceConfigWithZod(config) {
|
|
95
|
+
const result = PerformanceConfigSchema.safeParse(config);
|
|
96
|
+
if (result.success) {
|
|
97
|
+
return [];
|
|
98
|
+
}
|
|
99
|
+
return result.error.errors.map((e) => {
|
|
100
|
+
const path = e.path.length > 0 ? `${e.path.join(".")}: ` : "";
|
|
101
|
+
return `${path}${e.message}`;
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Parse and validate a performance config, returning the validated data.
|
|
106
|
+
* Throws ZodError if validation fails.
|
|
107
|
+
*
|
|
108
|
+
* @param config - Config to parse and validate
|
|
109
|
+
* @returns Validated partial config
|
|
110
|
+
* @throws ZodError if validation fails
|
|
111
|
+
*/
|
|
112
|
+
export function parsePerformanceConfig(config) {
|
|
113
|
+
return PerformanceConfigSchema.parse(config);
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Safely parse a performance config without throwing.
|
|
117
|
+
*
|
|
118
|
+
* @param config - Config to parse and validate
|
|
119
|
+
* @returns SafeParseResult with success status and data/error
|
|
120
|
+
*/
|
|
121
|
+
export function safeParsePerformanceConfig(config) {
|
|
122
|
+
return PerformanceConfigSchema.safeParse(config);
|
|
123
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conformance Assessor Module
|
|
3
|
+
*
|
|
4
|
+
* Integrates official MCP conformance tests from @modelcontextprotocol/conformance.
|
|
5
|
+
* Runs server-side conformance validation against the MCP specification.
|
|
6
|
+
*
|
|
7
|
+
* Requirements:
|
|
8
|
+
* - HTTP/SSE transport (requires serverUrl in config)
|
|
9
|
+
* - Opt-in via --conformance flag or assessmentCategories.conformance = true
|
|
10
|
+
*
|
|
11
|
+
* @module assessment/modules/ConformanceAssessor
|
|
12
|
+
*/
|
|
13
|
+
import { BaseAssessor } from "./BaseAssessor.js";
|
|
14
|
+
import { AssessmentContext } from "../AssessmentOrchestrator.js";
|
|
15
|
+
import type { ConformanceAssessment } from "../../../lib/assessment/extendedTypes.js";
|
|
16
|
+
/**
|
|
17
|
+
* Conformance Assessor
|
|
18
|
+
*
|
|
19
|
+
* Runs official MCP conformance tests against the server.
|
|
20
|
+
* Requires HTTP/SSE transport with serverUrl available.
|
|
21
|
+
*/
|
|
22
|
+
export declare class ConformanceAssessor extends BaseAssessor<ConformanceAssessment> {
|
|
23
|
+
/**
|
|
24
|
+
* Run conformance assessment
|
|
25
|
+
*/
|
|
26
|
+
assess(context: AssessmentContext): Promise<ConformanceAssessment>;
|
|
27
|
+
/**
|
|
28
|
+
* Run a single conformance scenario
|
|
29
|
+
*/
|
|
30
|
+
private runScenario;
|
|
31
|
+
/**
|
|
32
|
+
* Find the checks.json file in the results directory
|
|
33
|
+
*/
|
|
34
|
+
private findChecksFile;
|
|
35
|
+
/**
|
|
36
|
+
* Parse checks.json file from conformance results
|
|
37
|
+
*/
|
|
38
|
+
private parseChecksFile;
|
|
39
|
+
/**
|
|
40
|
+
* Cleanup temporary directory
|
|
41
|
+
*/
|
|
42
|
+
private cleanupTempDir;
|
|
43
|
+
/**
|
|
44
|
+
* Determine overall conformance status
|
|
45
|
+
*/
|
|
46
|
+
private determineConformanceStatus;
|
|
47
|
+
/**
|
|
48
|
+
* Generate human-readable explanation
|
|
49
|
+
*/
|
|
50
|
+
private generateExplanation;
|
|
51
|
+
/**
|
|
52
|
+
* Generate recommendations based on failures
|
|
53
|
+
*/
|
|
54
|
+
private generateRecommendations;
|
|
55
|
+
/**
|
|
56
|
+
* Create a skipped result when conformance tests cannot run
|
|
57
|
+
*/
|
|
58
|
+
private createSkippedResult;
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=ConformanceAssessor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ConformanceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ConformanceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAOH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EACV,qBAAqB,EAGtB,MAAM,gCAAgC,CAAC;AAgCxC;;;;;GAKG;AACH,qBAAa,mBAAoB,SAAQ,YAAY,CAAC,qBAAqB,CAAC;IAC1E;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwFxE;;OAEG;YACW,WAAW;IAwEzB;;OAEG;IACH,OAAO,CAAC,cAAc;IAwBtB;;OAEG;IACH,OAAO,CAAC,eAAe;IAmBvB;;OAEG;IACH,OAAO,CAAC,cAAc;IAQtB;;OAEG;IACH,OAAO,CAAC,0BAA0B;IA+BlC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAyB3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA+C/B;;OAEG;IACH,OAAO,CAAC,mBAAmB;CAmB5B"}
|