@bryan-thompson/inspector-assessment 1.36.5 → 1.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/build/lib/assessment-runner/assessment-executor.js +40 -0
- package/cli/build/lib/assessment-runner/source-loader.js +11 -0
- package/cli/package.json +1 -1
- package/client/dist/assets/{OAuthCallback-DJ1av7om.js → OAuthCallback-AngeBaCl.js} +1 -1
- package/client/dist/assets/{OAuthDebugCallback-lRXgX7wV.js → OAuthDebugCallback--FE6_fPs.js} +1 -1
- package/client/dist/assets/{index-DEdS99fp.js → index-BQC95Boo.js} +4 -4
- package/client/dist/index.html +1 -1
- package/client/lib/lib/assessment/coreTypes.d.ts +37 -0
- package/client/lib/lib/assessment/coreTypes.d.ts.map +1 -1
- package/client/lib/lib/assessment/resultTypes.d.ts +30 -1
- package/client/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/client/lib/lib/assessment/sharedSchemas.d.ts +10 -0
- package/client/lib/lib/assessment/sharedSchemas.d.ts.map +1 -1
- package/client/lib/lib/assessment/sharedSchemas.js +4 -0
- package/client/lib/lib/securityPatterns/advancedExploitPatterns.d.ts +13 -0
- package/client/lib/lib/securityPatterns/advancedExploitPatterns.d.ts.map +1 -0
- package/client/lib/lib/securityPatterns/advancedExploitPatterns.js +504 -0
- package/client/lib/lib/securityPatterns/authSessionPatterns.d.ts +12 -0
- package/client/lib/lib/securityPatterns/authSessionPatterns.d.ts.map +1 -0
- package/client/lib/lib/securityPatterns/authSessionPatterns.js +357 -0
- package/client/lib/lib/securityPatterns/index.d.ts +18 -0
- package/client/lib/lib/securityPatterns/index.d.ts.map +1 -0
- package/client/lib/lib/securityPatterns/index.js +18 -0
- package/client/lib/lib/securityPatterns/injectionPatterns.d.ts +13 -0
- package/client/lib/lib/securityPatterns/injectionPatterns.d.ts.map +1 -0
- package/client/lib/lib/securityPatterns/injectionPatterns.js +356 -0
- package/client/lib/lib/securityPatterns/resourceExhaustionPatterns.d.ts +12 -0
- package/client/lib/lib/securityPatterns/resourceExhaustionPatterns.d.ts.map +1 -0
- package/client/lib/lib/securityPatterns/resourceExhaustionPatterns.js +215 -0
- package/client/lib/lib/securityPatterns/toolSpecificPatterns.d.ts +13 -0
- package/client/lib/lib/securityPatterns/toolSpecificPatterns.d.ts.map +1 -0
- package/client/lib/lib/securityPatterns/toolSpecificPatterns.js +373 -0
- package/client/lib/lib/securityPatterns/types.d.ts +20 -0
- package/client/lib/lib/securityPatterns/types.d.ts.map +1 -0
- package/client/lib/lib/securityPatterns/types.js +6 -0
- package/client/lib/lib/securityPatterns/utils.d.ts +56 -0
- package/client/lib/lib/securityPatterns/utils.d.ts.map +1 -0
- package/client/lib/lib/securityPatterns/utils.js +96 -0
- package/client/lib/lib/securityPatterns/validationPatterns.d.ts +13 -0
- package/client/lib/lib/securityPatterns/validationPatterns.d.ts.map +1 -0
- package/client/lib/lib/securityPatterns/validationPatterns.js +110 -0
- package/client/lib/lib/securityPatterns.d.ts +18 -69
- package/client/lib/lib/securityPatterns.d.ts.map +1 -1
- package/client/lib/lib/securityPatterns.js +18 -1946
- package/client/lib/services/assessment/AssessmentOrchestrator.d.ts +6 -1
- package/client/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/client/lib/services/assessment/config/performanceConfig.d.ts +18 -0
- package/client/lib/services/assessment/config/performanceConfig.d.ts.map +1 -1
- package/client/lib/services/assessment/config/performanceConfig.js +6 -0
- package/client/lib/services/assessment/config/performanceConfigSchemas.d.ts +18 -0
- package/client/lib/services/assessment/config/performanceConfigSchemas.d.ts.map +1 -1
- package/client/lib/services/assessment/config/performanceConfigSchemas.js +20 -0
- package/client/lib/services/assessment/helpers/ExternalAPIDependencyDetector.d.ts +165 -0
- package/client/lib/services/assessment/helpers/ExternalAPIDependencyDetector.d.ts.map +1 -0
- package/client/lib/services/assessment/helpers/ExternalAPIDependencyDetector.js +317 -0
- package/client/lib/services/assessment/helpers/StdioTransportDetector.d.ts +137 -0
- package/client/lib/services/assessment/helpers/StdioTransportDetector.d.ts.map +1 -0
- package/client/lib/services/assessment/helpers/StdioTransportDetector.js +315 -0
- package/client/lib/services/assessment/helpers/ToolAnnotationExtractor.d.ts +34 -0
- package/client/lib/services/assessment/helpers/ToolAnnotationExtractor.d.ts.map +1 -0
- package/client/lib/services/assessment/helpers/ToolAnnotationExtractor.js +85 -0
- package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts +23 -0
- package/client/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/ErrorHandlingAssessor.js +255 -20
- package/client/lib/services/assessment/modules/FunctionalityAssessor.d.ts +10 -0
- package/client/lib/services/assessment/modules/FunctionalityAssessor.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/FunctionalityAssessor.js +65 -3
- package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/ProtocolComplianceAssessor.js +30 -0
- package/client/lib/services/assessment/modules/SecurityAssessor.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/SecurityAssessor.js +6 -0
- package/client/lib/services/assessment/modules/TemporalAssessor.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/TemporalAssessor.js +16 -3
- package/client/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/annotations/AlignmentChecker.js +6 -2
- package/client/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.js +16 -7
- package/client/lib/services/assessment/modules/securityTests/AnnotationAwareSeverity.d.ts +55 -0
- package/client/lib/services/assessment/modules/securityTests/AnnotationAwareSeverity.d.ts.map +1 -0
- package/client/lib/services/assessment/modules/securityTests/AnnotationAwareSeverity.js +135 -0
- package/client/lib/services/assessment/modules/securityTests/ErrorClassifier.d.ts +14 -0
- package/client/lib/services/assessment/modules/securityTests/ErrorClassifier.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/securityTests/ErrorClassifier.js +24 -1
- package/client/lib/services/assessment/modules/securityTests/SafeResponseDetector.d.ts +6 -0
- package/client/lib/services/assessment/modules/securityTests/SafeResponseDetector.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/securityTests/SafeResponseDetector.js +9 -1
- package/client/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.d.ts +43 -1
- package/client/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.js +87 -1
- package/client/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts +39 -1
- package/client/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/securityTests/SecurityPayloadTester.js +93 -3
- package/client/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts +1 -1
- package/client/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.js +10 -1
- package/client/lib/services/assessment/modules/securityTests/index.d.ts +1 -0
- package/client/lib/services/assessment/modules/securityTests/index.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/securityTests/index.js +1 -0
- package/client/lib/services/assessment/modules/temporal/VarianceClassifier.d.ts +16 -1
- package/client/lib/services/assessment/modules/temporal/VarianceClassifier.d.ts.map +1 -1
- package/client/lib/services/assessment/modules/temporal/VarianceClassifier.js +43 -1
- package/client/package.json +1 -1
- package/package.json +1 -1
- package/server/package.json +1 -1
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Extracted from SecurityAssessor.ts for maintainability.
|
|
6
6
|
* Handles test execution, batching, and progress tracking.
|
|
7
7
|
*/
|
|
8
|
-
import { SecurityTestResult } from "../../../../lib/assessmentTypes.js";
|
|
8
|
+
import { SecurityTestResult, ToolAnnotationsContext } from "../../../../lib/assessmentTypes.js";
|
|
9
9
|
import { ProgressCallback } from "../../../../lib/assessment/progressTypes.js";
|
|
10
10
|
import { CompatibilityCallToolResult, Tool } from "@modelcontextprotocol/sdk/types.js";
|
|
11
11
|
import { SecurityPayload } from "../../../../lib/securityPatterns.js";
|
|
@@ -21,6 +21,21 @@ export interface PayloadTestConfig {
|
|
|
21
21
|
maxParallelTests?: number;
|
|
22
22
|
securityTestTimeout?: number;
|
|
23
23
|
selectedToolsForTesting?: string[];
|
|
24
|
+
/**
|
|
25
|
+
* Maximum retry attempts for transient errors (Issue #157)
|
|
26
|
+
* Uses PerformanceConfig.securityRetryMaxAttempts if not specified
|
|
27
|
+
*/
|
|
28
|
+
securityRetryMaxAttempts?: number;
|
|
29
|
+
/**
|
|
30
|
+
* Initial backoff delay in ms for retries (Issue #157)
|
|
31
|
+
* Uses PerformanceConfig.securityRetryBackoffMs if not specified
|
|
32
|
+
*/
|
|
33
|
+
securityRetryBackoffMs?: number;
|
|
34
|
+
/**
|
|
35
|
+
* Tool annotations context for severity adjustment (Issue #170)
|
|
36
|
+
* When provided, enables annotation-aware false positive reduction
|
|
37
|
+
*/
|
|
38
|
+
toolAnnotationsContext?: ToolAnnotationsContext;
|
|
24
39
|
}
|
|
25
40
|
/**
|
|
26
41
|
* Logger interface for test execution
|
|
@@ -41,6 +56,11 @@ export declare class SecurityPayloadTester {
|
|
|
41
56
|
private sanitizationDetector;
|
|
42
57
|
private testCount;
|
|
43
58
|
constructor(config: PayloadTestConfig, logger: TestLogger, executeWithTimeout: <T>(promise: Promise<T>, timeout: number) => Promise<T>);
|
|
59
|
+
/**
|
|
60
|
+
* Set tool annotations context for severity adjustment (Issue #170)
|
|
61
|
+
* Call before running tests to enable annotation-aware false positive reduction
|
|
62
|
+
*/
|
|
63
|
+
setToolAnnotationsContext(context: ToolAnnotationsContext | undefined): void;
|
|
44
64
|
/**
|
|
45
65
|
* Run comprehensive security tests (advanced mode)
|
|
46
66
|
* Tests selected tools with ALL 23 security patterns using diverse payloads
|
|
@@ -55,6 +75,24 @@ export declare class SecurityPayloadTester {
|
|
|
55
75
|
* Test tool with a specific payload
|
|
56
76
|
*/
|
|
57
77
|
testPayload(tool: Tool, attackName: string, payload: SecurityPayload, callTool: (name: string, params: Record<string, unknown>) => Promise<CompatibilityCallToolResult>): Promise<SecurityTestResult>;
|
|
78
|
+
/**
|
|
79
|
+
* Test payload with retry logic for transient errors.
|
|
80
|
+
* Implements exponential backoff: 100ms → 200ms → 400ms
|
|
81
|
+
*
|
|
82
|
+
* Issue #157: Connection retry logic for reliability
|
|
83
|
+
*
|
|
84
|
+
* @param tool - Tool to test
|
|
85
|
+
* @param attackName - Name of attack pattern
|
|
86
|
+
* @param payload - Security payload to test
|
|
87
|
+
* @param callTool - Function to call the tool
|
|
88
|
+
* @returns SecurityTestResult with retry metadata if applicable
|
|
89
|
+
*/
|
|
90
|
+
testPayloadWithRetry(tool: Tool, attackName: string, payload: SecurityPayload, callTool: (name: string, params: Record<string, unknown>) => Promise<CompatibilityCallToolResult>): Promise<SecurityTestResult>;
|
|
91
|
+
/**
|
|
92
|
+
* Add retry metadata to result.
|
|
93
|
+
* Issue #157: Track retry attempts for reliability metrics
|
|
94
|
+
*/
|
|
95
|
+
private addRetryMetadata;
|
|
58
96
|
/**
|
|
59
97
|
* Extract error message from caught exception
|
|
60
98
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SecurityPayloadTester.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/securityTests/SecurityPayloadTester.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,
|
|
1
|
+
{"version":3,"file":"SecurityPayloadTester.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/securityTests/SecurityPayloadTester.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACvB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,gBAAgB,EAIjB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EACL,2BAA2B,EAC3B,IAAI,EACL,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAGL,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAQhC;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,gBAAgB,CAAC;AAEpD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,uBAAuB,CAAC,EAAE,MAAM,EAAE,CAAC;IACnC;;;OAGG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC;;;OAGG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC;;;OAGG;IACH,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;CACjD;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAC/B,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;CACrD;AAED;;GAEG;AACH,qBAAa,qBAAqB;IAO9B,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,kBAAkB;IAR5B,OAAO,CAAC,gBAAgB,CAA2B;IACnD,OAAO,CAAC,gBAAgB,CAA2B;IACnD,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,SAAS,CAAK;gBAGZ,MAAM,EAAE,iBAAiB,EACzB,MAAM,EAAE,UAAU,EAClB,kBAAkB,EAAE,CAAC,CAAC,EAC5B,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,EACnB,OAAO,EAAE,MAAM,KACZ,OAAO,CAAC,CAAC,CAAC;IAOjB;;;OAGG;IACH,yBAAyB,CAAC,OAAO,EAAE,sBAAsB,GAAG,SAAS,GAAG,IAAI;IAI5E;;;OAGG;IACG,yBAAyB,CAC7B,KAAK,EAAE,IAAI,EAAE,EACb,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC5B,OAAO,CAAC,2BAA2B,CAAC,EACzC,UAAU,CAAC,EAAE,oBAAoB,GAChC,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAsMhC;;;OAGG;IACG,qBAAqB,CACzB,KAAK,EAAE,IAAI,EAAE,EACb,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC5B,OAAO,CAAC,2BAA2B,CAAC,EACzC,UAAU,CAAC,EAAE,oBAAoB,GAChC,OAAO,CAAC,kBAAkB,EAAE,CAAC;IA8LhC;;OAEG;IACG,WAAW,CACf,IAAI,EAAE,IAAI,EACV,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,eAAe,EACxB,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC5B,OAAO,CAAC,2BAA2B,CAAC,GACxC,OAAO,CAAC,kBAAkB,CAAC;IAqT9B;;;;;;;;;;;OAWG;IACG,oBAAoB,CACxB,IAAI,EAAE,IAAI,EACV,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,eAAe,EACxB,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC5B,OAAO,CAAC,2BAA2B,CAAC,GACxC,OAAO,CAAC,kBAAkB,CAAC;IAqD9B;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAqBxB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAO3B;;OAEG;IACH,OAAO,CAAC,KAAK;CAGd"}
|
|
@@ -5,12 +5,14 @@
|
|
|
5
5
|
* Extracted from SecurityAssessor.ts for maintainability.
|
|
6
6
|
* Handles test execution, batching, and progress tracking.
|
|
7
7
|
*/
|
|
8
|
+
import { adjustSeverityForAnnotations } from "./AnnotationAwareSeverity.js";
|
|
8
9
|
import { getAllAttackPatterns, getPayloadsForAttack, } from "../../../../lib/securityPatterns.js";
|
|
9
10
|
import { createConcurrencyLimit } from "../../lib/concurrencyLimit.js";
|
|
10
11
|
import { SecurityResponseAnalyzer } from "./SecurityResponseAnalyzer.js";
|
|
11
12
|
import { SecurityPayloadGenerator } from "./SecurityPayloadGenerator.js";
|
|
12
13
|
import { SanitizationDetector } from "./SanitizationDetector.js";
|
|
13
14
|
import { DEFAULT_PERFORMANCE_CONFIG } from "../../config/performanceConfig.js";
|
|
15
|
+
import { isTransientErrorPattern } from "./SecurityPatternLibrary.js";
|
|
14
16
|
/**
|
|
15
17
|
* Executes security tests with payloads against MCP tools
|
|
16
18
|
*/
|
|
@@ -30,6 +32,13 @@ export class SecurityPayloadTester {
|
|
|
30
32
|
this.payloadGenerator = new SecurityPayloadGenerator();
|
|
31
33
|
this.sanitizationDetector = new SanitizationDetector();
|
|
32
34
|
}
|
|
35
|
+
/**
|
|
36
|
+
* Set tool annotations context for severity adjustment (Issue #170)
|
|
37
|
+
* Call before running tests to enable annotation-aware false positive reduction
|
|
38
|
+
*/
|
|
39
|
+
setToolAnnotationsContext(context) {
|
|
40
|
+
this.config.toolAnnotationsContext = context;
|
|
41
|
+
}
|
|
33
42
|
/**
|
|
34
43
|
* Run comprehensive security tests (advanced mode)
|
|
35
44
|
* Tests selected tools with ALL 23 security patterns using diverse payloads
|
|
@@ -117,7 +126,8 @@ export class SecurityPayloadTester {
|
|
|
117
126
|
completedTests++;
|
|
118
127
|
batchCount++;
|
|
119
128
|
try {
|
|
120
|
-
|
|
129
|
+
// Issue #157: Use retry-enabled wrapper for transient error resilience
|
|
130
|
+
const result = await this.testPayloadWithRetry(tool, attackPattern.attackName, payload, callTool);
|
|
121
131
|
toolResults.push(result);
|
|
122
132
|
if (result.vulnerable && onProgress) {
|
|
123
133
|
this.logger.log(`🚨 VULNERABILITY: ${tool.name} - ${attackPattern.attackName} (${payload.payloadType}: ${payload.description})`);
|
|
@@ -267,7 +277,8 @@ export class SecurityPayloadTester {
|
|
|
267
277
|
completedTests++;
|
|
268
278
|
batchCount++;
|
|
269
279
|
try {
|
|
270
|
-
|
|
280
|
+
// Issue #157: Use retry-enabled wrapper for transient error resilience
|
|
281
|
+
const result = await this.testPayloadWithRetry(tool, attackPattern.attackName, payload, callTool);
|
|
271
282
|
results.push(result);
|
|
272
283
|
toolResults.push(result);
|
|
273
284
|
if (result.vulnerable && onProgress) {
|
|
@@ -450,7 +461,8 @@ export class SecurityPayloadTester {
|
|
|
450
461
|
excessivePermissionsEvidence: scopeResult.evidence,
|
|
451
462
|
};
|
|
452
463
|
}
|
|
453
|
-
|
|
464
|
+
// Build result object
|
|
465
|
+
const result = {
|
|
454
466
|
testName: attackName,
|
|
455
467
|
description: payload.description,
|
|
456
468
|
payload: payload.payload,
|
|
@@ -476,6 +488,22 @@ export class SecurityPayloadTester {
|
|
|
476
488
|
...excessivePermissionsFields,
|
|
477
489
|
...confidenceResult,
|
|
478
490
|
};
|
|
491
|
+
// Issue #170: Apply annotation-aware severity adjustment
|
|
492
|
+
// Reduces false positives for read-only servers
|
|
493
|
+
if (this.config.toolAnnotationsContext) {
|
|
494
|
+
const toolAnnotations = this.config.toolAnnotationsContext.toolAnnotations.get(tool.name);
|
|
495
|
+
const adjustment = adjustSeverityForAnnotations(attackName, result.riskLevel, toolAnnotations, this.config.toolAnnotationsContext.serverIsReadOnly, this.config.toolAnnotationsContext.serverIsClosed);
|
|
496
|
+
if (adjustment.wasAdjusted) {
|
|
497
|
+
result.riskLevel = adjustment.adjustedRiskLevel;
|
|
498
|
+
result.annotationAdjustment = {
|
|
499
|
+
original: adjustment.originalRiskLevel,
|
|
500
|
+
adjusted: adjustment.adjustedRiskLevel,
|
|
501
|
+
reason: adjustment.adjustmentReason ||
|
|
502
|
+
"Adjusted based on tool annotations",
|
|
503
|
+
};
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
return result;
|
|
479
507
|
}
|
|
480
508
|
catch (error) {
|
|
481
509
|
// Check if error is a connection/server failure
|
|
@@ -507,6 +535,68 @@ export class SecurityPayloadTester {
|
|
|
507
535
|
};
|
|
508
536
|
}
|
|
509
537
|
}
|
|
538
|
+
/**
|
|
539
|
+
* Test payload with retry logic for transient errors.
|
|
540
|
+
* Implements exponential backoff: 100ms → 200ms → 400ms
|
|
541
|
+
*
|
|
542
|
+
* Issue #157: Connection retry logic for reliability
|
|
543
|
+
*
|
|
544
|
+
* @param tool - Tool to test
|
|
545
|
+
* @param attackName - Name of attack pattern
|
|
546
|
+
* @param payload - Security payload to test
|
|
547
|
+
* @param callTool - Function to call the tool
|
|
548
|
+
* @returns SecurityTestResult with retry metadata if applicable
|
|
549
|
+
*/
|
|
550
|
+
async testPayloadWithRetry(tool, attackName, payload, callTool) {
|
|
551
|
+
const maxRetries = this.config.securityRetryMaxAttempts ??
|
|
552
|
+
DEFAULT_PERFORMANCE_CONFIG.securityRetryMaxAttempts;
|
|
553
|
+
const backoffMs = this.config.securityRetryBackoffMs ??
|
|
554
|
+
DEFAULT_PERFORMANCE_CONFIG.securityRetryBackoffMs;
|
|
555
|
+
let lastResult = null;
|
|
556
|
+
let retryAttempts = 0;
|
|
557
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
558
|
+
const result = await this.testPayload(tool, attackName, payload, callTool);
|
|
559
|
+
// Check if result indicates transient error worth retrying
|
|
560
|
+
if (result.connectionError && attempt < maxRetries) {
|
|
561
|
+
const errorText = (result.response || "").toLowerCase();
|
|
562
|
+
if (isTransientErrorPattern(errorText)) {
|
|
563
|
+
retryAttempts++;
|
|
564
|
+
lastResult = result;
|
|
565
|
+
this.logger.log(`Transient error on ${tool.name}, retrying (${attempt + 1}/${maxRetries}): ${errorText.slice(0, 100)}`);
|
|
566
|
+
// Exponential backoff: 100ms → 200ms → 400ms
|
|
567
|
+
await this.sleep(backoffMs * Math.pow(2, attempt));
|
|
568
|
+
continue;
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
// Success or permanent error - return with retry metadata
|
|
572
|
+
return this.addRetryMetadata(result, retryAttempts, !result.connectionError);
|
|
573
|
+
}
|
|
574
|
+
// All retries exhausted - return last result with failure metadata
|
|
575
|
+
if (lastResult) {
|
|
576
|
+
return this.addRetryMetadata(lastResult, retryAttempts, false);
|
|
577
|
+
}
|
|
578
|
+
// Should not reach here, but handle gracefully
|
|
579
|
+
throw new Error(`Unexpected retry loop exit for ${tool.name}`);
|
|
580
|
+
}
|
|
581
|
+
/**
|
|
582
|
+
* Add retry metadata to result.
|
|
583
|
+
* Issue #157: Track retry attempts for reliability metrics
|
|
584
|
+
*/
|
|
585
|
+
addRetryMetadata(result, retryAttempts, succeeded) {
|
|
586
|
+
if (retryAttempts === 0) {
|
|
587
|
+
// No retries needed - return as-is with completed status
|
|
588
|
+
return {
|
|
589
|
+
...result,
|
|
590
|
+
testReliability: result.connectionError ? "failed" : "completed",
|
|
591
|
+
};
|
|
592
|
+
}
|
|
593
|
+
return {
|
|
594
|
+
...result,
|
|
595
|
+
retryAttempts,
|
|
596
|
+
retriedSuccessfully: succeeded,
|
|
597
|
+
testReliability: succeeded ? "retried" : "failed",
|
|
598
|
+
};
|
|
599
|
+
}
|
|
510
600
|
/**
|
|
511
601
|
* Extract error message from caught exception
|
|
512
602
|
*/
|
|
@@ -363,7 +363,7 @@ export declare class SecurityResponseAnalyzer {
|
|
|
363
363
|
isCreationResponse(responseText: string): boolean;
|
|
364
364
|
/**
|
|
365
365
|
* Check for safe error responses that indicate proper input rejection
|
|
366
|
-
* Handles: MCP validation errors (-32602), HTTP 4xx/5xx errors
|
|
366
|
+
* Handles: MCP validation errors (-32602), HTTP 4xx/5xx errors, AppleScript syntax errors
|
|
367
367
|
*/
|
|
368
368
|
private checkSafeErrorResponses;
|
|
369
369
|
/**
|
package/client/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SecurityResponseAnalyzer.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/securityTests/SecurityResponseAnalyzer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EACL,2BAA2B,EAC3B,IAAI,EACL,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAC;AAK1E,OAAO,EAAgB,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAElE,OAAO,EAAoB,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAyBxE,YAAY,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,YAAY,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,WAAW,GAAG,aAAa,GAAG,SAAS,CAAC;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,OAAO,CAAC;IACpB,IAAI,EAAE,OAAO,CAAC;IACd,eAAe,EAAE,cAAc,GAAG,aAAa,GAAG,SAAS,CAAC;IAC5D,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,kBAAkB,GAAG,mBAAmB,GAAG,SAAS,CAAC;IACjE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,OAAO,CAAC;IAClB,aAAa,EACT,uBAAuB,GACvB,sBAAsB,GACtB,WAAW,GACX,SAAS,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,OAAO,CAAC;IAClB,iBAAiB,EACb,kBAAkB,GAClB,mBAAmB,GACnB,YAAY,GACZ,WAAW,GACX,iBAAiB,GACjB,SAAS,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,OAAO,CAAC;IAClB,iBAAiB,EACb,WAAW,GACX,aAAa,GACb,iBAAiB,GACjB,eAAe,GACf,UAAU,GACV,eAAe,GACf,UAAU,GACV,iBAAiB,GACjB,SAAS,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;GAKG;AACH,MAAM,WAAW,+BAA+B;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,aAAa,EACT,iBAAiB,GACjB,kBAAkB,GAClB,MAAM,GACN,SAAS,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC1B,sBAAsB,GACtB,iBAAiB,GACjB,SAAS,GACT,SAAS,CAAC;AAEd;;GAEG;AACH,MAAM,MAAM,0BAA0B,GAClC,kBAAkB,GAClB,iBAAiB,GACjB,2BAA2B,GAC3B,gBAAgB,GAChB,qBAAqB,GACrB,iBAAiB,CAAC;AAEtB;;;GAGG;AACH,MAAM,WAAW,yBAAyB;IACxC,UAAU,EAAE,OAAO,CAAC;IACpB,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,kBAAkB,CAAC;IAC9B,uBAAuB,EAAE,0BAA0B,EAAE,CAAC;IACtD,QAAQ,EAAE;QACR,kBAAkB,EAAE,MAAM,EAAE,CAAC;QAC7B,YAAY,EAAE,MAAM,EAAE,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,YAAY,GAAG,QAAQ,GAAG,UAAU,CAAC;AAEvE;;;;;;GAMG;AACH,qBAAa,wBAAwB;IAEnC,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,iBAAiB,CAA4B;IACrD,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,gBAAgB,CAAmB;;IAc3C;;;;;;OAMG;IACH,eAAe,CACb,QAAQ,EAAE,2BAA2B,EACrC,OAAO,EAAE,eAAe,EACxB,IAAI,EAAE,IAAI,GACT,cAAc;IAqCjB;;OAEG;IACH,mBAAmB,CACjB,IAAI,EAAE,IAAI,EACV,YAAY,EAAE,OAAO,EACrB,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,eAAe,EACxB,kBAAkB,CAAC,EAAE,2BAA2B,GAC/C,gBAAgB;IAWnB;;;OAGG;IACH,yBAAyB,CACvB,QAAQ,EAAE,2BAA2B,GACpC,gBAAgB;IAoBnB;;;;;;;;;OASG;IACH,2BAA2B,CACzB,QAAQ,EAAE,2BAA2B,GACpC,oBAAoB;IAmCvB;;;;;;;;;;OAUG;IACH,8BAA8B,CAC5B,QAAQ,EAAE,2BAA2B,GACpC,qBAAqB;IAyFxB;;;;;;;;;;;OAWG;IACH,8BAA8B,CAC5B,QAAQ,EAAE,2BAA2B,GACpC,qBAAqB;IA6FxB;;;;;;;;;;OAUG;IACH,gCAAgC,CAC9B,QAAQ,EAAE,2BAA2B,GACpC,uBAAuB;IAwJ1B;;;;;;;;;;;;;OAaG;IACH,4BAA4B,CAC1B,QAAQ,EAAE,2BAA2B,GACpC,mBAAmB;IAqPtB;;;;;;;;;;;;OAYG;IACH,wBAAwB,CACtB,QAAQ,EAAE,2BAA2B,GACpC,yBAAyB;IA6D5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,kBAAkB,CAAC,QAAQ,EAAE,2BAA2B,GAAG;QACzD,QAAQ,EAAE,OAAO,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB;IAwCD;;OAEG;IACH,iBAAiB,CAAC,QAAQ,EAAE,2BAA2B,GAAG,OAAO;IAIjE;;OAEG;IACH,8BAA8B,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO;IAIvD;;OAEG;IACH,aAAa,CAAC,QAAQ,EAAE,2BAA2B,GAAG,mBAAmB;IAIzE;;OAEG;IACH,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,mBAAmB;IAI/D;;OAEG;IACH,sBAAsB,CAAC,QAAQ,EAAE,2BAA2B,GAAG,MAAM;IAQrE;;OAEG;IACH,oBAAoB,CAClB,SAAS,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,EACvD,YAAY,EAAE,MAAM,GACnB,OAAO;IAIV;;OAEG;IACH,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIlD;;OAEG;IACH,mBAAmB,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO;IAIrD;;OAEG;IACH,oBAAoB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAInD;;;OAGG;IACH,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO;IAIpE;;OAEG;IACH,qCAAqC,CACnC,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,GACnB,OAAO;IAOV;;OAEG;IACH,yBAAyB,CACvB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,EACpB,IAAI,CAAC,EAAE,IAAI,GACV,kBAAkB;IAQrB;;OAEG;IACH,oBAAoB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAInD;;OAEG;IACH,wBAAwB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIvD;;OAEG;IACH,8BAA8B,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAI7D;;OAEG;IACH,qBAAqB,CAAC,QAAQ,EAAE,2BAA2B,GAAG,OAAO;IAIrE;;OAEG;IACH,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO;IAOxE;;OAEG;IACH,sBAAsB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIrD;;OAEG;IACH,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAQjD;;;OAGG;IACH,OAAO,CAAC,uBAAuB;
|
|
1
|
+
{"version":3,"file":"SecurityResponseAnalyzer.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/securityTests/SecurityResponseAnalyzer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EACL,2BAA2B,EAC3B,IAAI,EACL,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAC;AAK1E,OAAO,EAAgB,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAElE,OAAO,EAAoB,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAyBxE,YAAY,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,YAAY,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,WAAW,GAAG,aAAa,GAAG,SAAS,CAAC;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,OAAO,CAAC;IACpB,IAAI,EAAE,OAAO,CAAC;IACd,eAAe,EAAE,cAAc,GAAG,aAAa,GAAG,SAAS,CAAC;IAC5D,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,kBAAkB,GAAG,mBAAmB,GAAG,SAAS,CAAC;IACjE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,OAAO,CAAC;IAClB,aAAa,EACT,uBAAuB,GACvB,sBAAsB,GACtB,WAAW,GACX,SAAS,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,OAAO,CAAC;IAClB,iBAAiB,EACb,kBAAkB,GAClB,mBAAmB,GACnB,YAAY,GACZ,WAAW,GACX,iBAAiB,GACjB,SAAS,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,OAAO,CAAC;IAClB,iBAAiB,EACb,WAAW,GACX,aAAa,GACb,iBAAiB,GACjB,eAAe,GACf,UAAU,GACV,eAAe,GACf,UAAU,GACV,iBAAiB,GACjB,SAAS,CAAC;IACd,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;GAKG;AACH,MAAM,WAAW,+BAA+B;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,aAAa,EACT,iBAAiB,GACjB,kBAAkB,GAClB,MAAM,GACN,SAAS,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC1B,sBAAsB,GACtB,iBAAiB,GACjB,SAAS,GACT,SAAS,CAAC;AAEd;;GAEG;AACH,MAAM,MAAM,0BAA0B,GAClC,kBAAkB,GAClB,iBAAiB,GACjB,2BAA2B,GAC3B,gBAAgB,GAChB,qBAAqB,GACrB,iBAAiB,CAAC;AAEtB;;;GAGG;AACH,MAAM,WAAW,yBAAyB;IACxC,UAAU,EAAE,OAAO,CAAC;IACpB,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,kBAAkB,CAAC;IAC9B,uBAAuB,EAAE,0BAA0B,EAAE,CAAC;IACtD,QAAQ,EAAE;QACR,kBAAkB,EAAE,MAAM,EAAE,CAAC;QAC7B,YAAY,EAAE,MAAM,EAAE,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,YAAY,GAAG,QAAQ,GAAG,UAAU,CAAC;AAEvE;;;;;;GAMG;AACH,qBAAa,wBAAwB;IAEnC,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,iBAAiB,CAA4B;IACrD,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,gBAAgB,CAAmB;;IAc3C;;;;;;OAMG;IACH,eAAe,CACb,QAAQ,EAAE,2BAA2B,EACrC,OAAO,EAAE,eAAe,EACxB,IAAI,EAAE,IAAI,GACT,cAAc;IAqCjB;;OAEG;IACH,mBAAmB,CACjB,IAAI,EAAE,IAAI,EACV,YAAY,EAAE,OAAO,EACrB,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,eAAe,EACxB,kBAAkB,CAAC,EAAE,2BAA2B,GAC/C,gBAAgB;IAWnB;;;OAGG;IACH,yBAAyB,CACvB,QAAQ,EAAE,2BAA2B,GACpC,gBAAgB;IAoBnB;;;;;;;;;OASG;IACH,2BAA2B,CACzB,QAAQ,EAAE,2BAA2B,GACpC,oBAAoB;IAmCvB;;;;;;;;;;OAUG;IACH,8BAA8B,CAC5B,QAAQ,EAAE,2BAA2B,GACpC,qBAAqB;IAyFxB;;;;;;;;;;;OAWG;IACH,8BAA8B,CAC5B,QAAQ,EAAE,2BAA2B,GACpC,qBAAqB;IA6FxB;;;;;;;;;;OAUG;IACH,gCAAgC,CAC9B,QAAQ,EAAE,2BAA2B,GACpC,uBAAuB;IAwJ1B;;;;;;;;;;;;;OAaG;IACH,4BAA4B,CAC1B,QAAQ,EAAE,2BAA2B,GACpC,mBAAmB;IAqPtB;;;;;;;;;;;;OAYG;IACH,wBAAwB,CACtB,QAAQ,EAAE,2BAA2B,GACpC,yBAAyB;IA6D5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,kBAAkB,CAAC,QAAQ,EAAE,2BAA2B,GAAG;QACzD,QAAQ,EAAE,OAAO,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB;IAwCD;;OAEG;IACH,iBAAiB,CAAC,QAAQ,EAAE,2BAA2B,GAAG,OAAO;IAIjE;;OAEG;IACH,8BAA8B,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO;IAIvD;;OAEG;IACH,aAAa,CAAC,QAAQ,EAAE,2BAA2B,GAAG,mBAAmB;IAIzE;;OAEG;IACH,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,mBAAmB;IAI/D;;OAEG;IACH,sBAAsB,CAAC,QAAQ,EAAE,2BAA2B,GAAG,MAAM;IAQrE;;OAEG;IACH,oBAAoB,CAClB,SAAS,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,EACvD,YAAY,EAAE,MAAM,GACnB,OAAO;IAIV;;OAEG;IACH,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIlD;;OAEG;IACH,mBAAmB,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO;IAIrD;;OAEG;IACH,oBAAoB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAInD;;;OAGG;IACH,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO;IAIpE;;OAEG;IACH,qCAAqC,CACnC,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,GACnB,OAAO;IAOV;;OAEG;IACH,yBAAyB,CACvB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,EACpB,IAAI,CAAC,EAAE,IAAI,GACV,kBAAkB;IAQrB;;OAEG;IACH,oBAAoB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAInD;;OAEG;IACH,wBAAwB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIvD;;OAEG;IACH,8BAA8B,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAI7D;;OAEG;IACH,qBAAqB,CAAC,QAAQ,EAAE,2BAA2B,GAAG,OAAO;IAIrE;;OAEG;IACH,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO;IAOxE;;OAEG;IACH,sBAAsB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIrD;;OAEG;IACH,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAQjD;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAoC/B;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAqF7B;;;OAGG;IACH,OAAO,CAAC,0BAA0B;IA8DlC;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,4BAA4B;IA8CpC;;;;;;;;;OASG;IACH,mCAAmC,CACjC,QAAQ,EAAE,2BAA2B,GACpC,+BAA+B;IA6ElC;;OAEG;IACH,OAAO,CAAC,wBAAwB;CAmBjC"}
|
|
@@ -983,7 +983,7 @@ export class SecurityResponseAnalyzer {
|
|
|
983
983
|
// ============================================================================
|
|
984
984
|
/**
|
|
985
985
|
* Check for safe error responses that indicate proper input rejection
|
|
986
|
-
* Handles: MCP validation errors (-32602), HTTP 4xx/5xx errors
|
|
986
|
+
* Handles: MCP validation errors (-32602), HTTP 4xx/5xx errors, AppleScript syntax errors
|
|
987
987
|
*/
|
|
988
988
|
checkSafeErrorResponses(responseText, errorInfo) {
|
|
989
989
|
// MCP validation errors (HIGHEST PRIORITY)
|
|
@@ -994,6 +994,15 @@ export class SecurityResponseAnalyzer {
|
|
|
994
994
|
evidence: `MCP validation error${errorCode}: Tool properly rejected invalid input before processing`,
|
|
995
995
|
};
|
|
996
996
|
}
|
|
997
|
+
// Issue #175: AppleScript syntax errors (not XXE)
|
|
998
|
+
// AppleScript errors can trigger false positives when the XXE payload is echoed
|
|
999
|
+
// back in the error message, matching patterns like "parameter" + "entity"
|
|
1000
|
+
if (this.safeDetector.isAppleScriptSyntaxError(responseText)) {
|
|
1001
|
+
return {
|
|
1002
|
+
isVulnerable: false,
|
|
1003
|
+
evidence: "AppleScript syntax error - not XXE vulnerability (echoed payload in error message)",
|
|
1004
|
+
};
|
|
1005
|
+
}
|
|
997
1006
|
// HTTP error responses (Issue #26)
|
|
998
1007
|
if (this.safeDetector.isHttpErrorResponse(responseText)) {
|
|
999
1008
|
return {
|
|
@@ -13,4 +13,5 @@ export { SecurityPayloadGenerator } from "./SecurityPayloadGenerator.js";
|
|
|
13
13
|
export { CrossToolStateTester, type CrossToolTestResult, type ToolPair, type CallToolFunction, type CrossToolTestConfig, } from "./CrossToolStateTester.js";
|
|
14
14
|
export { ChainExecutionTester, type ChainExecutionTestResult, type ChainExploitationSummary, type ChainExecutionTesterConfig, type ChainTestReason, } from "./ChainExecutionTester.js";
|
|
15
15
|
export { TestValidityAnalyzer, type TestValidityConfig, type TestValidityResult, } from "./TestValidityAnalyzer.js";
|
|
16
|
+
export { adjustSeverityForAnnotations, type SeverityAdjustment, } from "./AnnotationAwareSeverity.js";
|
|
16
17
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/securityTests/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,wBAAwB,EACxB,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,mBAAmB,EACxB,KAAK,oBAAoB,EACzB,KAAK,yBAAyB,EAC9B,KAAK,kBAAkB,EACvB,KAAK,0BAA0B,GAChC,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EACL,qBAAqB,EACrB,KAAK,oBAAoB,EACzB,KAAK,iBAAiB,EACtB,KAAK,UAAU,GAChB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,wBAAwB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,OAAO,EACL,oBAAoB,EACpB,KAAK,mBAAmB,EACxB,KAAK,QAAQ,EACb,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,GACzB,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EACL,oBAAoB,EACpB,KAAK,wBAAwB,EAC7B,KAAK,wBAAwB,EAC7B,KAAK,0BAA0B,EAC/B,KAAK,eAAe,GACrB,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EACL,oBAAoB,EACpB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,GACxB,MAAM,wBAAwB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/securityTests/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,wBAAwB,EACxB,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,mBAAmB,EACxB,KAAK,oBAAoB,EACzB,KAAK,yBAAyB,EAC9B,KAAK,kBAAkB,EACvB,KAAK,0BAA0B,GAChC,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EACL,qBAAqB,EACrB,KAAK,oBAAoB,EACzB,KAAK,iBAAiB,EACtB,KAAK,UAAU,GAChB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,wBAAwB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,OAAO,EACL,oBAAoB,EACpB,KAAK,mBAAmB,EACxB,KAAK,QAAQ,EACb,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,GACzB,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EACL,oBAAoB,EACpB,KAAK,wBAAwB,EAC7B,KAAK,wBAAwB,EAC7B,KAAK,0BAA0B,EAC/B,KAAK,eAAe,GACrB,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EACL,oBAAoB,EACpB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,GACxB,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EACL,4BAA4B,EAC5B,KAAK,kBAAkB,GACxB,MAAM,2BAA2B,CAAC"}
|
|
@@ -13,3 +13,4 @@ export { SecurityPayloadGenerator } from "./SecurityPayloadGenerator.js";
|
|
|
13
13
|
export { CrossToolStateTester, } from "./CrossToolStateTester.js";
|
|
14
14
|
export { ChainExecutionTester, } from "./ChainExecutionTester.js";
|
|
15
15
|
export { TestValidityAnalyzer, } from "./TestValidityAnalyzer.js";
|
|
16
|
+
export { adjustSeverityForAnnotations, } from "./AnnotationAwareSeverity.js";
|
|
@@ -13,6 +13,7 @@ import { MutationDetector } from "./MutationDetector.js";
|
|
|
13
13
|
*/
|
|
14
14
|
export declare class VarianceClassifier {
|
|
15
15
|
private mutationDetector;
|
|
16
|
+
private externalAPIDetector;
|
|
16
17
|
private readonly DESTRUCTIVE_PATTERNS;
|
|
17
18
|
/**
|
|
18
19
|
* Tool name patterns that are expected to have state-dependent responses.
|
|
@@ -79,12 +80,26 @@ export declare class VarianceClassifier {
|
|
|
79
80
|
* - "recreate_view" does NOT match "create" (must be at word boundary)
|
|
80
81
|
*/
|
|
81
82
|
isResourceCreatingTool(tool: Tool): boolean;
|
|
83
|
+
/**
|
|
84
|
+
* Issue #166: Check if a tool fetches data from external APIs.
|
|
85
|
+
* External API tools legitimately return different data each call
|
|
86
|
+
* due to: live data updates, API errors (500, 429), rate limiting, etc.
|
|
87
|
+
*
|
|
88
|
+
* Issue #168: Delegates to shared ExternalAPIDependencyDetector for consistent
|
|
89
|
+
* detection logic across all assessors.
|
|
90
|
+
*
|
|
91
|
+
* Uses BOTH name patterns AND description analysis for detection.
|
|
92
|
+
*/
|
|
93
|
+
isExternalAPITool(tool: Tool): boolean;
|
|
82
94
|
/**
|
|
83
95
|
* Issue #69: Classify variance between two responses to reduce false positives.
|
|
84
96
|
* Returns LEGITIMATE for expected variance (IDs, timestamps), SUSPICIOUS for
|
|
85
97
|
* schema changes, and BEHAVIORAL for semantic changes (promotional keywords, errors).
|
|
98
|
+
*
|
|
99
|
+
* Issue #166: Added optional tool parameter to enable external API handling.
|
|
100
|
+
* External API tools may have error vs success variance which is LEGITIMATE.
|
|
86
101
|
*/
|
|
87
|
-
classifyVariance(baseline: unknown, current: unknown): VarianceClassification;
|
|
102
|
+
classifyVariance(baseline: unknown, current: unknown, tool?: Tool): VarianceClassification;
|
|
88
103
|
/**
|
|
89
104
|
* Issue #69: Check if a field name represents legitimate variance.
|
|
90
105
|
* Fields containing IDs, timestamps, tokens, etc. are expected to vary.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"VarianceClassifier.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/temporal/VarianceClassifier.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAE,MAAM,oCAAoC,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"VarianceClassifier.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/temporal/VarianceClassifier.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAE,MAAM,oCAAoC,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAItD;;;GAGG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,gBAAgB,CAAmB;IAE3C,OAAO,CAAC,mBAAmB,CAAgC;IAG3D,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAoBnC;IAEF;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAqBrC;IAEF;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAYzC;gBAKU,gBAAgB,CAAC,EAAE,gBAAgB;IAM/C;;;;OAIG;IACH,iBAAiB,CAAC,QAAQ,EAAE,OAAO,GAAG,MAAM;IAiF5C;;OAEG;IACH,iBAAiB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO;IAKtC;;;;;;;;OAQG;IACH,cAAc,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO;IAenC;;;;;;;;;;;OAWG;IACH,sBAAsB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO;IAQ3C;;;;;;;;;OASG;IACH,iBAAiB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO;IAKtC;;;;;;;OAOG;IACH,gBAAgB,CACd,QAAQ,EAAE,OAAO,EACjB,OAAO,EAAE,OAAO,EAChB,IAAI,CAAC,EAAE,IAAI,GACV,sBAAsB;IAuFzB;;;OAGG;IACH,yBAAyB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAgEjD;;;OAGG;IACH,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,SAAK,GAAG,MAAM,EAAE;IAuDrE;;;;;;OAMG;IACH,cAAc,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,GAAG,OAAO;IAuB/D;;;OAGG;IACH,iBAAiB,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,SAAK,GAAG,MAAM,EAAE;CAgCvD"}
|
|
@@ -5,12 +5,16 @@
|
|
|
5
5
|
* Extracted from TemporalAssessor as part of Issue #106 refactoring.
|
|
6
6
|
*/
|
|
7
7
|
import { MutationDetector } from "./MutationDetector.js";
|
|
8
|
+
// Issue #168: Shared external API dependency detector
|
|
9
|
+
import { ExternalAPIDependencyDetector } from "../../helpers/ExternalAPIDependencyDetector.js";
|
|
8
10
|
/**
|
|
9
11
|
* Classifies response variance and categorizes tools by their expected behavior patterns.
|
|
10
12
|
* Used to reduce false positives in temporal assessment by understanding legitimate variance.
|
|
11
13
|
*/
|
|
12
14
|
export class VarianceClassifier {
|
|
13
15
|
mutationDetector;
|
|
16
|
+
// Issue #168: Shared detector for external API pattern matching
|
|
17
|
+
externalAPIDetector;
|
|
14
18
|
// Patterns that suggest a tool may have side effects
|
|
15
19
|
DESTRUCTIVE_PATTERNS = [
|
|
16
20
|
"create",
|
|
@@ -97,8 +101,12 @@ export class VarianceClassifier {
|
|
|
97
101
|
"init",
|
|
98
102
|
"make",
|
|
99
103
|
];
|
|
104
|
+
// Note: External API patterns moved to ExternalAPIDependencyDetector (Issue #168)
|
|
105
|
+
// The externalAPIDetector field handles all external API detection via shared helper
|
|
100
106
|
constructor(mutationDetector) {
|
|
101
107
|
this.mutationDetector = mutationDetector ?? new MutationDetector();
|
|
108
|
+
// Issue #168: Initialize shared external API detector
|
|
109
|
+
this.externalAPIDetector = new ExternalAPIDependencyDetector();
|
|
102
110
|
}
|
|
103
111
|
/**
|
|
104
112
|
* Normalize response for comparison by removing naturally varying data.
|
|
@@ -207,12 +215,46 @@ export class VarianceClassifier {
|
|
|
207
215
|
return wordBoundaryRegex.test(toolName);
|
|
208
216
|
});
|
|
209
217
|
}
|
|
218
|
+
/**
|
|
219
|
+
* Issue #166: Check if a tool fetches data from external APIs.
|
|
220
|
+
* External API tools legitimately return different data each call
|
|
221
|
+
* due to: live data updates, API errors (500, 429), rate limiting, etc.
|
|
222
|
+
*
|
|
223
|
+
* Issue #168: Delegates to shared ExternalAPIDependencyDetector for consistent
|
|
224
|
+
* detection logic across all assessors.
|
|
225
|
+
*
|
|
226
|
+
* Uses BOTH name patterns AND description analysis for detection.
|
|
227
|
+
*/
|
|
228
|
+
isExternalAPITool(tool) {
|
|
229
|
+
// Issue #168: Delegate to shared detector for consistent detection
|
|
230
|
+
return this.externalAPIDetector.isExternalAPITool(tool);
|
|
231
|
+
}
|
|
210
232
|
/**
|
|
211
233
|
* Issue #69: Classify variance between two responses to reduce false positives.
|
|
212
234
|
* Returns LEGITIMATE for expected variance (IDs, timestamps), SUSPICIOUS for
|
|
213
235
|
* schema changes, and BEHAVIORAL for semantic changes (promotional keywords, errors).
|
|
236
|
+
*
|
|
237
|
+
* Issue #166: Added optional tool parameter to enable external API handling.
|
|
238
|
+
* External API tools may have error vs success variance which is LEGITIMATE.
|
|
214
239
|
*/
|
|
215
|
-
classifyVariance(baseline, current) {
|
|
240
|
+
classifyVariance(baseline, current, tool) {
|
|
241
|
+
// Issue #166: Check for isError variance (external API behavior)
|
|
242
|
+
// If one response is an error and one is success, for stateful/external API tools
|
|
243
|
+
// this is expected behavior, not a rug pull
|
|
244
|
+
const baselineIsError = baseline?.isError === true;
|
|
245
|
+
const currentIsError = current?.isError === true;
|
|
246
|
+
if (baselineIsError !== currentIsError) {
|
|
247
|
+
// One is error, one is success - check if this is expected for the tool type
|
|
248
|
+
if (tool && (this.isStatefulTool(tool) || this.isExternalAPITool(tool))) {
|
|
249
|
+
return {
|
|
250
|
+
type: "LEGITIMATE",
|
|
251
|
+
confidence: "medium",
|
|
252
|
+
reasons: [
|
|
253
|
+
"API error vs success variance (expected for external API/stateful tools)",
|
|
254
|
+
],
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
}
|
|
216
258
|
// 1. Schema comparison - structural changes are SUSPICIOUS
|
|
217
259
|
const schemaMatch = this.compareSchemas(baseline, current);
|
|
218
260
|
if (!schemaMatch) {
|
package/client/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bryan-thompson/inspector-assessment-client",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.38.0",
|
|
4
4
|
"description": "Client-side application for the Enhanced MCP Inspector with assessment capabilities",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Bryan Thompson <bryan@triepod.ai>",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bryan-thompson/inspector-assessment",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.38.0",
|
|
4
4
|
"description": "Enhanced MCP Inspector with comprehensive assessment capabilities for server validation",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Bryan Thompson <bryan@triepod.ai>",
|
package/server/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bryan-thompson/inspector-assessment-server",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.38.0",
|
|
4
4
|
"description": "Server-side application for the Enhanced MCP Inspector with assessment capabilities",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Bryan Thompson <bryan@triepod.ai>",
|