@bryan-thompson/inspector-assessment-client 1.26.5 → 1.26.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-DpdInvWI.js → OAuthCallback-CCWVtjr7.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-D1ImpKK5.js → OAuthDebugCallback-DqbXfUi4.js} +1 -1
- package/dist/assets/{index-umcoGmYw.js → index-CsDJSSWq.js} +4 -4
- package/dist/index.html +1 -1
- package/lib/lib/assessment/configTypes.d.ts +2 -0
- package/lib/lib/assessment/configTypes.d.ts.map +1 -1
- package/lib/lib/securityPatterns.d.ts +4 -2
- package/lib/lib/securityPatterns.d.ts.map +1 -1
- package/lib/lib/securityPatterns.js +146 -2
- package/lib/services/assessment/modules/AUPComplianceAssessor.js +9 -9
- package/lib/services/assessment/modules/AuthenticationAssessor.js +4 -4
- package/lib/services/assessment/modules/BaseAssessor.d.ts +0 -14
- package/lib/services/assessment/modules/BaseAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/BaseAssessor.js +1 -33
- package/lib/services/assessment/modules/CrossCapabilitySecurityAssessor.js +1 -1
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.js +1 -1
- package/lib/services/assessment/modules/DocumentationAssessor.js +2 -2
- package/lib/services/assessment/modules/ErrorHandlingAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ErrorHandlingAssessor.js +8 -8
- package/lib/services/assessment/modules/ExternalAPIScannerAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ExternalAPIScannerAssessor.js +3 -3
- package/lib/services/assessment/modules/FunctionalityAssessor.js +9 -9
- package/lib/services/assessment/modules/MCPSpecComplianceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/MCPSpecComplianceAssessor.js +12 -12
- package/lib/services/assessment/modules/ManifestValidationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ManifestValidationAssessor.js +9 -5
- package/lib/services/assessment/modules/PortabilityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/PortabilityAssessor.js +3 -3
- package/lib/services/assessment/modules/ProhibitedLibrariesAssessor.js +4 -4
- package/lib/services/assessment/modules/PromptAssessor.js +2 -2
- package/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ProtocolComplianceAssessor.js +7 -7
- package/lib/services/assessment/modules/ProtocolConformanceAssessor.js +1 -1
- package/lib/services/assessment/modules/ResourceAssessor.js +1 -1
- package/lib/services/assessment/modules/SecurityAssessor.d.ts +25 -2
- package/lib/services/assessment/modules/SecurityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/SecurityAssessor.js +149 -17
- package/lib/services/assessment/modules/TemporalAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/TemporalAssessor.js +10 -10
- package/lib/services/assessment/modules/ToolAnnotationAssessor.js +9 -9
- package/lib/services/assessment/modules/UsabilityAssessor.js +1 -1
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts.map +1 -1
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.js +37 -0
- package/lib/services/assessment/modules/index.d.ts +3 -0
- package/lib/services/assessment/modules/index.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/ChainExecutionTester.d.ts +104 -0
- package/lib/services/assessment/modules/securityTests/ChainExecutionTester.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/ChainExecutionTester.js +257 -0
- package/lib/services/assessment/modules/securityTests/ConfidenceScorer.d.ts +57 -0
- package/lib/services/assessment/modules/securityTests/ConfidenceScorer.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/ConfidenceScorer.js +199 -0
- package/lib/services/assessment/modules/securityTests/CrossToolStateTester.d.ts +91 -0
- package/lib/services/assessment/modules/securityTests/CrossToolStateTester.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/CrossToolStateTester.js +225 -0
- package/lib/services/assessment/modules/securityTests/ErrorClassifier.d.ts +57 -0
- package/lib/services/assessment/modules/securityTests/ErrorClassifier.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/ErrorClassifier.js +113 -0
- package/lib/services/assessment/modules/securityTests/ExecutionArtifactDetector.d.ts +49 -0
- package/lib/services/assessment/modules/securityTests/ExecutionArtifactDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/ExecutionArtifactDetector.js +74 -0
- package/lib/services/assessment/modules/securityTests/MathAnalyzer.d.ts +58 -0
- package/lib/services/assessment/modules/securityTests/MathAnalyzer.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/MathAnalyzer.js +251 -0
- package/lib/services/assessment/modules/securityTests/SafeResponseDetector.d.ts +59 -0
- package/lib/services/assessment/modules/securityTests/SafeResponseDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SafeResponseDetector.js +151 -0
- package/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.d.ts +349 -0
- package/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.js +904 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.js +49 -24
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts +122 -85
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.js +443 -1176
- package/lib/services/assessment/modules/securityTests/index.d.ts +3 -1
- package/lib/services/assessment/modules/securityTests/index.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/index.js +2 -0
- package/package.json +1 -1
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Chain Execution Tester
|
|
3
|
+
* Dynamic testing for multi-tool chain exploitation vulnerabilities
|
|
4
|
+
*
|
|
5
|
+
* Issue #93, Challenge #6: Multi-tool chained exploitation attacks
|
|
6
|
+
* Tests for:
|
|
7
|
+
* 1. Arbitrary tool invocation without allowlist
|
|
8
|
+
* 2. Output injection via {{output}} template substitution
|
|
9
|
+
* 3. Recursive chain execution (DoS potential)
|
|
10
|
+
* 4. State poisoning between chain steps
|
|
11
|
+
* 5. Missing depth/size limits
|
|
12
|
+
*
|
|
13
|
+
* A/B Validation:
|
|
14
|
+
* - vulnerable-mcp (10900): Should detect all vulnerability categories
|
|
15
|
+
* - hardened-mcp (10901): 0 false positives (validation-only behavior)
|
|
16
|
+
*/
|
|
17
|
+
import { SecurityResponseAnalyzer, } from "./SecurityResponseAnalyzer.js";
|
|
18
|
+
/**
|
|
19
|
+
* Tests for multi-tool chain exploitation vulnerabilities
|
|
20
|
+
*/
|
|
21
|
+
export class ChainExecutionTester {
|
|
22
|
+
verbose;
|
|
23
|
+
analyzer;
|
|
24
|
+
constructor(config = {}) {
|
|
25
|
+
this.verbose = config.verbose ?? false;
|
|
26
|
+
this.analyzer = new SecurityResponseAnalyzer();
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Log message if verbose logging is enabled
|
|
30
|
+
*/
|
|
31
|
+
log(message) {
|
|
32
|
+
if (this.verbose) {
|
|
33
|
+
// eslint-disable-next-line no-console
|
|
34
|
+
console.log(`[ChainExecutionTester] ${message}`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Identify tools that might be chain executors
|
|
39
|
+
* Looks for tools with names/descriptions/parameters suggesting chain execution
|
|
40
|
+
*/
|
|
41
|
+
identifyChainExecutorTools(tools) {
|
|
42
|
+
const chainNamePatterns = [
|
|
43
|
+
/chain/i,
|
|
44
|
+
/executor/i,
|
|
45
|
+
/pipeline/i,
|
|
46
|
+
/sequence/i,
|
|
47
|
+
/workflow/i,
|
|
48
|
+
/orchestrat/i,
|
|
49
|
+
/multi.*tool/i,
|
|
50
|
+
/batch/i,
|
|
51
|
+
];
|
|
52
|
+
const chainParamPatterns = [
|
|
53
|
+
/chain/i,
|
|
54
|
+
/steps/i,
|
|
55
|
+
/sequence/i,
|
|
56
|
+
/pipeline/i,
|
|
57
|
+
/tools/i,
|
|
58
|
+
/commands/i,
|
|
59
|
+
];
|
|
60
|
+
return tools.filter((tool) => {
|
|
61
|
+
// Check tool name
|
|
62
|
+
const nameMatches = chainNamePatterns.some((p) => p.test(tool.name));
|
|
63
|
+
// Check description
|
|
64
|
+
const descMatches = tool.description &&
|
|
65
|
+
chainNamePatterns.some((p) => p.test(tool.description || ""));
|
|
66
|
+
// Check parameter names
|
|
67
|
+
const schema = tool.inputSchema;
|
|
68
|
+
const paramNames = Object.keys(schema?.properties || {});
|
|
69
|
+
const paramMatches = paramNames.some((param) => chainParamPatterns.some((p) => p.test(param)));
|
|
70
|
+
return nameMatches || descMatches || paramMatches;
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Get the parameter name for chain input from tool schema
|
|
75
|
+
*/
|
|
76
|
+
getChainParamName(tool) {
|
|
77
|
+
const schema = tool.inputSchema;
|
|
78
|
+
const paramNames = Object.keys(schema?.properties || {});
|
|
79
|
+
// Look for chain-like parameter names
|
|
80
|
+
const chainParam = paramNames.find((p) => /chain|steps|sequence|pipeline/i.test(p));
|
|
81
|
+
return chainParam || "chain";
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Extract text content from tool response
|
|
85
|
+
*/
|
|
86
|
+
extractResponseText(response) {
|
|
87
|
+
if (!response)
|
|
88
|
+
return "";
|
|
89
|
+
// Handle content array format
|
|
90
|
+
if (response.content && Array.isArray(response.content)) {
|
|
91
|
+
return response.content
|
|
92
|
+
.map((item) => {
|
|
93
|
+
if (typeof item === "string")
|
|
94
|
+
return item;
|
|
95
|
+
if (item && typeof item === "object" && "text" in item)
|
|
96
|
+
return String(item.text);
|
|
97
|
+
return JSON.stringify(item);
|
|
98
|
+
})
|
|
99
|
+
.join("\n");
|
|
100
|
+
}
|
|
101
|
+
return "";
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Determine the vulnerability reason from analysis result
|
|
105
|
+
*/
|
|
106
|
+
determineVulnerabilityReason(analysis) {
|
|
107
|
+
if (analysis.vulnerabilityCategories.includes("OUTPUT_INJECTION")) {
|
|
108
|
+
return "output_injection_detected";
|
|
109
|
+
}
|
|
110
|
+
if (analysis.vulnerabilityCategories.includes("RECURSIVE_CHAIN")) {
|
|
111
|
+
return "recursive_execution_detected";
|
|
112
|
+
}
|
|
113
|
+
if (analysis.vulnerabilityCategories.includes("ARBITRARY_TOOL_INVOCATION")) {
|
|
114
|
+
return "arbitrary_tool_accepted";
|
|
115
|
+
}
|
|
116
|
+
if (analysis.vulnerabilityCategories.includes("STATE_POISONING")) {
|
|
117
|
+
return "state_poisoning_detected";
|
|
118
|
+
}
|
|
119
|
+
return "chain_execution_confirmed";
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Test single chain payload against a tool
|
|
123
|
+
*/
|
|
124
|
+
async testChainPayload(callTool, tool, chainPayload, paramName) {
|
|
125
|
+
const actualParamName = paramName || this.getChainParamName(tool);
|
|
126
|
+
try {
|
|
127
|
+
this.log(`Testing payload on ${tool.name} with param ${actualParamName}`);
|
|
128
|
+
const response = await callTool(tool.name, {
|
|
129
|
+
[actualParamName]: chainPayload,
|
|
130
|
+
});
|
|
131
|
+
const responseText = this.extractResponseText(response);
|
|
132
|
+
// Analyze response for chain exploitation
|
|
133
|
+
const analysis = this.analyzer.analyzeChainExploitation(response);
|
|
134
|
+
if (analysis.vulnerable) {
|
|
135
|
+
return {
|
|
136
|
+
vulnerable: true,
|
|
137
|
+
reason: this.determineVulnerabilityReason(analysis),
|
|
138
|
+
vulnerabilityCategories: analysis.vulnerabilityCategories,
|
|
139
|
+
evidence: {
|
|
140
|
+
chainPayload,
|
|
141
|
+
response: responseText.substring(0, 500),
|
|
142
|
+
analysisResult: analysis,
|
|
143
|
+
},
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
if (analysis.safe) {
|
|
147
|
+
return {
|
|
148
|
+
vulnerable: false,
|
|
149
|
+
reason: analysis.chainType === "SAFE_VALIDATION"
|
|
150
|
+
? "validation_only"
|
|
151
|
+
: "chain_rejected",
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
return {
|
|
155
|
+
vulnerable: false,
|
|
156
|
+
reason: "tool_not_chain_executor",
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
catch (error) {
|
|
160
|
+
return {
|
|
161
|
+
vulnerable: false,
|
|
162
|
+
reason: "test_error",
|
|
163
|
+
error: error instanceof Error ? error.message : String(error),
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Get test payloads for chain exploitation testing
|
|
169
|
+
*/
|
|
170
|
+
getTestPayloads() {
|
|
171
|
+
return [
|
|
172
|
+
{
|
|
173
|
+
name: "single_step_execution",
|
|
174
|
+
payload: '[{"tool": "vulnerable_calculator", "args": {"query": "2+2"}}]',
|
|
175
|
+
description: "Test basic chain execution without allowlist",
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
name: "output_injection",
|
|
179
|
+
payload: '[{"tool": "calc", "args": {"q": "test"}}, {"tool": "exec", "args": {"cmd": "{{output}}"}}]',
|
|
180
|
+
description: "Test output injection via {{output}} template",
|
|
181
|
+
},
|
|
182
|
+
{
|
|
183
|
+
name: "recursive_chain",
|
|
184
|
+
payload: '[{"tool": "vulnerable_chain_executor", "args": {"chain": "[]"}}]',
|
|
185
|
+
description: "Test recursive/circular chain execution",
|
|
186
|
+
},
|
|
187
|
+
{
|
|
188
|
+
name: "arbitrary_tool",
|
|
189
|
+
payload: '[{"tool": "UNKNOWN_MALICIOUS_TOOL_XYZ", "args": {}}]',
|
|
190
|
+
description: "Test acceptance of arbitrary tool names",
|
|
191
|
+
},
|
|
192
|
+
{
|
|
193
|
+
name: "state_poisoning",
|
|
194
|
+
payload: '[{"tool": "vulnerable_config_modifier", "args": {"setting": "admin_mode=true"}}, {"tool": "vulnerable_admin_action", "args": {"action": "read secrets"}}]',
|
|
195
|
+
description: "Test state poisoning between chain steps",
|
|
196
|
+
},
|
|
197
|
+
{
|
|
198
|
+
name: "depth_bypass",
|
|
199
|
+
payload: JSON.stringify(Array(15)
|
|
200
|
+
.fill(null)
|
|
201
|
+
.map(() => ({ tool: "test", args: {} }))),
|
|
202
|
+
description: "Test depth limit bypass with 15 steps",
|
|
203
|
+
},
|
|
204
|
+
];
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Run comprehensive chain exploitation tests on a tool
|
|
208
|
+
*/
|
|
209
|
+
async runChainExploitationTests(callTool, tool) {
|
|
210
|
+
const results = new Map();
|
|
211
|
+
const testPayloads = this.getTestPayloads();
|
|
212
|
+
const chainParam = this.getChainParamName(tool);
|
|
213
|
+
this.log(`Running chain exploitation tests on ${tool.name}`);
|
|
214
|
+
this.log(`Using parameter: ${chainParam}`);
|
|
215
|
+
for (const test of testPayloads) {
|
|
216
|
+
this.log(` Test: ${test.name} - ${test.description}`);
|
|
217
|
+
const result = await this.testChainPayload(callTool, tool, test.payload, chainParam);
|
|
218
|
+
results.set(test.name, result);
|
|
219
|
+
if (this.verbose) {
|
|
220
|
+
// eslint-disable-next-line no-console
|
|
221
|
+
console.log(` Result: ${result.vulnerable ? "VULNERABLE" : "SAFE"} (${result.reason})`);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return results;
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Summarize chain exploitation test results
|
|
228
|
+
*/
|
|
229
|
+
summarizeResults(results) {
|
|
230
|
+
let vulnerable = 0;
|
|
231
|
+
let safe = 0;
|
|
232
|
+
let errors = 0;
|
|
233
|
+
const vulnerableTests = [];
|
|
234
|
+
const categories = new Set();
|
|
235
|
+
for (const [testName, result] of results) {
|
|
236
|
+
if (result.reason === "test_error") {
|
|
237
|
+
errors++;
|
|
238
|
+
}
|
|
239
|
+
else if (result.vulnerable) {
|
|
240
|
+
vulnerable++;
|
|
241
|
+
vulnerableTests.push(testName);
|
|
242
|
+
result.vulnerabilityCategories?.forEach((c) => categories.add(c));
|
|
243
|
+
}
|
|
244
|
+
else {
|
|
245
|
+
safe++;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
return {
|
|
249
|
+
total: results.size,
|
|
250
|
+
vulnerable,
|
|
251
|
+
safe,
|
|
252
|
+
errors,
|
|
253
|
+
vulnerableTests,
|
|
254
|
+
vulnerabilityCategories: Array.from(categories),
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Confidence Scorer
|
|
3
|
+
* Calculates confidence levels for vulnerability detections
|
|
4
|
+
*
|
|
5
|
+
* Extracted from SecurityResponseAnalyzer.ts (Issue #53)
|
|
6
|
+
* Handles: confidence calculation, structured data tool detection, validation pattern checks
|
|
7
|
+
*/
|
|
8
|
+
import { Tool } from "@modelcontextprotocol/sdk/types.js";
|
|
9
|
+
import { SecurityPayload } from "../../../../lib/securityPatterns.js";
|
|
10
|
+
import type { SanitizationDetectionResult } from "./SanitizationDetector.js";
|
|
11
|
+
/**
|
|
12
|
+
* Result of confidence calculation
|
|
13
|
+
*/
|
|
14
|
+
export interface ConfidenceResult {
|
|
15
|
+
confidence: "high" | "medium" | "low";
|
|
16
|
+
requiresManualReview: boolean;
|
|
17
|
+
manualReviewReason?: string;
|
|
18
|
+
reviewGuidance?: string;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Calculates confidence levels for security vulnerability detections
|
|
22
|
+
*/
|
|
23
|
+
export declare class ConfidenceScorer {
|
|
24
|
+
/**
|
|
25
|
+
* Calculate confidence level for a vulnerability detection
|
|
26
|
+
*
|
|
27
|
+
* Factors considered:
|
|
28
|
+
* - Sanitization detection (Issue #56)
|
|
29
|
+
* - Structured data tool context
|
|
30
|
+
* - Evidence quality
|
|
31
|
+
* - Response characteristics
|
|
32
|
+
*
|
|
33
|
+
* @param tool - The tool being tested
|
|
34
|
+
* @param isVulnerable - Whether the tool was flagged as vulnerable
|
|
35
|
+
* @param evidence - Evidence string from vulnerability detection
|
|
36
|
+
* @param responseText - The response text from the tool
|
|
37
|
+
* @param payload - The security payload used for testing
|
|
38
|
+
* @param sanitizationResult - Optional sanitization detection result (Issue #56)
|
|
39
|
+
* @returns Confidence result with manual review requirements
|
|
40
|
+
*/
|
|
41
|
+
calculateConfidence(tool: Tool, isVulnerable: boolean, evidence: string, responseText: string, payload: SecurityPayload, sanitizationResult?: SanitizationDetectionResult): ConfidenceResult;
|
|
42
|
+
/**
|
|
43
|
+
* Check if tool is a structured data tool (search, lookup, retrieval)
|
|
44
|
+
*
|
|
45
|
+
* These tools are more likely to return data containing patterns
|
|
46
|
+
* that look like vulnerabilities but are actually just data.
|
|
47
|
+
*/
|
|
48
|
+
isStructuredDataTool(toolName: string, toolDescription: string): boolean;
|
|
49
|
+
/**
|
|
50
|
+
* Check if evidence pattern is ambiguous (validation-like)
|
|
51
|
+
*
|
|
52
|
+
* Some patterns match both security issues AND normal validation errors.
|
|
53
|
+
* These require more careful analysis.
|
|
54
|
+
*/
|
|
55
|
+
isValidationPattern(evidencePattern: RegExp): boolean;
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=ConfidenceScorer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ConfidenceScorer.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/securityTests/ConfidenceScorer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,oCAAoC,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAC;AAE1E;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,oBAAoB,EAAE,OAAO,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAmCD;;GAEG;AACH,qBAAa,gBAAgB;IAC3B;;;;;;;;;;;;;;;;OAgBG;IACH,mBAAmB,CACjB,IAAI,EAAE,IAAI,EACV,YAAY,EAAE,OAAO,EACrB,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,eAAe,EACxB,kBAAkB,CAAC,EAAE,2BAA2B,GAC/C,gBAAgB;IA4JnB;;;;;OAKG;IACH,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO;IAOxE;;;;;OAKG;IACH,mBAAmB,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO;CAOtD"}
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Confidence Scorer
|
|
3
|
+
* Calculates confidence levels for vulnerability detections
|
|
4
|
+
*
|
|
5
|
+
* Extracted from SecurityResponseAnalyzer.ts (Issue #53)
|
|
6
|
+
* Handles: confidence calculation, structured data tool detection, validation pattern checks
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* Patterns for detecting structured data tools
|
|
10
|
+
*/
|
|
11
|
+
const STRUCTURED_DATA_TOOL_PATTERNS = [
|
|
12
|
+
/search/i,
|
|
13
|
+
/find/i,
|
|
14
|
+
/lookup/i,
|
|
15
|
+
/query/i,
|
|
16
|
+
/retrieve/i,
|
|
17
|
+
/fetch/i,
|
|
18
|
+
/get/i,
|
|
19
|
+
/list/i,
|
|
20
|
+
/resolve/i,
|
|
21
|
+
/discover/i,
|
|
22
|
+
/browse/i,
|
|
23
|
+
];
|
|
24
|
+
/**
|
|
25
|
+
* Ambiguous validation patterns that may cause false positives
|
|
26
|
+
*/
|
|
27
|
+
const VALIDATION_AMBIGUOUS_PATTERNS = [
|
|
28
|
+
"type.*error",
|
|
29
|
+
"invalid.*type",
|
|
30
|
+
"error",
|
|
31
|
+
"invalid",
|
|
32
|
+
"failed",
|
|
33
|
+
"negative.*not.*allowed",
|
|
34
|
+
"must.*be.*positive",
|
|
35
|
+
"invalid.*value",
|
|
36
|
+
"overflow",
|
|
37
|
+
"out.*of.*range",
|
|
38
|
+
];
|
|
39
|
+
/**
|
|
40
|
+
* Calculates confidence levels for security vulnerability detections
|
|
41
|
+
*/
|
|
42
|
+
export class ConfidenceScorer {
|
|
43
|
+
/**
|
|
44
|
+
* Calculate confidence level for a vulnerability detection
|
|
45
|
+
*
|
|
46
|
+
* Factors considered:
|
|
47
|
+
* - Sanitization detection (Issue #56)
|
|
48
|
+
* - Structured data tool context
|
|
49
|
+
* - Evidence quality
|
|
50
|
+
* - Response characteristics
|
|
51
|
+
*
|
|
52
|
+
* @param tool - The tool being tested
|
|
53
|
+
* @param isVulnerable - Whether the tool was flagged as vulnerable
|
|
54
|
+
* @param evidence - Evidence string from vulnerability detection
|
|
55
|
+
* @param responseText - The response text from the tool
|
|
56
|
+
* @param payload - The security payload used for testing
|
|
57
|
+
* @param sanitizationResult - Optional sanitization detection result (Issue #56)
|
|
58
|
+
* @returns Confidence result with manual review requirements
|
|
59
|
+
*/
|
|
60
|
+
calculateConfidence(tool, isVulnerable, evidence, responseText, payload, sanitizationResult) {
|
|
61
|
+
// Issue #56: If sanitization is detected, reduce confidence for vulnerabilities
|
|
62
|
+
// This helps reduce false positives on well-protected servers
|
|
63
|
+
if (isVulnerable && sanitizationResult?.detected) {
|
|
64
|
+
const adjustment = sanitizationResult.totalConfidenceAdjustment;
|
|
65
|
+
// Strong sanitization evidence (adjustment >= 30) - downgrade to low confidence
|
|
66
|
+
// This indicates the tool has specific security libraries in place
|
|
67
|
+
if (adjustment >= 30) {
|
|
68
|
+
const libraries = sanitizationResult.libraries.join(", ") || "general";
|
|
69
|
+
return {
|
|
70
|
+
confidence: "low",
|
|
71
|
+
requiresManualReview: true,
|
|
72
|
+
manualReviewReason: `Sanitization detected (${libraries}). ` +
|
|
73
|
+
`Pattern match may be false positive due to security measures in place.`,
|
|
74
|
+
reviewGuidance: `Tool uses sanitization libraries. Verify if the detected vulnerability ` +
|
|
75
|
+
`actually bypasses the sanitization layer. Check: 1) Does the payload execute ` +
|
|
76
|
+
`after sanitization? 2) Is the sanitization comprehensive for this attack type? ` +
|
|
77
|
+
`3) Evidence: ${sanitizationResult.evidence.join("; ")}`,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
// Moderate sanitization evidence (adjustment >= 15) - downgrade high to medium
|
|
81
|
+
if (adjustment >= 15) {
|
|
82
|
+
const patterns = sanitizationResult.libraries.length > 0
|
|
83
|
+
? sanitizationResult.libraries.join(", ")
|
|
84
|
+
: sanitizationResult.genericPatterns.join(", ");
|
|
85
|
+
return {
|
|
86
|
+
confidence: "medium",
|
|
87
|
+
requiresManualReview: true,
|
|
88
|
+
manualReviewReason: `Sanitization patterns detected (${patterns}). Verify actual vulnerability.`,
|
|
89
|
+
reviewGuidance: `Tool mentions sanitization in description or shows sanitization in response. ` +
|
|
90
|
+
`Verify if the detected pattern represents actual code execution or if it's ` +
|
|
91
|
+
`safely handled. Evidence: ${sanitizationResult.evidence.join("; ")}`,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const toolDescription = (tool.description || "").toLowerCase();
|
|
96
|
+
const toolName = tool.name.toLowerCase();
|
|
97
|
+
const responseLower = responseText.toLowerCase();
|
|
98
|
+
const payloadLower = payload.payload.toLowerCase();
|
|
99
|
+
// HIGH CONFIDENCE: Clear cases
|
|
100
|
+
if (!isVulnerable &&
|
|
101
|
+
(evidence.includes("safely reflected") ||
|
|
102
|
+
evidence.includes("API wrapper") ||
|
|
103
|
+
evidence.includes("safe: true"))) {
|
|
104
|
+
return {
|
|
105
|
+
confidence: "high",
|
|
106
|
+
requiresManualReview: false,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
if (isVulnerable &&
|
|
110
|
+
evidence.includes("executed") &&
|
|
111
|
+
!this.isStructuredDataTool(toolName, toolDescription)) {
|
|
112
|
+
return {
|
|
113
|
+
confidence: "high",
|
|
114
|
+
requiresManualReview: false,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
// LOW CONFIDENCE: Ambiguous pattern matches in structured data
|
|
118
|
+
if (isVulnerable) {
|
|
119
|
+
const isDataTool = this.isStructuredDataTool(toolName, toolDescription);
|
|
120
|
+
const hasStructuredData = /title:|name:|description:|trust score:|id:|snippets:/i.test(responseText) ||
|
|
121
|
+
/^\s*-\s+/m.test(responseText) ||
|
|
122
|
+
/"[^"]+"\s*:\s*"[^"]+"/g.test(responseText);
|
|
123
|
+
const patternInInput = payload.evidence?.test(payloadLower);
|
|
124
|
+
const echosInput = responseLower.includes(payloadLower);
|
|
125
|
+
if (isDataTool && (hasStructuredData || echosInput) && patternInInput) {
|
|
126
|
+
return {
|
|
127
|
+
confidence: "low",
|
|
128
|
+
requiresManualReview: true,
|
|
129
|
+
manualReviewReason: "Pattern matched in structured data response. Tool may be legitimately " +
|
|
130
|
+
"returning data containing search terms rather than executing malicious code.",
|
|
131
|
+
reviewGuidance: "Verify: 1) Does the tool actually execute/compute the input? " +
|
|
132
|
+
"2) Or does it just return pre-existing data that happens to contain the pattern? " +
|
|
133
|
+
`3) Check if '${payload.evidence}' appears in legitimate tool output vs. execution results.`,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
if (payload.evidence &&
|
|
137
|
+
/\b\d\b/.test(payload.evidence.toString()) &&
|
|
138
|
+
/\b(score|count|trust|rating|id|version)\b/i.test(responseText)) {
|
|
139
|
+
return {
|
|
140
|
+
confidence: "low",
|
|
141
|
+
requiresManualReview: true,
|
|
142
|
+
manualReviewReason: "Numeric pattern found in response with numeric metadata (scores, counts, etc.). " +
|
|
143
|
+
"May be coincidental data rather than arithmetic execution.",
|
|
144
|
+
reviewGuidance: "Verify: 1) Did the tool actually compute an arithmetic result? " +
|
|
145
|
+
"2) Or does the number appear in metadata like trust scores, version numbers, or counts? " +
|
|
146
|
+
"3) Compare pattern location in response with tool's expected output format.",
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
if (/admin|role|privilege|elevated/i.test(payload.payload) &&
|
|
150
|
+
/\b(library|search|documentation|api|wrapper)\b/i.test(toolDescription)) {
|
|
151
|
+
return {
|
|
152
|
+
confidence: "low",
|
|
153
|
+
requiresManualReview: true,
|
|
154
|
+
manualReviewReason: "Admin-related keywords found in search/retrieval tool results. " +
|
|
155
|
+
"Tool may be returning data about admin-related libraries/APIs rather than elevating privileges.",
|
|
156
|
+
reviewGuidance: "Verify: 1) Did the tool actually change behavior or assume admin role? " +
|
|
157
|
+
"2) Or did it return search results for admin-related content? " +
|
|
158
|
+
"3) Test if tool behavior actually changed after this request.",
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
// MEDIUM CONFIDENCE: Execution evidence but some ambiguity
|
|
163
|
+
if (isVulnerable && evidence.includes("executed")) {
|
|
164
|
+
return {
|
|
165
|
+
confidence: "medium",
|
|
166
|
+
requiresManualReview: true,
|
|
167
|
+
manualReviewReason: "Execution indicators found but context suggests possible ambiguity.",
|
|
168
|
+
reviewGuidance: "Verify: 1) Review the full response to confirm actual code execution. " +
|
|
169
|
+
"2) Check if tool's intended function involves execution. " +
|
|
170
|
+
"3) Test with variations to confirm consistency.",
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
// Default: HIGH confidence for clear safe cases
|
|
174
|
+
return {
|
|
175
|
+
confidence: "high",
|
|
176
|
+
requiresManualReview: false,
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Check if tool is a structured data tool (search, lookup, retrieval)
|
|
181
|
+
*
|
|
182
|
+
* These tools are more likely to return data containing patterns
|
|
183
|
+
* that look like vulnerabilities but are actually just data.
|
|
184
|
+
*/
|
|
185
|
+
isStructuredDataTool(toolName, toolDescription) {
|
|
186
|
+
const combined = `${toolName} ${toolDescription}`;
|
|
187
|
+
return STRUCTURED_DATA_TOOL_PATTERNS.some((pattern) => pattern.test(combined));
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Check if evidence pattern is ambiguous (validation-like)
|
|
191
|
+
*
|
|
192
|
+
* Some patterns match both security issues AND normal validation errors.
|
|
193
|
+
* These require more careful analysis.
|
|
194
|
+
*/
|
|
195
|
+
isValidationPattern(evidencePattern) {
|
|
196
|
+
const patternStr = evidencePattern.toString().toLowerCase();
|
|
197
|
+
return VALIDATION_AMBIGUOUS_PATTERNS.some((ambiguous) => patternStr.includes(ambiguous));
|
|
198
|
+
}
|
|
199
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cross-Tool State Tester
|
|
3
|
+
* Tests for privilege escalation by calling tools in sequence
|
|
4
|
+
*
|
|
5
|
+
* Issue #92, Challenge #7: Cross-tool state-based authorization bypass
|
|
6
|
+
* Detects when one tool can modify shared state that affects another tool's authorization.
|
|
7
|
+
*
|
|
8
|
+
* Attack flow:
|
|
9
|
+
* 1. Call admin_action → should get "access denied"
|
|
10
|
+
* 2. Call config_modifier with "admin_mode=true"
|
|
11
|
+
* 3. Call admin_action again → if now succeeds, VULNERABLE
|
|
12
|
+
*/
|
|
13
|
+
import { CompatibilityCallToolResult, Tool } from "@modelcontextprotocol/sdk/types.js";
|
|
14
|
+
import { ProgressCallback } from "../../../../lib/assessment/progressTypes.js";
|
|
15
|
+
/**
|
|
16
|
+
* Function type for calling MCP tools
|
|
17
|
+
*/
|
|
18
|
+
export type CallToolFunction = (name: string, params: Record<string, unknown>) => Promise<CompatibilityCallToolResult>;
|
|
19
|
+
/**
|
|
20
|
+
* Result of cross-tool privilege escalation test
|
|
21
|
+
*/
|
|
22
|
+
export interface CrossToolTestResult {
|
|
23
|
+
vulnerable: boolean;
|
|
24
|
+
reason: "privilege_escalation_confirmed" | "escalation_blocked" | "baseline_has_access" | "modifier_rejected" | "test_error";
|
|
25
|
+
evidence?: {
|
|
26
|
+
baseline: string;
|
|
27
|
+
afterModifier: string;
|
|
28
|
+
enableResult?: string;
|
|
29
|
+
};
|
|
30
|
+
error?: string;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Identified tool pair for cross-tool testing
|
|
34
|
+
*/
|
|
35
|
+
export interface ToolPair {
|
|
36
|
+
admin: Tool;
|
|
37
|
+
modifier: Tool;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Configuration for cross-tool state testing
|
|
41
|
+
*/
|
|
42
|
+
export interface CrossToolTestConfig {
|
|
43
|
+
/** Timeout for each tool call in ms (default: 5000) */
|
|
44
|
+
timeout?: number;
|
|
45
|
+
/** Enable verbose logging */
|
|
46
|
+
verbose?: boolean;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Tests for cross-tool privilege escalation via shared mutable state
|
|
50
|
+
*/
|
|
51
|
+
export declare class CrossToolStateTester {
|
|
52
|
+
private readonly verbose;
|
|
53
|
+
constructor(config?: CrossToolTestConfig);
|
|
54
|
+
/**
|
|
55
|
+
* Log message if verbose logging is enabled
|
|
56
|
+
*/
|
|
57
|
+
private log;
|
|
58
|
+
/**
|
|
59
|
+
* Identify potential cross-tool pairs for testing
|
|
60
|
+
* Looks for admin_action/privileged tools and config_modifier/setting tools
|
|
61
|
+
*/
|
|
62
|
+
identifyCrossToolPairs(tools: Tool[]): ToolPair[];
|
|
63
|
+
/**
|
|
64
|
+
* Test cross-tool privilege escalation
|
|
65
|
+
*
|
|
66
|
+
* Attack flow:
|
|
67
|
+
* 1. Call admin_action → expect "access denied"
|
|
68
|
+
* 2. Call config_modifier with "admin_mode=true"
|
|
69
|
+
* 3. Call admin_action again → if now succeeds, VULNERABLE
|
|
70
|
+
*/
|
|
71
|
+
testPrivilegeEscalation(callTool: CallToolFunction, adminTool: Tool, modifierTool: Tool, onProgress?: ProgressCallback): Promise<CrossToolTestResult>;
|
|
72
|
+
/**
|
|
73
|
+
* Run sequence tests on all identified tool pairs
|
|
74
|
+
*/
|
|
75
|
+
runAllSequenceTests(tools: Tool[], callTool: CallToolFunction, onProgress?: ProgressCallback): Promise<Map<string, CrossToolTestResult>>;
|
|
76
|
+
/**
|
|
77
|
+
* Get summary of sequence test results
|
|
78
|
+
*/
|
|
79
|
+
summarizeResults(results: Map<string, CrossToolTestResult>): {
|
|
80
|
+
total: number;
|
|
81
|
+
vulnerable: number;
|
|
82
|
+
safe: number;
|
|
83
|
+
errors: number;
|
|
84
|
+
vulnerablePairs: string[];
|
|
85
|
+
};
|
|
86
|
+
/**
|
|
87
|
+
* Extract text content from MCP response
|
|
88
|
+
*/
|
|
89
|
+
private extractResponseText;
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=CrossToolStateTester.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CrossToolStateTester.d.ts","sourceRoot":"","sources":["../../../../../src/services/assessment/modules/securityTests/CrossToolStateTester.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EACL,2BAA2B,EAC3B,IAAI,EACL,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAElE;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,CAC7B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC5B,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE1C;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,OAAO,CAAC;IACpB,MAAM,EACF,gCAAgC,GAChC,oBAAoB,GACpB,qBAAqB,GACrB,mBAAmB,GACnB,YAAY,CAAC;IACjB,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;QACjB,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,IAAI,CAAC;IACZ,QAAQ,EAAE,IAAI,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;gBAEtB,MAAM,GAAE,mBAAwB;IAK5C;;OAEG;IACH,OAAO,CAAC,GAAG;IAOX;;;OAGG;IACH,sBAAsB,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,QAAQ,EAAE;IA8BjD;;;;;;;OAOG;IACG,uBAAuB,CAC3B,QAAQ,EAAE,gBAAgB,EAC1B,SAAS,EAAE,IAAI,EACf,YAAY,EAAE,IAAI,EAClB,UAAU,CAAC,EAAE,gBAAgB,GAC5B,OAAO,CAAC,mBAAmB,CAAC;IAkI/B;;OAEG;IACG,mBAAmB,CACvB,KAAK,EAAE,IAAI,EAAE,EACb,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,CAAC,EAAE,gBAAgB,GAC5B,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;IAkB5C;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,GAAG;QAC3D,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,eAAe,EAAE,MAAM,EAAE,CAAC;KAC3B;IA0BD;;OAEG;IACH,OAAO,CAAC,mBAAmB;CAkB5B"}
|