@bryan-thompson/inspector-assessment-client 1.25.0 → 1.25.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-CNC5_mEQ.js → OAuthCallback-DE62cdTZ.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-BstXB61i.js → OAuthDebugCallback-CWjFdCIE.js} +1 -1
- package/dist/assets/{index-jmzR9VGa.js → index-PCQVSwHa.js} +4 -4
- package/dist/index.html +1 -1
- package/lib/lib/assessment/configTypes.d.ts +3 -0
- package/lib/lib/assessment/configTypes.d.ts.map +1 -1
- package/lib/lib/assessment/configTypes.js +11 -6
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +1 -5
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +36 -236
- package/lib/services/assessment/ToolClassifier.d.ts +154 -27
- package/lib/services/assessment/ToolClassifier.d.ts.map +1 -1
- package/lib/services/assessment/ToolClassifier.js +171 -326
- package/lib/services/assessment/orchestratorHelpers.d.ts +83 -0
- package/lib/services/assessment/orchestratorHelpers.d.ts.map +1 -0
- package/lib/services/assessment/orchestratorHelpers.js +212 -0
- package/lib/services/assessment/tool-classifier-patterns.d.ts +84 -0
- package/lib/services/assessment/tool-classifier-patterns.d.ts.map +1 -0
- package/lib/services/assessment/tool-classifier-patterns.js +348 -0
- package/package.json +1 -1
|
@@ -3,312 +3,121 @@
|
|
|
3
3
|
* Categorizes MCP tools based on name/description to select appropriate security test patterns
|
|
4
4
|
*
|
|
5
5
|
* Validated against broken-mcp server with 16 tools (6 HIGH, 4 MEDIUM, 6 SAFE)
|
|
6
|
+
*
|
|
7
|
+
* ## Pattern Matching Design
|
|
8
|
+
*
|
|
9
|
+
* This classifier uses two types of regex patterns intentionally:
|
|
10
|
+
*
|
|
11
|
+
* 1. **Substring patterns** (e.g., `/calculator/i`): Match anywhere in the text.
|
|
12
|
+
* Used for HIGH-risk category keywords that should trigger even when embedded.
|
|
13
|
+
* Example: "recalculator_v2" matches CALCULATOR because any calculator-like
|
|
14
|
+
* tool warrants security scrutiny.
|
|
15
|
+
*
|
|
16
|
+
* 2. **Word boundary patterns** (e.g., `/\bget\b/i`): Match isolated words only.
|
|
17
|
+
* Used for common words that would cause false positives as substrings.
|
|
18
|
+
* Example: "target_selector" should NOT match DATA_ACCESS's `/\bget\b/` pattern.
|
|
19
|
+
*
|
|
20
|
+
* **Underscore vs Hyphen Behavior:**
|
|
21
|
+
* - Word boundaries (`\b`) treat hyphens as boundaries but underscores as word characters
|
|
22
|
+
* - `api-get-data` matches `/\bget\b/` (hyphen is boundary)
|
|
23
|
+
* - `api_get_data` does NOT match `/\bget\b/` (underscore is word char)
|
|
24
|
+
* - This is intentional: underscore-joined names are typically single identifiers
|
|
25
|
+
*
|
|
26
|
+
* See tests in ToolClassifier.test.ts for comprehensive pattern behavior validation.
|
|
27
|
+
*
|
|
28
|
+
* @module ToolClassifier
|
|
6
29
|
*/
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
ToolCategory["SYSTEM_EXEC"] = "system_exec";
|
|
11
|
-
ToolCategory["CODE_EXECUTOR"] = "code_executor";
|
|
12
|
-
ToolCategory["DATA_ACCESS"] = "data_access";
|
|
13
|
-
ToolCategory["TOOL_OVERRIDE"] = "tool_override";
|
|
14
|
-
ToolCategory["CONFIG_MODIFIER"] = "config_modifier";
|
|
15
|
-
ToolCategory["URL_FETCHER"] = "fetcher";
|
|
16
|
-
ToolCategory["UNICODE_PROCESSOR"] = "unicode";
|
|
17
|
-
ToolCategory["JSON_PARSER"] = "parser";
|
|
18
|
-
ToolCategory["PACKAGE_INSTALLER"] = "installer";
|
|
19
|
-
ToolCategory["RUG_PULL"] = "rug_pull";
|
|
20
|
-
ToolCategory["SAFE_STORAGE"] = "safe_storage";
|
|
21
|
-
ToolCategory["API_WRAPPER"] = "api_wrapper";
|
|
22
|
-
ToolCategory["SEARCH_RETRIEVAL"] = "search_retrieval";
|
|
23
|
-
ToolCategory["CRUD_CREATION"] = "crud_creation";
|
|
24
|
-
ToolCategory["READ_ONLY_INFO"] = "read_only_info";
|
|
25
|
-
ToolCategory["GENERIC"] = "generic";
|
|
26
|
-
})(ToolCategory || (ToolCategory = {}));
|
|
30
|
+
import { CATEGORY_PATTERNS, CATEGORY_CHECK_ORDER, GENERIC_CONFIG, ToolCategory, } from "./tool-classifier-patterns.js";
|
|
31
|
+
// Re-export types for backwards compatibility
|
|
32
|
+
export { ToolCategory };
|
|
27
33
|
/**
|
|
28
34
|
* Classifies MCP tools into vulnerability categories based on naming patterns
|
|
29
|
-
* and descriptions. Uses patterns
|
|
35
|
+
* and descriptions. Uses pre-compiled patterns for optimal performance.
|
|
36
|
+
*
|
|
37
|
+
* The classifier is stateless and thread-safe - multiple classifications can
|
|
38
|
+
* run concurrently without interference.
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* ```typescript
|
|
42
|
+
* const classifier = new ToolClassifier();
|
|
43
|
+
*
|
|
44
|
+
* // Single classification
|
|
45
|
+
* const result = classifier.classify('vulnerable_calculator_tool');
|
|
46
|
+
* console.log(result.categories); // [ToolCategory.CALCULATOR]
|
|
47
|
+
* console.log(result.confidence); // 90
|
|
48
|
+
*
|
|
49
|
+
* // Batch classification
|
|
50
|
+
* const tools = [
|
|
51
|
+
* { name: 'calculator_tool' },
|
|
52
|
+
* { name: 'search_api', description: 'Search for documents' }
|
|
53
|
+
* ];
|
|
54
|
+
* const results = classifier.classifyBatch(tools);
|
|
55
|
+
* ```
|
|
30
56
|
*/
|
|
31
57
|
export class ToolClassifier {
|
|
58
|
+
/** Maximum input length to prevent ReDoS with pathological inputs */
|
|
59
|
+
static MAX_INPUT_LENGTH = 10000;
|
|
32
60
|
/**
|
|
33
|
-
* Classify a tool into one or more categories
|
|
34
|
-
*
|
|
61
|
+
* Classify a tool into one or more security risk categories.
|
|
62
|
+
*
|
|
63
|
+
* The classifier analyzes both the tool name and optional description,
|
|
64
|
+
* matching against pre-compiled regex patterns for each category.
|
|
65
|
+
* A tool may match multiple categories if it contains multiple patterns.
|
|
66
|
+
*
|
|
67
|
+
* @param toolName - The MCP tool name to classify (e.g., "vulnerable_calculator_tool")
|
|
68
|
+
* @param description - Optional tool description for additional pattern matching
|
|
69
|
+
* @returns Classification result with categories, confidence score (0-100), and reasoning
|
|
70
|
+
*
|
|
71
|
+
* @example
|
|
72
|
+
* ```typescript
|
|
73
|
+
* const classifier = new ToolClassifier();
|
|
74
|
+
*
|
|
75
|
+
* // Basic classification by name
|
|
76
|
+
* const calc = classifier.classify('calculator_tool');
|
|
77
|
+
* // { toolName: 'calculator_tool', categories: ['calculator'], confidence: 90, ... }
|
|
78
|
+
*
|
|
79
|
+
* // Classification with description
|
|
80
|
+
* const tool = classifier.classify('my_tool', 'Executes shell commands');
|
|
81
|
+
* // { toolName: 'my_tool', categories: ['system_exec'], confidence: 95, ... }
|
|
82
|
+
*
|
|
83
|
+
* // Multi-category match
|
|
84
|
+
* const multi = classifier.classify('calc_exec_command');
|
|
85
|
+
* // { categories: ['calculator', 'system_exec'], confidence: 92, ... }
|
|
86
|
+
* ```
|
|
87
|
+
*
|
|
88
|
+
* @throws Never throws - returns GENERIC category for invalid inputs
|
|
35
89
|
*/
|
|
36
90
|
classify(toolName, description) {
|
|
91
|
+
// Defensive validation for runtime safety (handles JS callers, deserialized data)
|
|
92
|
+
const safeName = typeof toolName === "string" ? toolName : "";
|
|
93
|
+
const safeDesc = typeof description === "string" ? description : "";
|
|
94
|
+
// Handle invalid or empty tool name
|
|
95
|
+
if (!safeName.trim()) {
|
|
96
|
+
return {
|
|
97
|
+
toolName: safeName,
|
|
98
|
+
categories: [ToolCategory.GENERIC],
|
|
99
|
+
confidence: 0,
|
|
100
|
+
reasoning: "Invalid or empty tool name provided",
|
|
101
|
+
};
|
|
102
|
+
}
|
|
37
103
|
const categories = [];
|
|
38
104
|
const confidenceScores = [];
|
|
39
105
|
const reasons = [];
|
|
40
|
-
const toolText = `${
|
|
41
|
-
//
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
/arithmetic/i,
|
|
50
|
-
/expression/i,
|
|
51
|
-
])) {
|
|
52
|
-
categories.push(ToolCategory.CALCULATOR);
|
|
53
|
-
confidenceScores.push(90);
|
|
54
|
-
reasons.push("Calculator pattern detected (arithmetic execution risk)");
|
|
55
|
-
}
|
|
56
|
-
// System execution tools (HIGH RISK)
|
|
57
|
-
// Validated: vulnerable_system_exec_tool
|
|
58
|
-
if (this.matchesPattern(toolText, [
|
|
59
|
-
/system.*exec/i,
|
|
60
|
-
/exec.*tool/i,
|
|
61
|
-
/command/i,
|
|
62
|
-
/shell/i,
|
|
63
|
-
/\brun\b/i,
|
|
64
|
-
/execute/i,
|
|
65
|
-
/process/i,
|
|
66
|
-
])) {
|
|
67
|
-
categories.push(ToolCategory.SYSTEM_EXEC);
|
|
68
|
-
confidenceScores.push(95);
|
|
69
|
-
reasons.push("System execution pattern detected (command injection risk)");
|
|
70
|
-
}
|
|
71
|
-
// Code execution tools (HIGH RISK)
|
|
72
|
-
// Tools that execute arbitrary code in specific languages (Python, JavaScript, etc.)
|
|
73
|
-
// These require language-specific payloads, not shell commands
|
|
74
|
-
if (this.matchesPattern(toolText, [
|
|
75
|
-
/execute.*code/i,
|
|
76
|
-
/run.*code/i,
|
|
77
|
-
/code.*execut/i,
|
|
78
|
-
/run.*script/i,
|
|
79
|
-
/exec.*script/i,
|
|
80
|
-
/\bpython.*code\b/i,
|
|
81
|
-
/\bjavascript.*code\b/i,
|
|
82
|
-
/\bjs.*code\b/i,
|
|
83
|
-
/\beval.*code\b/i,
|
|
84
|
-
/code.*runner/i,
|
|
85
|
-
/script.*runner/i,
|
|
86
|
-
/\bexec\b.*\b(python|js|javascript)\b/i,
|
|
87
|
-
/\b(python|js|javascript)\b.*\bexec\b/i,
|
|
88
|
-
/interpret/i,
|
|
89
|
-
/\brepl\b/i,
|
|
90
|
-
])) {
|
|
91
|
-
categories.push(ToolCategory.CODE_EXECUTOR);
|
|
92
|
-
confidenceScores.push(95);
|
|
93
|
-
reasons.push("Code executor pattern detected (arbitrary code execution risk)");
|
|
94
|
-
}
|
|
95
|
-
// Data access/leak tools (HIGH RISK)
|
|
96
|
-
// Validated: vulnerable_data_leak_tool
|
|
97
|
-
if (this.matchesPattern(toolText, [
|
|
98
|
-
/leak/i,
|
|
99
|
-
/\bdata\b/i,
|
|
100
|
-
/show/i,
|
|
101
|
-
/\bget\b/i,
|
|
102
|
-
/\blist\b/i,
|
|
103
|
-
/display/i,
|
|
104
|
-
/\benv/i,
|
|
105
|
-
/secret/i,
|
|
106
|
-
/\bkey\b/i,
|
|
107
|
-
/credential/i,
|
|
108
|
-
/exfiltrat/i,
|
|
109
|
-
])) {
|
|
110
|
-
categories.push(ToolCategory.DATA_ACCESS);
|
|
111
|
-
confidenceScores.push(85);
|
|
112
|
-
reasons.push("Data access pattern detected (data exfiltration risk)");
|
|
113
|
-
}
|
|
114
|
-
// Tool override/shadowing (HIGH RISK)
|
|
115
|
-
// Validated: vulnerable_tool_override_tool
|
|
116
|
-
if (this.matchesPattern(toolText, [
|
|
117
|
-
/override/i,
|
|
118
|
-
/shadow/i,
|
|
119
|
-
/poison/i,
|
|
120
|
-
/create.*tool/i,
|
|
121
|
-
/register.*tool/i,
|
|
122
|
-
/define.*tool/i,
|
|
123
|
-
/tool.*creator/i,
|
|
124
|
-
/add.*tool/i,
|
|
125
|
-
])) {
|
|
126
|
-
categories.push(ToolCategory.TOOL_OVERRIDE);
|
|
127
|
-
confidenceScores.push(92);
|
|
128
|
-
reasons.push("Tool override pattern detected (shadowing/poisoning risk)");
|
|
129
|
-
}
|
|
130
|
-
// Config modification tools (HIGH RISK)
|
|
131
|
-
// Validated: vulnerable_config_modifier_tool
|
|
132
|
-
if (this.matchesPattern(toolText, [
|
|
133
|
-
/config/i,
|
|
134
|
-
/setting/i,
|
|
135
|
-
/modifier/i,
|
|
136
|
-
/\badmin\b/i,
|
|
137
|
-
/privilege/i,
|
|
138
|
-
/permission/i,
|
|
139
|
-
/configure/i,
|
|
140
|
-
/drift/i,
|
|
141
|
-
])) {
|
|
142
|
-
categories.push(ToolCategory.CONFIG_MODIFIER);
|
|
143
|
-
confidenceScores.push(88);
|
|
144
|
-
reasons.push("Config modification pattern detected (configuration drift risk)");
|
|
145
|
-
}
|
|
146
|
-
// URL fetching tools (HIGH RISK)
|
|
147
|
-
// Validated: vulnerable_fetcher_tool
|
|
148
|
-
if (this.matchesPattern(toolText, [
|
|
149
|
-
/fetch/i,
|
|
150
|
-
/\burl\b/i,
|
|
151
|
-
/http/i,
|
|
152
|
-
/download/i,
|
|
153
|
-
/load/i,
|
|
154
|
-
/retrieve/i,
|
|
155
|
-
/\bget\b.*url/i,
|
|
156
|
-
/external/i,
|
|
157
|
-
])) {
|
|
158
|
-
categories.push(ToolCategory.URL_FETCHER);
|
|
159
|
-
confidenceScores.push(87);
|
|
160
|
-
reasons.push("URL fetcher pattern detected (indirect prompt injection risk)");
|
|
161
|
-
}
|
|
162
|
-
// Unicode processing tools (MEDIUM RISK)
|
|
163
|
-
// Validated: vulnerable_unicode_processor_tool
|
|
164
|
-
if (this.matchesPattern(toolText, [
|
|
165
|
-
/unicode/i,
|
|
166
|
-
/encode/i,
|
|
167
|
-
/decode/i,
|
|
168
|
-
/charset/i,
|
|
169
|
-
/utf/i,
|
|
170
|
-
/hex/i,
|
|
171
|
-
/escape/i,
|
|
172
|
-
])) {
|
|
173
|
-
categories.push(ToolCategory.UNICODE_PROCESSOR);
|
|
174
|
-
confidenceScores.push(75);
|
|
175
|
-
reasons.push("Unicode processor pattern detected (bypass encoding risk)");
|
|
176
|
-
}
|
|
177
|
-
// JSON/nested parsing tools (MEDIUM RISK)
|
|
178
|
-
// Validated: vulnerable_nested_parser_tool
|
|
179
|
-
if (this.matchesPattern(toolText, [
|
|
180
|
-
/parser/i,
|
|
181
|
-
/parse/i,
|
|
182
|
-
/json/i,
|
|
183
|
-
/xml/i,
|
|
184
|
-
/yaml/i,
|
|
185
|
-
/nested/i,
|
|
186
|
-
/deserialize/i,
|
|
187
|
-
/unmarshal/i,
|
|
188
|
-
])) {
|
|
189
|
-
categories.push(ToolCategory.JSON_PARSER);
|
|
190
|
-
confidenceScores.push(78);
|
|
191
|
-
reasons.push("JSON/nested parser pattern detected (nested injection risk)");
|
|
192
|
-
}
|
|
193
|
-
// Package installation tools (MEDIUM RISK)
|
|
194
|
-
// Validated: vulnerable_package_installer_tool
|
|
195
|
-
if (this.matchesPattern(toolText, [
|
|
196
|
-
/install/i,
|
|
197
|
-
/package/i,
|
|
198
|
-
/\bnpm\b/i,
|
|
199
|
-
/\bpip\b/i,
|
|
200
|
-
/dependency/i,
|
|
201
|
-
/module/i,
|
|
202
|
-
/library/i,
|
|
203
|
-
/\bgem\b/i,
|
|
204
|
-
])) {
|
|
205
|
-
categories.push(ToolCategory.PACKAGE_INSTALLER);
|
|
206
|
-
confidenceScores.push(70);
|
|
207
|
-
reasons.push("Package installer pattern detected (typosquatting risk)");
|
|
208
|
-
}
|
|
209
|
-
// Rug pull (behavioral change over time) (MEDIUM RISK)
|
|
210
|
-
// Validated: vulnerable_rug_pull_tool
|
|
211
|
-
if (this.matchesPattern(toolText, [
|
|
212
|
-
/rug.*pull/i,
|
|
213
|
-
/trust/i,
|
|
214
|
-
/behavior.*change/i,
|
|
215
|
-
/malicious.*after/i,
|
|
216
|
-
/invocation.*count/i,
|
|
217
|
-
])) {
|
|
218
|
-
categories.push(ToolCategory.RUG_PULL);
|
|
219
|
-
confidenceScores.push(80);
|
|
220
|
-
reasons.push("Rug pull pattern detected (behavioral change risk)");
|
|
221
|
-
}
|
|
222
|
-
// API wrapper tools (SAFE - data passing, not code execution)
|
|
223
|
-
// These tools call external APIs and return data as text, not execute it as code
|
|
224
|
-
// Examples: Firecrawl (scrape, crawl, search), HTTP clients, REST/GraphQL clients
|
|
225
|
-
if (this.matchesPattern(toolText, [
|
|
226
|
-
/firecrawl/i,
|
|
227
|
-
/\bscrape\b/i,
|
|
228
|
-
/\bcrawl\b/i,
|
|
229
|
-
/web.*scraping/i,
|
|
230
|
-
/api.*wrapper/i,
|
|
231
|
-
/http.*client/i,
|
|
232
|
-
/web.*client/i,
|
|
233
|
-
/rest.*client/i,
|
|
234
|
-
/graphql.*client/i,
|
|
235
|
-
/fetch.*web.*content/i,
|
|
236
|
-
])) {
|
|
237
|
-
categories.push(ToolCategory.API_WRAPPER);
|
|
238
|
-
confidenceScores.push(95);
|
|
239
|
-
reasons.push("API wrapper pattern detected (safe data passing, not code execution)");
|
|
240
|
-
}
|
|
241
|
-
// Search and retrieval tools (SAFE - returns search results/data, not code execution)
|
|
242
|
-
// Examples: notion-search, notion-query-database, search, find, lookup
|
|
243
|
-
if (this.matchesPattern(toolText, [
|
|
244
|
-
/\bsearch\b/i,
|
|
245
|
-
/\bfind\b/i,
|
|
246
|
-
/\blookup\b/i,
|
|
247
|
-
/\bquery\b/i,
|
|
248
|
-
/retrieve/i,
|
|
249
|
-
/\blist\b/i,
|
|
250
|
-
/get.*users/i,
|
|
251
|
-
/get.*pages/i,
|
|
252
|
-
/get.*database/i,
|
|
253
|
-
])) {
|
|
254
|
-
categories.push(ToolCategory.SEARCH_RETRIEVAL);
|
|
255
|
-
confidenceScores.push(93);
|
|
256
|
-
reasons.push("Search/retrieval pattern detected (returns data, not code execution)");
|
|
257
|
-
}
|
|
258
|
-
// CRUD creation/modification tools (SAFE - creates/modifies resources, not code execution)
|
|
259
|
-
// Examples: notion-create-database, notion-create-page, create, add, insert, update
|
|
260
|
-
if (this.matchesPattern(toolText, [
|
|
261
|
-
/\bcreate\b/i,
|
|
262
|
-
/\badd\b/i,
|
|
263
|
-
/\binsert\b/i,
|
|
264
|
-
/\bupdate\b/i,
|
|
265
|
-
/\bmodify\b/i,
|
|
266
|
-
/\bdelete\b/i,
|
|
267
|
-
/\bduplicate\b/i,
|
|
268
|
-
/\bmove\b/i,
|
|
269
|
-
/\bappend\b/i,
|
|
270
|
-
])) {
|
|
271
|
-
categories.push(ToolCategory.CRUD_CREATION);
|
|
272
|
-
confidenceScores.push(92);
|
|
273
|
-
reasons.push("CRUD operation pattern detected (data manipulation, not code execution)");
|
|
274
|
-
}
|
|
275
|
-
// Read-only info tools (SAFE - returns user/workspace info, intended data exposure)
|
|
276
|
-
// Examples: notion-get-self, notion-get-teams, get-self, whoami, get-info, get-status
|
|
277
|
-
if (this.matchesPattern(toolText, [
|
|
278
|
-
/get.*self/i,
|
|
279
|
-
/get.*teams/i,
|
|
280
|
-
/get.*info/i,
|
|
281
|
-
/get.*status/i,
|
|
282
|
-
/\bwhoami\b/i,
|
|
283
|
-
/get.*workspace/i,
|
|
284
|
-
/get.*user/i,
|
|
285
|
-
/current.*user/i,
|
|
286
|
-
])) {
|
|
287
|
-
categories.push(ToolCategory.READ_ONLY_INFO);
|
|
288
|
-
confidenceScores.push(94);
|
|
289
|
-
reasons.push("Read-only info pattern detected (intended data exposure, not vulnerability)");
|
|
290
|
-
}
|
|
291
|
-
// Safe storage tools (CONTROL GROUP - should never show vulnerabilities)
|
|
292
|
-
// Validated: safe_storage_tool_mcp, safe_search_tool_mcp, safe_list_tool_mcp,
|
|
293
|
-
// safe_info_tool_mcp, safe_echo_tool_mcp, safe_validate_tool_mcp
|
|
294
|
-
if (this.matchesPattern(toolText, [
|
|
295
|
-
/safe.*storage/i,
|
|
296
|
-
/safe.*search/i,
|
|
297
|
-
/safe.*list/i,
|
|
298
|
-
/safe.*info/i,
|
|
299
|
-
/safe.*echo/i,
|
|
300
|
-
/safe.*validate/i,
|
|
301
|
-
/safe.*tool/i,
|
|
302
|
-
])) {
|
|
303
|
-
categories.push(ToolCategory.SAFE_STORAGE);
|
|
304
|
-
confidenceScores.push(99);
|
|
305
|
-
reasons.push("Safe tool pattern detected (control group - should be safe)");
|
|
106
|
+
const toolText = `${safeName} ${safeDesc}`.toLowerCase();
|
|
107
|
+
// Check each category in defined order (HIGH -> MEDIUM -> LOW)
|
|
108
|
+
for (const category of CATEGORY_CHECK_ORDER) {
|
|
109
|
+
const config = CATEGORY_PATTERNS[category];
|
|
110
|
+
if (this.matchesPattern(toolText, config.patterns)) {
|
|
111
|
+
categories.push(category);
|
|
112
|
+
confidenceScores.push(config.confidence);
|
|
113
|
+
reasons.push(config.reasoning);
|
|
114
|
+
}
|
|
306
115
|
}
|
|
307
116
|
// Default to generic if no specific matches
|
|
308
117
|
if (categories.length === 0) {
|
|
309
118
|
categories.push(ToolCategory.GENERIC);
|
|
310
|
-
confidenceScores.push(
|
|
311
|
-
reasons.push(
|
|
119
|
+
confidenceScores.push(GENERIC_CONFIG.confidence);
|
|
120
|
+
reasons.push(GENERIC_CONFIG.reasoning);
|
|
312
121
|
}
|
|
313
122
|
// Calculate overall confidence (average of matched pattern confidences)
|
|
314
123
|
const avgConfidence = confidenceScores.reduce((a, b) => a + b, 0) / confidenceScores.length;
|
|
@@ -320,54 +129,90 @@ export class ToolClassifier {
|
|
|
320
129
|
};
|
|
321
130
|
}
|
|
322
131
|
/**
|
|
323
|
-
* Check if text matches any of the provided patterns
|
|
132
|
+
* Check if text matches any of the provided patterns.
|
|
133
|
+
* Limits input length to prevent ReDoS attacks with very long strings.
|
|
134
|
+
*
|
|
135
|
+
* @param text - The text to search in (tool name + description)
|
|
136
|
+
* @param patterns - Pre-compiled regex patterns to match against
|
|
137
|
+
* @returns True if any pattern matches
|
|
324
138
|
*/
|
|
325
139
|
matchesPattern(text, patterns) {
|
|
326
|
-
|
|
140
|
+
// Truncate to prevent ReDoS with pathological inputs
|
|
141
|
+
const safeText = text.length > ToolClassifier.MAX_INPUT_LENGTH
|
|
142
|
+
? text.slice(0, ToolClassifier.MAX_INPUT_LENGTH)
|
|
143
|
+
: text;
|
|
144
|
+
return patterns.some((pattern) => pattern.test(safeText));
|
|
327
145
|
}
|
|
328
146
|
/**
|
|
329
|
-
* Get all tool categories
|
|
147
|
+
* Get all available tool categories.
|
|
148
|
+
*
|
|
149
|
+
* Useful for testing, debugging, or building UI components that need
|
|
150
|
+
* to display all possible categories.
|
|
151
|
+
*
|
|
152
|
+
* @returns Array of all ToolCategory enum values
|
|
153
|
+
*
|
|
154
|
+
* @example
|
|
155
|
+
* ```typescript
|
|
156
|
+
* const allCategories = ToolClassifier.getAllCategories();
|
|
157
|
+
* console.log(allCategories.length); // 17
|
|
158
|
+
* ```
|
|
330
159
|
*/
|
|
331
160
|
static getAllCategories() {
|
|
332
161
|
return Object.values(ToolCategory);
|
|
333
162
|
}
|
|
334
163
|
/**
|
|
335
|
-
* Get risk level for a category
|
|
164
|
+
* Get the security risk level for a category.
|
|
165
|
+
*
|
|
166
|
+
* Risk levels help prioritize security testing:
|
|
167
|
+
* - **HIGH**: Requires thorough security testing (code execution, data access)
|
|
168
|
+
* - **MEDIUM**: Requires moderate security testing (encoding bypass, supply chain)
|
|
169
|
+
* - **LOW**: Safe categories that typically don't need security testing
|
|
170
|
+
*
|
|
171
|
+
* @param category - The category to get the risk level for
|
|
172
|
+
* @returns Risk level: "HIGH", "MEDIUM", or "LOW"
|
|
173
|
+
*
|
|
174
|
+
* @example
|
|
175
|
+
* ```typescript
|
|
176
|
+
* ToolClassifier.getRiskLevel(ToolCategory.SYSTEM_EXEC); // "HIGH"
|
|
177
|
+
* ToolClassifier.getRiskLevel(ToolCategory.JSON_PARSER); // "MEDIUM"
|
|
178
|
+
* ToolClassifier.getRiskLevel(ToolCategory.SAFE_STORAGE); // "LOW"
|
|
179
|
+
* ```
|
|
336
180
|
*/
|
|
337
181
|
static getRiskLevel(category) {
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
ToolCategory.URL_FETCHER,
|
|
346
|
-
];
|
|
347
|
-
const mediumRiskCategories = [
|
|
348
|
-
ToolCategory.UNICODE_PROCESSOR,
|
|
349
|
-
ToolCategory.JSON_PARSER,
|
|
350
|
-
ToolCategory.PACKAGE_INSTALLER,
|
|
351
|
-
ToolCategory.RUG_PULL,
|
|
352
|
-
];
|
|
353
|
-
const lowRiskCategories = [
|
|
354
|
-
ToolCategory.API_WRAPPER,
|
|
355
|
-
ToolCategory.SEARCH_RETRIEVAL,
|
|
356
|
-
ToolCategory.CRUD_CREATION,
|
|
357
|
-
ToolCategory.READ_ONLY_INFO,
|
|
358
|
-
ToolCategory.SAFE_STORAGE,
|
|
359
|
-
ToolCategory.GENERIC,
|
|
360
|
-
];
|
|
361
|
-
if (highRiskCategories.includes(category))
|
|
362
|
-
return "HIGH";
|
|
363
|
-
if (mediumRiskCategories.includes(category))
|
|
364
|
-
return "MEDIUM";
|
|
365
|
-
if (lowRiskCategories.includes(category))
|
|
366
|
-
return "LOW";
|
|
367
|
-
return "LOW";
|
|
182
|
+
if (category === ToolCategory.GENERIC) {
|
|
183
|
+
return GENERIC_CONFIG.risk;
|
|
184
|
+
}
|
|
185
|
+
// Type assertion needed because TypeScript doesn't narrow the type after the GENERIC check
|
|
186
|
+
const config = CATEGORY_PATTERNS[category];
|
|
187
|
+
// Handle unknown categories gracefully (defensive programming)
|
|
188
|
+
return config?.risk ?? "LOW";
|
|
368
189
|
}
|
|
369
190
|
/**
|
|
370
|
-
* Classify multiple tools at once
|
|
191
|
+
* Classify multiple tools at once.
|
|
192
|
+
*
|
|
193
|
+
* More efficient than calling classify() in a loop when you have
|
|
194
|
+
* many tools to process. The classifier is stateless, so batch
|
|
195
|
+
* processing produces identical results to individual calls.
|
|
196
|
+
*
|
|
197
|
+
* @param tools - Array of tools with name and optional description
|
|
198
|
+
* @returns Array of classification results in the same order as input
|
|
199
|
+
*
|
|
200
|
+
* @example
|
|
201
|
+
* ```typescript
|
|
202
|
+
* const classifier = new ToolClassifier();
|
|
203
|
+
* const tools = [
|
|
204
|
+
* { name: 'calculator_tool' },
|
|
205
|
+
* { name: 'search_api', description: 'Search documents' },
|
|
206
|
+
* { name: 'unknown_tool' }
|
|
207
|
+
* ];
|
|
208
|
+
*
|
|
209
|
+
* const results = classifier.classifyBatch(tools);
|
|
210
|
+
* // [
|
|
211
|
+
* // { toolName: 'calculator_tool', categories: ['calculator'], ... },
|
|
212
|
+
* // { toolName: 'search_api', categories: ['search_retrieval'], ... },
|
|
213
|
+
* // { toolName: 'unknown_tool', categories: ['generic'], ... }
|
|
214
|
+
* // ]
|
|
215
|
+
* ```
|
|
371
216
|
*/
|
|
372
217
|
classifyBatch(tools) {
|
|
373
218
|
return tools.map((tool) => this.classify(tool.name, tool.description));
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assessment Orchestrator Helpers
|
|
3
|
+
*
|
|
4
|
+
* Pure functions extracted from AssessmentOrchestrator for testability.
|
|
5
|
+
* These functions handle:
|
|
6
|
+
* - AUP violation enrichment for JSONL events
|
|
7
|
+
* - Module progress/started event emission
|
|
8
|
+
* - Overall status determination
|
|
9
|
+
* - Summary and recommendations generation
|
|
10
|
+
*/
|
|
11
|
+
import { MCPDirectoryAssessment, AssessmentStatus } from "../../lib/assessmentTypes.js";
|
|
12
|
+
export declare const moduleStartTimes: Map<string, number>;
|
|
13
|
+
/**
|
|
14
|
+
* Emit module_started event and track start time for duration calculation.
|
|
15
|
+
* Emits JSONL to stderr with version field for consistent event structure.
|
|
16
|
+
*/
|
|
17
|
+
export declare function emitModuleStartedEvent(moduleName: string, estimatedTests: number, toolCount: number): void;
|
|
18
|
+
/**
|
|
19
|
+
* Emit module_complete event with score and duration.
|
|
20
|
+
* Uses shared score calculator for consistent scoring logic.
|
|
21
|
+
* For AUP module, includes enriched violation data for Claude analysis.
|
|
22
|
+
*/
|
|
23
|
+
export declare function emitModuleProgress(moduleName: string, status: string, result: unknown, testsRun?: number): void;
|
|
24
|
+
/**
|
|
25
|
+
* Build AUP enrichment data from an AUP compliance assessment result.
|
|
26
|
+
* Samples violations prioritizing by severity (CRITICAL > HIGH > MEDIUM).
|
|
27
|
+
*/
|
|
28
|
+
export declare function buildAUPEnrichment(aupResult: {
|
|
29
|
+
violations?: Array<{
|
|
30
|
+
severity: string;
|
|
31
|
+
category: string;
|
|
32
|
+
categoryName?: string;
|
|
33
|
+
matchedText?: string;
|
|
34
|
+
location?: string;
|
|
35
|
+
confidence?: string;
|
|
36
|
+
}>;
|
|
37
|
+
scannedLocations?: {
|
|
38
|
+
toolNames: boolean;
|
|
39
|
+
toolDescriptions: boolean;
|
|
40
|
+
readme: boolean;
|
|
41
|
+
sourceCode: boolean;
|
|
42
|
+
};
|
|
43
|
+
highRiskDomains?: string[];
|
|
44
|
+
}, maxSamples?: number): {
|
|
45
|
+
violationsSample: Array<{
|
|
46
|
+
category: string;
|
|
47
|
+
categoryName: string;
|
|
48
|
+
severity: string;
|
|
49
|
+
matchedText: string;
|
|
50
|
+
location: string;
|
|
51
|
+
confidence: string;
|
|
52
|
+
}>;
|
|
53
|
+
samplingNote: string;
|
|
54
|
+
violationMetrics: {
|
|
55
|
+
total: number;
|
|
56
|
+
critical: number;
|
|
57
|
+
high: number;
|
|
58
|
+
medium: number;
|
|
59
|
+
byCategory: Record<string, number>;
|
|
60
|
+
};
|
|
61
|
+
scannedLocations: {
|
|
62
|
+
toolNames: boolean;
|
|
63
|
+
toolDescriptions: boolean;
|
|
64
|
+
readme: boolean;
|
|
65
|
+
sourceCode: boolean;
|
|
66
|
+
};
|
|
67
|
+
highRiskDomains: string[];
|
|
68
|
+
};
|
|
69
|
+
/**
|
|
70
|
+
* Determine overall status from assessment results.
|
|
71
|
+
* Priority: FAIL > NEED_MORE_INFO > PASS
|
|
72
|
+
*/
|
|
73
|
+
export declare function determineOverallStatus(results: Partial<MCPDirectoryAssessment>): AssessmentStatus;
|
|
74
|
+
/**
|
|
75
|
+
* Generate summary text from assessment results.
|
|
76
|
+
*/
|
|
77
|
+
export declare function generateSummary(results: Partial<MCPDirectoryAssessment>): string;
|
|
78
|
+
/**
|
|
79
|
+
* Generate recommendations from assessment results.
|
|
80
|
+
* Aggregates, deduplicates, and limits to 10 recommendations.
|
|
81
|
+
*/
|
|
82
|
+
export declare function generateRecommendations(results: Partial<MCPDirectoryAssessment>): string[];
|
|
83
|
+
//# sourceMappingURL=orchestratorHelpers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"orchestratorHelpers.d.ts","sourceRoot":"","sources":["../../../src/services/assessment/orchestratorHelpers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,sBAAsB,EACtB,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAU/B,eAAO,MAAM,gBAAgB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAa,CAAC;AAE/D;;;GAGG;AACH,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,GAChB,IAAI,CAcN;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,OAAO,EACf,QAAQ,GAAE,MAAU,GACnB,IAAI,CAiCN;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,SAAS,EAAE;IACT,UAAU,CAAC,EAAE,KAAK,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC,CAAC;IACH,gBAAgB,CAAC,EAAE;QACjB,SAAS,EAAE,OAAO,CAAC;QACnB,gBAAgB,EAAE,OAAO,CAAC;QAC1B,MAAM,EAAE,OAAO,CAAC;QAChB,UAAU,EAAE,OAAO,CAAC;KACrB,CAAC;IACF,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B,EACD,UAAU,GAAE,MAAW,GACtB;IACD,gBAAgB,EAAE,KAAK,CAAC;QACtB,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KACpC,CAAC;IACF,gBAAgB,EAAE;QAChB,SAAS,EAAE,OAAO,CAAC;QACnB,gBAAgB,EAAE,OAAO,CAAC;QAC1B,MAAM,EAAE,OAAO,CAAC;QAChB,UAAU,EAAE,OAAO,CAAC;KACrB,CAAC;IACF,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B,CAkEA;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,OAAO,CAAC,sBAAsB,CAAC,GACvC,gBAAgB,CAsBlB;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,OAAO,CAAC,sBAAsB,CAAC,GACvC,MAAM,CA8ER;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,OAAO,CAAC,sBAAsB,CAAC,GACvC,MAAM,EAAE,CAiBV"}
|