@bryan-thompson/inspector-assessment-client 1.27.0 → 1.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-CJWH8Ytw.js → OAuthCallback-9Gbb39Ii.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-DL5adXJw.js → OAuthDebugCallback-B76J2MBn.js} +1 -1
- package/dist/assets/{index-Cu9XzUwB.js → index-CHTOR9VI.js} +77 -39
- package/dist/index.html +1 -1
- package/lib/lib/assessment/configTypes.d.ts +1 -0
- package/lib/lib/assessment/configTypes.d.ts.map +1 -1
- package/lib/lib/assessment/configTypes.js +10 -0
- package/lib/lib/assessment/extendedTypes.d.ts +74 -0
- package/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
- package/lib/lib/assessment/resultTypes.d.ts +11 -1
- package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/lib/lib/securityPatterns.d.ts +8 -3
- package/lib/lib/securityPatterns.d.ts.map +1 -1
- package/lib/lib/securityPatterns.js +205 -3
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +1 -0
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +31 -1
- package/lib/services/assessment/modules/FileModularizationAssessor.d.ts +87 -0
- package/lib/services/assessment/modules/FileModularizationAssessor.d.ts.map +1 -0
- package/lib/services/assessment/modules/FileModularizationAssessor.js +475 -0
- package/lib/services/assessment/modules/TemporalAssessor.d.ts +5 -129
- package/lib/services/assessment/modules/TemporalAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/TemporalAssessor.js +18 -554
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts +10 -70
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ToolAnnotationAssessor.js +32 -625
- package/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts +65 -0
- package/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/AlignmentChecker.js +289 -0
- package/lib/services/assessment/modules/annotations/ClaudeIntegration.d.ts +22 -0
- package/lib/services/assessment/modules/annotations/ClaudeIntegration.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/ClaudeIntegration.js +139 -0
- package/lib/services/assessment/modules/annotations/EventEmitter.d.ts +20 -0
- package/lib/services/assessment/modules/annotations/EventEmitter.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/EventEmitter.js +100 -0
- package/lib/services/assessment/modules/annotations/ExplanationGenerator.d.ts +25 -0
- package/lib/services/assessment/modules/annotations/ExplanationGenerator.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/ExplanationGenerator.js +122 -0
- package/lib/services/assessment/modules/annotations/index.d.ts +5 -0
- package/lib/services/assessment/modules/annotations/index.d.ts.map +1 -1
- package/lib/services/assessment/modules/annotations/index.js +8 -0
- package/lib/services/assessment/modules/annotations/types.d.ts +33 -0
- package/lib/services/assessment/modules/annotations/types.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/types.js +7 -0
- package/lib/services/assessment/modules/securityTests/SafeResponseDetector.d.ts +3 -0
- package/lib/services/assessment/modules/securityTests/SafeResponseDetector.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/SafeResponseDetector.js +14 -1
- package/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.d.ts +56 -0
- package/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.js +121 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.js +13 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.js +24 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts +80 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.js +273 -3
- package/lib/services/assessment/modules/temporal/MutationDetector.d.ts +75 -0
- package/lib/services/assessment/modules/temporal/MutationDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/temporal/MutationDetector.js +147 -0
- package/lib/services/assessment/modules/temporal/VarianceClassifier.d.ts +112 -0
- package/lib/services/assessment/modules/temporal/VarianceClassifier.d.ts.map +1 -0
- package/lib/services/assessment/modules/temporal/VarianceClassifier.js +427 -0
- package/lib/services/assessment/modules/temporal/index.d.ts +10 -0
- package/lib/services/assessment/modules/temporal/index.d.ts.map +1 -0
- package/lib/services/assessment/modules/temporal/index.js +9 -0
- package/package.json +1 -1
|
@@ -0,0 +1,475 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* File Modularization Assessor (Issue #104)
|
|
3
|
+
* Detects large monolithic tool files and recommends modularization
|
|
4
|
+
*
|
|
5
|
+
* Checks:
|
|
6
|
+
* - Single file >1,000 lines (WARNING, MEDIUM severity)
|
|
7
|
+
* - Single file >2,000 lines (ERROR, HIGH severity)
|
|
8
|
+
* - Tool file with >10 tools (WARNING, MEDIUM severity)
|
|
9
|
+
* - Tool file with >20 tools (ERROR, HIGH severity)
|
|
10
|
+
* - No modular structure (INFO, LOW severity)
|
|
11
|
+
*
|
|
12
|
+
* Scoring:
|
|
13
|
+
* - Starts at 100 points
|
|
14
|
+
* - -15 per file >2,000 lines
|
|
15
|
+
* - -8 per file 1,000-2,000 lines
|
|
16
|
+
* - -12 per file with >20 tools
|
|
17
|
+
* - -6 per file with 10-20 tools
|
|
18
|
+
* - -10 for no modular structure
|
|
19
|
+
* - +5 for tools/ subdirectory
|
|
20
|
+
* - +3 for multiple tool files (>3)
|
|
21
|
+
*/
|
|
22
|
+
import { BaseAssessor } from "./BaseAssessor.js";
|
|
23
|
+
/**
|
|
24
|
+
* Tool detection patterns by language
|
|
25
|
+
*/
|
|
26
|
+
const TOOL_PATTERNS = {
|
|
27
|
+
python: [
|
|
28
|
+
/@mcp\.tool/g, // FastMCP decorator
|
|
29
|
+
/(?<!async\s)def\s+\w+_tool\s*\(/g, // Convention: *_tool functions (not async)
|
|
30
|
+
/async\s+def\s+\w+_tool\s*\(/g, // Async tool functions
|
|
31
|
+
/@server\.tool/g, // MCP server decorator
|
|
32
|
+
/@app\.tool/g, // Alternative app-based decorator
|
|
33
|
+
],
|
|
34
|
+
typescript: [
|
|
35
|
+
/server\.tool\s*\(/g, // MCP SDK tool registration
|
|
36
|
+
/\.setRequestHandler\s*\(/g, // Request handler pattern
|
|
37
|
+
/tools\.push\s*\(/g, // Array-based registration
|
|
38
|
+
/registerTool\s*\(/g, // Common pattern
|
|
39
|
+
/\.addTool\s*\(/g, // Add tool pattern
|
|
40
|
+
],
|
|
41
|
+
javascript: [
|
|
42
|
+
/server\.tool\s*\(/g,
|
|
43
|
+
/\.setRequestHandler\s*\(/g,
|
|
44
|
+
/tools\.push\s*\(/g,
|
|
45
|
+
/registerTool\s*\(/g,
|
|
46
|
+
/\.addTool\s*\(/g,
|
|
47
|
+
],
|
|
48
|
+
go: [
|
|
49
|
+
/func\s+\w*Tool\s*\(/g, // Go tool functions
|
|
50
|
+
/mcp\.NewTool\s*\(/g, // MCP Go SDK
|
|
51
|
+
/tools\.Register\s*\(/g, // Tool registration
|
|
52
|
+
],
|
|
53
|
+
rust: [
|
|
54
|
+
/fn\s+\w+_tool\s*\(/g, // Rust tool functions
|
|
55
|
+
/#\[tool\]/g, // Attribute macro
|
|
56
|
+
/\.register_tool\s*\(/g, // Registration pattern
|
|
57
|
+
],
|
|
58
|
+
};
|
|
59
|
+
/**
|
|
60
|
+
* File extension to language mapping
|
|
61
|
+
*/
|
|
62
|
+
const EXTENSION_TO_LANGUAGE = {
|
|
63
|
+
".py": "python",
|
|
64
|
+
".ts": "typescript",
|
|
65
|
+
".tsx": "typescript",
|
|
66
|
+
".js": "javascript",
|
|
67
|
+
".jsx": "javascript",
|
|
68
|
+
".mjs": "javascript",
|
|
69
|
+
".cjs": "javascript",
|
|
70
|
+
".go": "go",
|
|
71
|
+
".rs": "rust",
|
|
72
|
+
};
|
|
73
|
+
/**
|
|
74
|
+
* Thresholds for modularization checks
|
|
75
|
+
*/
|
|
76
|
+
const THRESHOLDS = {
|
|
77
|
+
LINE_WARNING: 1000,
|
|
78
|
+
LINE_ERROR: 2000,
|
|
79
|
+
TOOL_COUNT_WARNING: 10,
|
|
80
|
+
TOOL_COUNT_ERROR: 20,
|
|
81
|
+
};
|
|
82
|
+
export class FileModularizationAssessor extends BaseAssessor {
|
|
83
|
+
/**
|
|
84
|
+
* Run file modularization assessment
|
|
85
|
+
*/
|
|
86
|
+
async assess(context) {
|
|
87
|
+
this.logger.info("Starting file modularization assessment");
|
|
88
|
+
this.testCount = 0;
|
|
89
|
+
// Check if source code analysis is enabled
|
|
90
|
+
if (!context.sourceCodeFiles || !context.config.enableSourceCodeAnalysis) {
|
|
91
|
+
this.logger.info("Source code analysis not enabled, returning NEED_MORE_INFO");
|
|
92
|
+
return this.createSkippedResult();
|
|
93
|
+
}
|
|
94
|
+
// Analyze each source file
|
|
95
|
+
const fileAnalyses = this.analyzeFiles(context.sourceCodeFiles);
|
|
96
|
+
// Calculate metrics
|
|
97
|
+
const metrics = this.calculateMetrics(fileAnalyses);
|
|
98
|
+
// Run checks against thresholds
|
|
99
|
+
const checks = this.runChecks(metrics, fileAnalyses);
|
|
100
|
+
// Determine status based on checks
|
|
101
|
+
const status = this.determineStatusFromChecks(checks);
|
|
102
|
+
// Generate explanation and recommendations
|
|
103
|
+
const explanation = this.generateExplanation(metrics, checks);
|
|
104
|
+
const recommendations = this.generateRecommendations(metrics, fileAnalyses);
|
|
105
|
+
this.logger.info(`Assessment complete: score=${metrics.modularizationScore}, status=${status}`);
|
|
106
|
+
return {
|
|
107
|
+
metrics,
|
|
108
|
+
checks,
|
|
109
|
+
status,
|
|
110
|
+
explanation,
|
|
111
|
+
recommendations,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Create result when source code is not available
|
|
116
|
+
*/
|
|
117
|
+
createSkippedResult() {
|
|
118
|
+
return {
|
|
119
|
+
metrics: {
|
|
120
|
+
totalSourceFiles: 0,
|
|
121
|
+
totalLines: 0,
|
|
122
|
+
largestFiles: [],
|
|
123
|
+
filesOver1000Lines: 0,
|
|
124
|
+
filesOver2000Lines: 0,
|
|
125
|
+
filesWithOver10Tools: 0,
|
|
126
|
+
filesWithOver20Tools: 0,
|
|
127
|
+
hasModularStructure: false,
|
|
128
|
+
modularizationScore: 0,
|
|
129
|
+
},
|
|
130
|
+
checks: [],
|
|
131
|
+
status: "NEED_MORE_INFO",
|
|
132
|
+
explanation: "Source code analysis not enabled. Enable enableSourceCodeAnalysis in config to run file modularization checks.",
|
|
133
|
+
recommendations: [
|
|
134
|
+
"Enable source code analysis by setting enableSourceCodeAnalysis: true in assessment config",
|
|
135
|
+
"Provide sourceCodePath in assessment context to analyze file structure",
|
|
136
|
+
],
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Analyze all source files
|
|
141
|
+
*/
|
|
142
|
+
analyzeFiles(sourceCodeFiles) {
|
|
143
|
+
const analyses = new Map();
|
|
144
|
+
for (const [filePath, content] of sourceCodeFiles) {
|
|
145
|
+
if (!this.isSourceFile(filePath)) {
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
this.testCount++;
|
|
149
|
+
const lines = content.split("\n").length;
|
|
150
|
+
const language = this.detectLanguage(filePath);
|
|
151
|
+
const toolCount = this.countToolsInFile(content, language);
|
|
152
|
+
analyses.set(filePath, { lines, toolCount, language });
|
|
153
|
+
}
|
|
154
|
+
return analyses;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Check if file is a source file worth scanning
|
|
158
|
+
*/
|
|
159
|
+
isSourceFile(filePath) {
|
|
160
|
+
const sourceExtensions = Object.keys(EXTENSION_TO_LANGUAGE);
|
|
161
|
+
// Skip test files, node_modules, and build artifacts
|
|
162
|
+
// Check for paths containing these directories or starting with them
|
|
163
|
+
if (filePath.includes("node_modules") ||
|
|
164
|
+
filePath.includes(".test.") ||
|
|
165
|
+
filePath.includes(".spec.") ||
|
|
166
|
+
filePath.includes("__tests__") ||
|
|
167
|
+
filePath.includes("__pycache__") ||
|
|
168
|
+
filePath.includes("/dist/") ||
|
|
169
|
+
filePath.includes("/build/") ||
|
|
170
|
+
filePath.includes("/.venv/") ||
|
|
171
|
+
filePath.includes("/venv/") ||
|
|
172
|
+
filePath.startsWith("dist/") ||
|
|
173
|
+
filePath.startsWith("build/") ||
|
|
174
|
+
filePath.startsWith(".venv/") ||
|
|
175
|
+
filePath.startsWith("venv/")) {
|
|
176
|
+
return false;
|
|
177
|
+
}
|
|
178
|
+
return sourceExtensions.some((ext) => filePath.endsWith(ext));
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Detect language from file extension
|
|
182
|
+
*/
|
|
183
|
+
detectLanguage(filePath) {
|
|
184
|
+
for (const [ext, lang] of Object.entries(EXTENSION_TO_LANGUAGE)) {
|
|
185
|
+
if (filePath.endsWith(ext)) {
|
|
186
|
+
return lang;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return null;
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Count tool definitions in a file
|
|
193
|
+
*/
|
|
194
|
+
countToolsInFile(content, language) {
|
|
195
|
+
if (!language)
|
|
196
|
+
return 0;
|
|
197
|
+
const patterns = TOOL_PATTERNS[language] || [];
|
|
198
|
+
let count = 0;
|
|
199
|
+
for (const pattern of patterns) {
|
|
200
|
+
// Reset lastIndex since we're using global flags
|
|
201
|
+
pattern.lastIndex = 0;
|
|
202
|
+
const matches = content.match(pattern);
|
|
203
|
+
if (matches) {
|
|
204
|
+
count += matches.length;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
return count;
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Calculate aggregated metrics
|
|
211
|
+
*/
|
|
212
|
+
calculateMetrics(fileAnalyses) {
|
|
213
|
+
const largestFiles = [];
|
|
214
|
+
let totalLines = 0;
|
|
215
|
+
let filesOver1000Lines = 0;
|
|
216
|
+
let filesOver2000Lines = 0;
|
|
217
|
+
let filesWithOver10Tools = 0;
|
|
218
|
+
let filesWithOver20Tools = 0;
|
|
219
|
+
for (const [filePath, analysis] of fileAnalyses) {
|
|
220
|
+
totalLines += analysis.lines;
|
|
221
|
+
// Check line count thresholds
|
|
222
|
+
if (analysis.lines > THRESHOLDS.LINE_ERROR) {
|
|
223
|
+
filesOver2000Lines++;
|
|
224
|
+
filesOver1000Lines++; // Also counts for 1000+ threshold
|
|
225
|
+
}
|
|
226
|
+
else if (analysis.lines > THRESHOLDS.LINE_WARNING) {
|
|
227
|
+
filesOver1000Lines++;
|
|
228
|
+
}
|
|
229
|
+
// Check tool count thresholds
|
|
230
|
+
if (analysis.toolCount > THRESHOLDS.TOOL_COUNT_ERROR) {
|
|
231
|
+
filesWithOver20Tools++;
|
|
232
|
+
filesWithOver10Tools++; // Also counts for 10+ threshold
|
|
233
|
+
}
|
|
234
|
+
else if (analysis.toolCount > THRESHOLDS.TOOL_COUNT_WARNING) {
|
|
235
|
+
filesWithOver10Tools++;
|
|
236
|
+
}
|
|
237
|
+
// Track large files for reporting
|
|
238
|
+
if (analysis.lines > THRESHOLDS.LINE_WARNING ||
|
|
239
|
+
analysis.toolCount > THRESHOLDS.TOOL_COUNT_WARNING) {
|
|
240
|
+
const severity = this.determineSeverity(analysis.lines, analysis.toolCount);
|
|
241
|
+
const recommendation = this.generateFileRecommendation(filePath, analysis.lines, analysis.toolCount);
|
|
242
|
+
largestFiles.push({
|
|
243
|
+
path: filePath,
|
|
244
|
+
lines: analysis.lines,
|
|
245
|
+
toolCount: analysis.toolCount,
|
|
246
|
+
severity,
|
|
247
|
+
recommendation,
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
// Sort by lines descending
|
|
252
|
+
largestFiles.sort((a, b) => b.lines - a.lines);
|
|
253
|
+
// Check for modular structure
|
|
254
|
+
const hasModularStructure = this.checkModularStructure(fileAnalyses);
|
|
255
|
+
// Calculate modularization score
|
|
256
|
+
const modularizationScore = this.calculateScore(filesOver1000Lines, filesOver2000Lines, filesWithOver10Tools, filesWithOver20Tools, hasModularStructure, fileAnalyses);
|
|
257
|
+
return {
|
|
258
|
+
totalSourceFiles: fileAnalyses.size,
|
|
259
|
+
totalLines,
|
|
260
|
+
largestFiles,
|
|
261
|
+
filesOver1000Lines,
|
|
262
|
+
filesOver2000Lines,
|
|
263
|
+
filesWithOver10Tools,
|
|
264
|
+
filesWithOver20Tools,
|
|
265
|
+
hasModularStructure,
|
|
266
|
+
modularizationScore,
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Determine severity for a file
|
|
271
|
+
*/
|
|
272
|
+
determineSeverity(lines, toolCount) {
|
|
273
|
+
// HIGH if either threshold is exceeded at error level
|
|
274
|
+
if (lines > THRESHOLDS.LINE_ERROR ||
|
|
275
|
+
toolCount > THRESHOLDS.TOOL_COUNT_ERROR) {
|
|
276
|
+
return "HIGH";
|
|
277
|
+
}
|
|
278
|
+
// MEDIUM if warning thresholds are exceeded
|
|
279
|
+
if (lines > THRESHOLDS.LINE_WARNING ||
|
|
280
|
+
toolCount > THRESHOLDS.TOOL_COUNT_WARNING) {
|
|
281
|
+
return "MEDIUM";
|
|
282
|
+
}
|
|
283
|
+
return "LOW";
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Generate recommendation for a specific file
|
|
287
|
+
*/
|
|
288
|
+
generateFileRecommendation(filePath, lines, toolCount) {
|
|
289
|
+
const fileName = filePath.split("/").pop() || filePath;
|
|
290
|
+
const parts = [];
|
|
291
|
+
if (lines > THRESHOLDS.LINE_ERROR) {
|
|
292
|
+
parts.push(`Split ${fileName} (${lines} lines) into smaller modules of <500 lines each`);
|
|
293
|
+
}
|
|
294
|
+
else if (lines > THRESHOLDS.LINE_WARNING) {
|
|
295
|
+
parts.push(`Consider splitting ${fileName} (${lines} lines) to improve maintainability`);
|
|
296
|
+
}
|
|
297
|
+
if (toolCount > THRESHOLDS.TOOL_COUNT_ERROR) {
|
|
298
|
+
parts.push(`Separate ${toolCount} tools into category-based modules (e.g., tools/auth/, tools/data/)`);
|
|
299
|
+
}
|
|
300
|
+
else if (toolCount > THRESHOLDS.TOOL_COUNT_WARNING) {
|
|
301
|
+
parts.push(`Consider grouping ${toolCount} tools into logical categories`);
|
|
302
|
+
}
|
|
303
|
+
return parts.join(". ");
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Check if codebase has modular structure
|
|
307
|
+
*/
|
|
308
|
+
checkModularStructure(fileAnalyses) {
|
|
309
|
+
const filePaths = Array.from(fileAnalyses.keys());
|
|
310
|
+
// Check for tools/ subdirectory pattern
|
|
311
|
+
const hasToolsDir = filePaths.some((f) => f.includes("/tools/") || f.includes("\\tools\\"));
|
|
312
|
+
// Check for multiple tool files (not all tools in one file)
|
|
313
|
+
const toolFiles = Array.from(fileAnalyses.entries()).filter(([, analysis]) => analysis.toolCount > 0);
|
|
314
|
+
// Has modular structure if: has tools/ dir OR has 3+ tool files
|
|
315
|
+
return hasToolsDir || toolFiles.length >= 3;
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Calculate modularization score (0-100)
|
|
319
|
+
*/
|
|
320
|
+
calculateScore(filesOver1000Lines, filesOver2000Lines, filesWithOver10Tools, filesWithOver20Tools, hasModularStructure, fileAnalyses) {
|
|
321
|
+
let score = 100;
|
|
322
|
+
// Deductions
|
|
323
|
+
score -= filesOver2000Lines * 15; // -15 per file >2000 lines
|
|
324
|
+
score -= (filesOver1000Lines - filesOver2000Lines) * 8; // -8 per file 1000-2000 lines
|
|
325
|
+
score -= filesWithOver20Tools * 12; // -12 per file with >20 tools
|
|
326
|
+
score -= (filesWithOver10Tools - filesWithOver20Tools) * 6; // -6 per file 10-20 tools
|
|
327
|
+
if (!hasModularStructure) {
|
|
328
|
+
score -= 10; // -10 for no modular structure
|
|
329
|
+
}
|
|
330
|
+
// Positive signals (bonuses)
|
|
331
|
+
const filePaths = Array.from(fileAnalyses.keys());
|
|
332
|
+
const hasToolsDir = filePaths.some((f) => f.includes("/tools/") || f.includes("\\tools\\"));
|
|
333
|
+
const hasSharedUtils = filePaths.some((f) => f.includes("_common.") ||
|
|
334
|
+
f.includes("shared.") ||
|
|
335
|
+
f.includes("utils.") ||
|
|
336
|
+
f.includes("helpers."));
|
|
337
|
+
const toolFilesCount = Array.from(fileAnalyses.values()).filter((a) => a.toolCount > 0).length;
|
|
338
|
+
if (hasToolsDir)
|
|
339
|
+
score += 5; // +5 for tools/ subdirectory
|
|
340
|
+
if (toolFilesCount > 3)
|
|
341
|
+
score += 3; // +3 for multiple tool files
|
|
342
|
+
if (hasSharedUtils)
|
|
343
|
+
score += 2; // +2 for shared utilities
|
|
344
|
+
// Clamp to 0-100
|
|
345
|
+
return Math.max(0, Math.min(100, score));
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Run threshold checks
|
|
349
|
+
*/
|
|
350
|
+
runChecks(metrics, _fileAnalyses) {
|
|
351
|
+
const checks = [];
|
|
352
|
+
// Check 1: Files over 2000 lines (HIGH severity)
|
|
353
|
+
checks.push({
|
|
354
|
+
checkName: "file_line_count_error",
|
|
355
|
+
passed: metrics.filesOver2000Lines === 0,
|
|
356
|
+
severity: "HIGH",
|
|
357
|
+
evidence: metrics.filesOver2000Lines > 0
|
|
358
|
+
? `${metrics.filesOver2000Lines} file(s) exceed 2000 lines`
|
|
359
|
+
: "No files exceed 2000 lines",
|
|
360
|
+
threshold: THRESHOLDS.LINE_ERROR,
|
|
361
|
+
actualValue: metrics.filesOver2000Lines,
|
|
362
|
+
});
|
|
363
|
+
// Check 2: Files over 1000 lines (MEDIUM severity)
|
|
364
|
+
const filesOnlyOver1000 = metrics.filesOver1000Lines - metrics.filesOver2000Lines;
|
|
365
|
+
checks.push({
|
|
366
|
+
checkName: "file_line_count_warning",
|
|
367
|
+
passed: filesOnlyOver1000 === 0,
|
|
368
|
+
severity: "MEDIUM",
|
|
369
|
+
evidence: filesOnlyOver1000 > 0
|
|
370
|
+
? `${filesOnlyOver1000} file(s) exceed 1000 lines`
|
|
371
|
+
: "No additional files exceed 1000 lines",
|
|
372
|
+
threshold: THRESHOLDS.LINE_WARNING,
|
|
373
|
+
actualValue: filesOnlyOver1000,
|
|
374
|
+
});
|
|
375
|
+
// Check 3: Files with >20 tools (HIGH severity)
|
|
376
|
+
checks.push({
|
|
377
|
+
checkName: "tool_count_error",
|
|
378
|
+
passed: metrics.filesWithOver20Tools === 0,
|
|
379
|
+
severity: "HIGH",
|
|
380
|
+
evidence: metrics.filesWithOver20Tools > 0
|
|
381
|
+
? `${metrics.filesWithOver20Tools} file(s) contain more than 20 tools`
|
|
382
|
+
: "No files contain more than 20 tools",
|
|
383
|
+
threshold: THRESHOLDS.TOOL_COUNT_ERROR,
|
|
384
|
+
actualValue: metrics.filesWithOver20Tools,
|
|
385
|
+
});
|
|
386
|
+
// Check 4: Files with >10 tools (MEDIUM severity)
|
|
387
|
+
const filesOnlyOver10Tools = metrics.filesWithOver10Tools - metrics.filesWithOver20Tools;
|
|
388
|
+
checks.push({
|
|
389
|
+
checkName: "tool_count_warning",
|
|
390
|
+
passed: filesOnlyOver10Tools === 0,
|
|
391
|
+
severity: "MEDIUM",
|
|
392
|
+
evidence: filesOnlyOver10Tools > 0
|
|
393
|
+
? `${filesOnlyOver10Tools} file(s) contain more than 10 tools`
|
|
394
|
+
: "No additional files contain more than 10 tools",
|
|
395
|
+
threshold: THRESHOLDS.TOOL_COUNT_WARNING,
|
|
396
|
+
actualValue: filesOnlyOver10Tools,
|
|
397
|
+
});
|
|
398
|
+
// Check 5: Modular structure (LOW severity, info)
|
|
399
|
+
checks.push({
|
|
400
|
+
checkName: "modular_structure",
|
|
401
|
+
passed: metrics.hasModularStructure,
|
|
402
|
+
severity: "LOW",
|
|
403
|
+
evidence: metrics.hasModularStructure
|
|
404
|
+
? "Codebase has modular structure (tools/ directory or multiple tool files)"
|
|
405
|
+
: "No modular structure detected - all tools appear to be in single file",
|
|
406
|
+
});
|
|
407
|
+
return checks;
|
|
408
|
+
}
|
|
409
|
+
/**
|
|
410
|
+
* Determine status from checks
|
|
411
|
+
*/
|
|
412
|
+
determineStatusFromChecks(checks) {
|
|
413
|
+
// FAIL if any HIGH severity check fails
|
|
414
|
+
const highSeverityFailed = checks.some((c) => !c.passed && c.severity === "HIGH");
|
|
415
|
+
if (highSeverityFailed) {
|
|
416
|
+
return "FAIL";
|
|
417
|
+
}
|
|
418
|
+
// NEED_MORE_INFO if any MEDIUM severity check fails
|
|
419
|
+
const mediumSeverityFailed = checks.some((c) => !c.passed && c.severity === "MEDIUM");
|
|
420
|
+
if (mediumSeverityFailed) {
|
|
421
|
+
return "NEED_MORE_INFO";
|
|
422
|
+
}
|
|
423
|
+
return "PASS";
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* Generate explanation
|
|
427
|
+
*/
|
|
428
|
+
generateExplanation(metrics, checks) {
|
|
429
|
+
const parts = [];
|
|
430
|
+
parts.push(`Analyzed ${metrics.totalSourceFiles} source files (${metrics.totalLines} total lines).`);
|
|
431
|
+
parts.push(`Modularization score: ${metrics.modularizationScore}/100.`);
|
|
432
|
+
const failedChecks = checks.filter((c) => !c.passed);
|
|
433
|
+
if (failedChecks.length === 0) {
|
|
434
|
+
parts.push("All modularization checks passed. Code structure appears well-organized.");
|
|
435
|
+
}
|
|
436
|
+
else {
|
|
437
|
+
const highFailed = failedChecks.filter((c) => c.severity === "HIGH");
|
|
438
|
+
const mediumFailed = failedChecks.filter((c) => c.severity === "MEDIUM");
|
|
439
|
+
if (highFailed.length > 0) {
|
|
440
|
+
parts.push(`ERROR: ${highFailed.length} high-severity issue(s) - files are too large or contain too many tools.`);
|
|
441
|
+
}
|
|
442
|
+
if (mediumFailed.length > 0) {
|
|
443
|
+
parts.push(`WARNING: ${mediumFailed.length} medium-severity issue(s) - consider refactoring for better maintainability.`);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
return parts.join(" ");
|
|
447
|
+
}
|
|
448
|
+
/**
|
|
449
|
+
* Generate recommendations
|
|
450
|
+
*/
|
|
451
|
+
generateRecommendations(metrics, _fileAnalyses) {
|
|
452
|
+
const recommendations = [];
|
|
453
|
+
// Add specific file recommendations
|
|
454
|
+
for (const file of metrics.largestFiles) {
|
|
455
|
+
if (file.severity === "HIGH") {
|
|
456
|
+
recommendations.push(`HIGH: ${file.recommendation}`);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
// Add general recommendations based on checks
|
|
460
|
+
if (metrics.filesOver2000Lines > 0) {
|
|
461
|
+
recommendations.push("Split large files (>2000 lines) into smaller modules to improve maintainability and IDE performance.");
|
|
462
|
+
}
|
|
463
|
+
if (metrics.filesWithOver20Tools > 0) {
|
|
464
|
+
recommendations.push("Group tools by category (e.g., auth tools, data tools, utility tools) into separate modules.");
|
|
465
|
+
}
|
|
466
|
+
if (!metrics.hasModularStructure) {
|
|
467
|
+
recommendations.push("Create a tools/ subdirectory to organize tool implementations by category.");
|
|
468
|
+
recommendations.push("Extract shared utilities into a common module (e.g., _common.py, shared.ts).");
|
|
469
|
+
}
|
|
470
|
+
if (recommendations.length === 0) {
|
|
471
|
+
recommendations.push("Code structure is well-modularized. Continue following current patterns.");
|
|
472
|
+
}
|
|
473
|
+
return recommendations;
|
|
474
|
+
}
|
|
475
|
+
}
|
|
@@ -5,151 +5,27 @@
|
|
|
5
5
|
*
|
|
6
6
|
* This addresses a critical gap: standard assessments call tools with many different
|
|
7
7
|
* payloads but never call the same tool repeatedly with identical payloads.
|
|
8
|
+
*
|
|
9
|
+
* Refactored in Issue #106 to extract MutationDetector and VarianceClassifier
|
|
10
|
+
* into focused helper modules for maintainability.
|
|
8
11
|
*/
|
|
9
12
|
import { AssessmentConfiguration, TemporalAssessment } from "../../../lib/assessmentTypes.js";
|
|
10
13
|
import { AssessmentContext } from "../AssessmentOrchestrator.js";
|
|
11
14
|
import { BaseAssessor } from "./BaseAssessor.js";
|
|
12
15
|
export declare class TemporalAssessor extends BaseAssessor {
|
|
13
16
|
private invocationsPerTool;
|
|
14
|
-
private
|
|
17
|
+
private mutationDetector;
|
|
18
|
+
private varianceClassifier;
|
|
15
19
|
private readonly PER_INVOCATION_TIMEOUT;
|
|
16
|
-
/**
|
|
17
|
-
* Tool name patterns that are expected to have state-dependent responses.
|
|
18
|
-
* These tools legitimately return different results based on data state,
|
|
19
|
-
* which is NOT a rug pull vulnerability.
|
|
20
|
-
*
|
|
21
|
-
* Includes both:
|
|
22
|
-
* - READ operations: search, list, query return more results after data stored
|
|
23
|
-
* - ACCUMULATION operations: add, append, store return accumulated state (counts, IDs)
|
|
24
|
-
*
|
|
25
|
-
* NOTE: Does NOT include patterns already in DESTRUCTIVE_PATTERNS (create, write,
|
|
26
|
-
* insert, etc.) - those need strict comparison to detect real rug pulls.
|
|
27
|
-
*
|
|
28
|
-
* Uses word-boundary matching to prevent false matches.
|
|
29
|
-
* "add_observations" matches "add" but "address_validator" does not.
|
|
30
|
-
*/
|
|
31
|
-
private readonly STATEFUL_TOOL_PATTERNS;
|
|
32
|
-
/**
|
|
33
|
-
* Issue #69: Patterns for resource-creating operations that legitimately return
|
|
34
|
-
* different IDs/resources each invocation.
|
|
35
|
-
*
|
|
36
|
-
* These tools CREATE new resources, so they should use schema comparison + variance
|
|
37
|
-
* classification rather than exact comparison. Unlike STATEFUL_TOOL_PATTERNS, these
|
|
38
|
-
* may overlap with DESTRUCTIVE_PATTERNS (e.g., "create", "insert") but should still
|
|
39
|
-
* use intelligent variance classification to avoid false positives.
|
|
40
|
-
*
|
|
41
|
-
* Examples:
|
|
42
|
-
* - create_billing_product → new product_id each time (LEGITIMATE variance)
|
|
43
|
-
* - generate_report → new report_id each time (LEGITIMATE variance)
|
|
44
|
-
* - insert_record → new record_id each time (LEGITIMATE variance)
|
|
45
|
-
*/
|
|
46
|
-
private readonly RESOURCE_CREATING_PATTERNS;
|
|
47
20
|
constructor(config: AssessmentConfiguration);
|
|
48
21
|
assess(context: AssessmentContext): Promise<TemporalAssessment>;
|
|
49
22
|
private assessTool;
|
|
50
|
-
/**
|
|
51
|
-
* Detect mutations in tool definition across invocation snapshots.
|
|
52
|
-
* DVMCP Challenge 4: Tool descriptions that mutate after N calls.
|
|
53
|
-
*/
|
|
54
|
-
private detectDefinitionMutation;
|
|
55
23
|
private analyzeResponses;
|
|
56
24
|
/**
|
|
57
25
|
* Generate a safe/neutral payload for a tool based on its input schema.
|
|
58
26
|
* Only populates required parameters with minimal test values.
|
|
59
27
|
*/
|
|
60
28
|
private generateSafePayload;
|
|
61
|
-
/**
|
|
62
|
-
* Normalize response for comparison by removing naturally varying data.
|
|
63
|
-
* Prevents false positives from timestamps, UUIDs, request IDs, counters, etc.
|
|
64
|
-
* Handles both direct JSON and nested JSON strings (e.g., content[].text).
|
|
65
|
-
*/
|
|
66
|
-
private normalizeResponse;
|
|
67
|
-
/**
|
|
68
|
-
* Detect if a tool may have side effects based on naming patterns.
|
|
69
|
-
*/
|
|
70
|
-
private isDestructiveTool;
|
|
71
|
-
/**
|
|
72
|
-
* Check if a tool is expected to have state-dependent behavior.
|
|
73
|
-
* Stateful tools (search, list, add, store, etc.) legitimately return different
|
|
74
|
-
* results as underlying data changes - this is NOT a rug pull.
|
|
75
|
-
*
|
|
76
|
-
* Uses word-boundary matching to prevent false positives:
|
|
77
|
-
* - "add_observations" matches "add" ✓
|
|
78
|
-
* - "address_validator" does NOT match "add" ✓
|
|
79
|
-
*/
|
|
80
|
-
private isStatefulTool;
|
|
81
|
-
/**
|
|
82
|
-
* Issue #69: Check if a tool creates new resources that legitimately vary per invocation.
|
|
83
|
-
* Resource-creating tools return different IDs, creation timestamps, etc.
|
|
84
|
-
* for each new resource - this is expected behavior, NOT a rug pull.
|
|
85
|
-
*
|
|
86
|
-
* Unlike isStatefulTool(), this DOES include patterns that overlap with DESTRUCTIVE_PATTERNS
|
|
87
|
-
* because resource-creating tools need intelligent variance classification, not exact comparison.
|
|
88
|
-
*
|
|
89
|
-
* Uses word-boundary matching like isStatefulTool() to prevent false matches.
|
|
90
|
-
* - "create_billing_product" matches "create" ✓
|
|
91
|
-
* - "recreate_view" does NOT match "create" ✓ (must be at word boundary)
|
|
92
|
-
*/
|
|
93
|
-
private isResourceCreatingTool;
|
|
94
|
-
/**
|
|
95
|
-
* Issue #69: Classify variance between two responses to reduce false positives.
|
|
96
|
-
* Returns LEGITIMATE for expected variance (IDs, timestamps), SUSPICIOUS for
|
|
97
|
-
* schema changes, and BEHAVIORAL for semantic changes (promotional keywords, errors).
|
|
98
|
-
*/
|
|
99
|
-
private classifyVariance;
|
|
100
|
-
/**
|
|
101
|
-
* Issue #69: Check if a field name represents legitimate variance.
|
|
102
|
-
* Fields containing IDs, timestamps, tokens, etc. are expected to vary.
|
|
103
|
-
*/
|
|
104
|
-
private isLegitimateFieldVariance;
|
|
105
|
-
/**
|
|
106
|
-
* Issue #69: Find which fields differ between two responses.
|
|
107
|
-
* Returns field paths that have different values.
|
|
108
|
-
*/
|
|
109
|
-
private findVariedFields;
|
|
110
|
-
/**
|
|
111
|
-
* Compare response schemas (field names) rather than full content.
|
|
112
|
-
* Stateful tools may have different values but should have consistent fields.
|
|
113
|
-
*
|
|
114
|
-
* For stateful tools, allows schema growth (empty arrays → populated arrays)
|
|
115
|
-
* but flags when baseline fields disappear (suspicious behavior).
|
|
116
|
-
*/
|
|
117
|
-
private compareSchemas;
|
|
118
|
-
/**
|
|
119
|
-
* Extract all field names from an object recursively.
|
|
120
|
-
* Handles arrays by sampling multiple elements to detect heterogeneous schemas.
|
|
121
|
-
*/
|
|
122
|
-
private extractFieldNames;
|
|
123
|
-
/**
|
|
124
|
-
* Secondary detection for stateful tools that pass schema comparison.
|
|
125
|
-
* Catches rug pulls that change content semantically while keeping schema intact.
|
|
126
|
-
*
|
|
127
|
-
* Examples detected:
|
|
128
|
-
* - Weather data → "Rate limit exceeded, upgrade to premium"
|
|
129
|
-
* - Stock prices → "Subscribe for $9.99/month to continue"
|
|
130
|
-
* - Search results → "Error: Service unavailable"
|
|
131
|
-
*/
|
|
132
|
-
private detectStatefulContentChange;
|
|
133
|
-
/**
|
|
134
|
-
* Extract text content from a response for semantic analysis.
|
|
135
|
-
*/
|
|
136
|
-
private extractTextContent;
|
|
137
|
-
/**
|
|
138
|
-
* Check for error-related keywords that indicate service degradation.
|
|
139
|
-
*/
|
|
140
|
-
private hasErrorKeywords;
|
|
141
|
-
/**
|
|
142
|
-
* Check for promotional/monetization keywords that indicate a monetization rug pull.
|
|
143
|
-
* Enhanced to catch CH4-style rug pulls with limited-time offers, referral codes, etc.
|
|
144
|
-
*
|
|
145
|
-
* Combined into single regex for O(text_length) performance instead of O(18 * text_length).
|
|
146
|
-
*/
|
|
147
|
-
private hasPromotionalKeywords;
|
|
148
|
-
/**
|
|
149
|
-
* Check for suspicious URL/link injection that wasn't present initially.
|
|
150
|
-
* Rug pulls often inject links to external malicious or monetization pages.
|
|
151
|
-
*/
|
|
152
|
-
private hasSuspiciousLinks;
|
|
153
29
|
private determineTemporalStatus;
|
|
154
30
|
private generateExplanation;
|
|
155
31
|
private generateRecommendations;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TemporalAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/TemporalAssessor.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"TemporalAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/TemporalAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EACL,uBAAuB,EAEvB,kBAAkB,EAGnB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAiB9C,qBAAa,gBAAiB,SAAQ,YAAY;IAChD,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAU;gBAErC,MAAM,EAAE,uBAAuB;IAOrC,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;YAqEvD,UAAU;IAuHxB,OAAO,CAAC,gBAAgB;IA6JxB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAsC3B,OAAO,CAAC,uBAAuB;IAa/B,OAAO,CAAC,mBAAmB;IA+C3B,OAAO,CAAC,uBAAuB;CA+DhC"}
|