@bryan-thompson/inspector-assessment-client 1.28.0 → 1.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-JnKCxulS.js → OAuthCallback-9Gbb39Ii.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-C2zSlEIQ.js → OAuthDebugCallback-B76J2MBn.js} +1 -1
- package/dist/assets/{index-C3xZdIFQ.js → index-CHTOR9VI.js} +4 -4
- package/dist/index.html +1 -1
- package/lib/lib/assessment/resultTypes.d.ts +8 -0
- package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/lib/lib/securityPatterns.d.ts +1 -1
- package/lib/lib/securityPatterns.js +1 -1
- package/lib/services/assessment/modules/TemporalAssessor.d.ts +5 -129
- package/lib/services/assessment/modules/TemporalAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/TemporalAssessor.js +18 -554
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts +10 -70
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ToolAnnotationAssessor.js +32 -625
- package/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts +65 -0
- package/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/AlignmentChecker.js +289 -0
- package/lib/services/assessment/modules/annotations/ClaudeIntegration.d.ts +22 -0
- package/lib/services/assessment/modules/annotations/ClaudeIntegration.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/ClaudeIntegration.js +139 -0
- package/lib/services/assessment/modules/annotations/EventEmitter.d.ts +20 -0
- package/lib/services/assessment/modules/annotations/EventEmitter.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/EventEmitter.js +100 -0
- package/lib/services/assessment/modules/annotations/ExplanationGenerator.d.ts +25 -0
- package/lib/services/assessment/modules/annotations/ExplanationGenerator.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/ExplanationGenerator.js +122 -0
- package/lib/services/assessment/modules/annotations/index.d.ts +5 -0
- package/lib/services/assessment/modules/annotations/index.d.ts.map +1 -1
- package/lib/services/assessment/modules/annotations/index.js +8 -0
- package/lib/services/assessment/modules/annotations/types.d.ts +33 -0
- package/lib/services/assessment/modules/annotations/types.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/types.js +7 -0
- package/lib/services/assessment/modules/securityTests/SafeResponseDetector.d.ts +3 -0
- package/lib/services/assessment/modules/securityTests/SafeResponseDetector.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/SafeResponseDetector.js +14 -1
- package/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.d.ts +29 -0
- package/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.js +71 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.js +24 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts +66 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts.map +1 -1
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.js +228 -3
- package/lib/services/assessment/modules/temporal/MutationDetector.d.ts +75 -0
- package/lib/services/assessment/modules/temporal/MutationDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/temporal/MutationDetector.js +147 -0
- package/lib/services/assessment/modules/temporal/VarianceClassifier.d.ts +112 -0
- package/lib/services/assessment/modules/temporal/VarianceClassifier.d.ts.map +1 -0
- package/lib/services/assessment/modules/temporal/VarianceClassifier.js +427 -0
- package/lib/services/assessment/modules/temporal/index.d.ts +10 -0
- package/lib/services/assessment/modules/temporal/index.d.ts.map +1 -0
- package/lib/services/assessment/modules/temporal/index.js +9 -0
- package/package.json +1 -1
|
@@ -5,103 +5,25 @@
|
|
|
5
5
|
*
|
|
6
6
|
* This addresses a critical gap: standard assessments call tools with many different
|
|
7
7
|
* payloads but never call the same tool repeatedly with identical payloads.
|
|
8
|
+
*
|
|
9
|
+
* Refactored in Issue #106 to extract MutationDetector and VarianceClassifier
|
|
10
|
+
* into focused helper modules for maintainability.
|
|
8
11
|
*/
|
|
9
12
|
import { BaseAssessor } from "./BaseAssessor.js";
|
|
13
|
+
import { MutationDetector, VarianceClassifier, } from "./temporal/index.js";
|
|
10
14
|
// Security: Maximum response size to prevent memory exhaustion attacks
|
|
11
15
|
const MAX_RESPONSE_SIZE = 1_000_000; // 1MB
|
|
12
16
|
export class TemporalAssessor extends BaseAssessor {
|
|
13
17
|
invocationsPerTool;
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
"create",
|
|
17
|
-
"write",
|
|
18
|
-
"delete",
|
|
19
|
-
"remove",
|
|
20
|
-
"update",
|
|
21
|
-
"insert",
|
|
22
|
-
"post",
|
|
23
|
-
"put",
|
|
24
|
-
"send",
|
|
25
|
-
"submit",
|
|
26
|
-
"execute",
|
|
27
|
-
"run",
|
|
28
|
-
// P2-3: Additional destructive patterns
|
|
29
|
-
"drop",
|
|
30
|
-
"truncate",
|
|
31
|
-
"clear",
|
|
32
|
-
"purge",
|
|
33
|
-
"destroy",
|
|
34
|
-
"reset",
|
|
35
|
-
];
|
|
18
|
+
mutationDetector;
|
|
19
|
+
varianceClassifier;
|
|
36
20
|
// P2-2: Per-invocation timeout to prevent long-running tools from blocking
|
|
37
21
|
PER_INVOCATION_TIMEOUT = 10_000; // 10 seconds
|
|
38
|
-
/**
|
|
39
|
-
* Tool name patterns that are expected to have state-dependent responses.
|
|
40
|
-
* These tools legitimately return different results based on data state,
|
|
41
|
-
* which is NOT a rug pull vulnerability.
|
|
42
|
-
*
|
|
43
|
-
* Includes both:
|
|
44
|
-
* - READ operations: search, list, query return more results after data stored
|
|
45
|
-
* - ACCUMULATION operations: add, append, store return accumulated state (counts, IDs)
|
|
46
|
-
*
|
|
47
|
-
* NOTE: Does NOT include patterns already in DESTRUCTIVE_PATTERNS (create, write,
|
|
48
|
-
* insert, etc.) - those need strict comparison to detect real rug pulls.
|
|
49
|
-
*
|
|
50
|
-
* Uses word-boundary matching to prevent false matches.
|
|
51
|
-
* "add_observations" matches "add" but "address_validator" does not.
|
|
52
|
-
*/
|
|
53
|
-
STATEFUL_TOOL_PATTERNS = [
|
|
54
|
-
// READ operations - results depend on current data state
|
|
55
|
-
"search",
|
|
56
|
-
"list",
|
|
57
|
-
"query",
|
|
58
|
-
"find",
|
|
59
|
-
"get",
|
|
60
|
-
"fetch",
|
|
61
|
-
"read",
|
|
62
|
-
"browse",
|
|
63
|
-
// ACCUMULATION operations (non-destructive) that return accumulated state
|
|
64
|
-
// These legitimately return different counts/IDs as data accumulates
|
|
65
|
-
// NOTE: "add" is NOT in DESTRUCTIVE_PATTERNS, unlike "insert", "create", "write"
|
|
66
|
-
"add",
|
|
67
|
-
"append",
|
|
68
|
-
"store",
|
|
69
|
-
"save",
|
|
70
|
-
"log",
|
|
71
|
-
"record",
|
|
72
|
-
"push",
|
|
73
|
-
"enqueue",
|
|
74
|
-
];
|
|
75
|
-
/**
|
|
76
|
-
* Issue #69: Patterns for resource-creating operations that legitimately return
|
|
77
|
-
* different IDs/resources each invocation.
|
|
78
|
-
*
|
|
79
|
-
* These tools CREATE new resources, so they should use schema comparison + variance
|
|
80
|
-
* classification rather than exact comparison. Unlike STATEFUL_TOOL_PATTERNS, these
|
|
81
|
-
* may overlap with DESTRUCTIVE_PATTERNS (e.g., "create", "insert") but should still
|
|
82
|
-
* use intelligent variance classification to avoid false positives.
|
|
83
|
-
*
|
|
84
|
-
* Examples:
|
|
85
|
-
* - create_billing_product → new product_id each time (LEGITIMATE variance)
|
|
86
|
-
* - generate_report → new report_id each time (LEGITIMATE variance)
|
|
87
|
-
* - insert_record → new record_id each time (LEGITIMATE variance)
|
|
88
|
-
*/
|
|
89
|
-
RESOURCE_CREATING_PATTERNS = [
|
|
90
|
-
"create",
|
|
91
|
-
"new",
|
|
92
|
-
"insert",
|
|
93
|
-
"generate",
|
|
94
|
-
"register",
|
|
95
|
-
"allocate",
|
|
96
|
-
"provision",
|
|
97
|
-
"spawn",
|
|
98
|
-
"instantiate",
|
|
99
|
-
"init",
|
|
100
|
-
"make",
|
|
101
|
-
];
|
|
102
22
|
constructor(config) {
|
|
103
23
|
super(config);
|
|
104
24
|
this.invocationsPerTool = config.temporalInvocations ?? 25;
|
|
25
|
+
this.mutationDetector = new MutationDetector();
|
|
26
|
+
this.varianceClassifier = new VarianceClassifier(this.mutationDetector);
|
|
105
27
|
}
|
|
106
28
|
async assess(context) {
|
|
107
29
|
const results = [];
|
|
@@ -155,7 +77,7 @@ export class TemporalAssessor extends BaseAssessor {
|
|
|
155
77
|
const definitionSnapshots = [];
|
|
156
78
|
const payload = this.generateSafePayload(tool);
|
|
157
79
|
// Reduce invocations for potentially destructive tools
|
|
158
|
-
const isDestructive = this.isDestructiveTool(tool);
|
|
80
|
+
const isDestructive = this.varianceClassifier.isDestructiveTool(tool);
|
|
159
81
|
const invocations = isDestructive
|
|
160
82
|
? Math.min(5, this.invocationsPerTool)
|
|
161
83
|
: this.invocationsPerTool;
|
|
@@ -221,7 +143,7 @@ export class TemporalAssessor extends BaseAssessor {
|
|
|
221
143
|
// Analyze responses for temporal behavior changes
|
|
222
144
|
const result = this.analyzeResponses(tool, responses);
|
|
223
145
|
// Analyze definitions for mutation (rug pull via description change)
|
|
224
|
-
const definitionMutation = this.detectDefinitionMutation(definitionSnapshots);
|
|
146
|
+
const definitionMutation = this.mutationDetector.detectDefinitionMutation(definitionSnapshots);
|
|
225
147
|
return {
|
|
226
148
|
...result,
|
|
227
149
|
reducedInvocations: isDestructive,
|
|
@@ -242,35 +164,6 @@ export class TemporalAssessor extends BaseAssessor {
|
|
|
242
164
|
severity: definitionMutation !== null || result.vulnerable ? "HIGH" : "NONE",
|
|
243
165
|
};
|
|
244
166
|
}
|
|
245
|
-
/**
|
|
246
|
-
* Detect mutations in tool definition across invocation snapshots.
|
|
247
|
-
* DVMCP Challenge 4: Tool descriptions that mutate after N calls.
|
|
248
|
-
*/
|
|
249
|
-
detectDefinitionMutation(snapshots) {
|
|
250
|
-
if (snapshots.length < 2)
|
|
251
|
-
return null;
|
|
252
|
-
const baseline = snapshots[0];
|
|
253
|
-
for (let i = 1; i < snapshots.length; i++) {
|
|
254
|
-
const current = snapshots[i];
|
|
255
|
-
// Check if description changed
|
|
256
|
-
const descriptionChanged = baseline.description !== current.description;
|
|
257
|
-
// Check if schema changed (deep comparison)
|
|
258
|
-
const schemaChanged = JSON.stringify(baseline.inputSchema) !==
|
|
259
|
-
JSON.stringify(current.inputSchema);
|
|
260
|
-
if (descriptionChanged || schemaChanged) {
|
|
261
|
-
return {
|
|
262
|
-
detectedAt: current.invocation,
|
|
263
|
-
baselineDescription: baseline.description,
|
|
264
|
-
mutatedDescription: descriptionChanged
|
|
265
|
-
? current.description
|
|
266
|
-
: undefined,
|
|
267
|
-
baselineSchema: schemaChanged ? baseline.inputSchema : undefined,
|
|
268
|
-
mutatedSchema: schemaChanged ? current.inputSchema : undefined,
|
|
269
|
-
};
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
return null;
|
|
273
|
-
}
|
|
274
167
|
analyzeResponses(tool, responses) {
|
|
275
168
|
if (responses.length === 0) {
|
|
276
169
|
return {
|
|
@@ -284,7 +177,7 @@ export class TemporalAssessor extends BaseAssessor {
|
|
|
284
177
|
severity: "NONE",
|
|
285
178
|
};
|
|
286
179
|
}
|
|
287
|
-
const baseline = this.normalizeResponse(responses[0].response);
|
|
180
|
+
const baseline = this.varianceClassifier.normalizeResponse(responses[0].response);
|
|
288
181
|
const deviations = [];
|
|
289
182
|
const errors = [];
|
|
290
183
|
// Issue #69: Track variance details for transparency
|
|
@@ -293,8 +186,8 @@ export class TemporalAssessor extends BaseAssessor {
|
|
|
293
186
|
// 1. Stateful tools (search, list, etc.) - use schema comparison
|
|
294
187
|
// 2. Resource-creating tools (create, insert, etc.) - use variance classification
|
|
295
188
|
// 3. All other tools - use exact comparison
|
|
296
|
-
const isStateful = this.isStatefulTool(tool);
|
|
297
|
-
const isResourceCreating = this.isResourceCreatingTool(tool);
|
|
189
|
+
const isStateful = this.varianceClassifier.isStatefulTool(tool);
|
|
190
|
+
const isResourceCreating = this.varianceClassifier.isResourceCreatingTool(tool);
|
|
298
191
|
if (isStateful) {
|
|
299
192
|
this.logger.info(`${tool.name} classified as stateful - using schema comparison`);
|
|
300
193
|
}
|
|
@@ -309,11 +202,11 @@ export class TemporalAssessor extends BaseAssessor {
|
|
|
309
202
|
else if (isStateful) {
|
|
310
203
|
// Original stateful tool logic: schema comparison + behavioral content check
|
|
311
204
|
// Content variance is allowed as long as schema is consistent
|
|
312
|
-
let isDifferent = !this.compareSchemas(responses[0].response, responses[i].response);
|
|
205
|
+
let isDifferent = !this.varianceClassifier.compareSchemas(responses[0].response, responses[i].response);
|
|
313
206
|
// Secondary detection: Check for content semantic changes (rug pull patterns)
|
|
314
207
|
// This catches cases where schema is same but content shifts from helpful to harmful
|
|
315
208
|
if (!isDifferent) {
|
|
316
|
-
const contentChange = this.detectStatefulContentChange(responses[0].response, responses[i].response);
|
|
209
|
+
const contentChange = this.mutationDetector.detectStatefulContentChange(responses[0].response, responses[i].response);
|
|
317
210
|
if (contentChange.detected) {
|
|
318
211
|
isDifferent = true;
|
|
319
212
|
this.logger.info(`${tool.name}: Content semantic change detected at invocation ${i + 1} - ${contentChange.reason}`);
|
|
@@ -326,7 +219,7 @@ export class TemporalAssessor extends BaseAssessor {
|
|
|
326
219
|
else if (isResourceCreating) {
|
|
327
220
|
// Issue #69: Use variance classification for resource-creating tools
|
|
328
221
|
// These need intelligent classification to distinguish ID variance from rug pulls
|
|
329
|
-
const classification = this.classifyVariance(
|
|
222
|
+
const classification = this.varianceClassifier.classifyVariance(responses[0].response, responses[i].response);
|
|
330
223
|
varianceDetails.push({
|
|
331
224
|
invocation: i + 1,
|
|
332
225
|
classification,
|
|
@@ -340,7 +233,7 @@ export class TemporalAssessor extends BaseAssessor {
|
|
|
340
233
|
}
|
|
341
234
|
else {
|
|
342
235
|
// Exact comparison for non-stateful, non-resource-creating tools
|
|
343
|
-
const normalized = this.normalizeResponse(responses[i].response);
|
|
236
|
+
const normalized = this.varianceClassifier.normalizeResponse(responses[i].response);
|
|
344
237
|
if (normalized !== baseline) {
|
|
345
238
|
deviations.push(i + 1); // 1-indexed
|
|
346
239
|
}
|
|
@@ -425,435 +318,6 @@ export class TemporalAssessor extends BaseAssessor {
|
|
|
425
318
|
}
|
|
426
319
|
return payload;
|
|
427
320
|
}
|
|
428
|
-
/**
|
|
429
|
-
* Normalize response for comparison by removing naturally varying data.
|
|
430
|
-
* Prevents false positives from timestamps, UUIDs, request IDs, counters, etc.
|
|
431
|
-
* Handles both direct JSON and nested JSON strings (e.g., content[].text).
|
|
432
|
-
*/
|
|
433
|
-
normalizeResponse(response) {
|
|
434
|
-
const str = JSON.stringify(response);
|
|
435
|
-
return (str
|
|
436
|
-
// ISO timestamps (bounded quantifier to prevent ReDoS)
|
|
437
|
-
.replace(/"\d{4}-\d{2}-\d{2}T[\d:.]{1,30}Z?"/g, '"<TIMESTAMP>"')
|
|
438
|
-
// Unix timestamps (13 digits)
|
|
439
|
-
.replace(/"\d{13}"/g, '"<TIMESTAMP>"')
|
|
440
|
-
// UUIDs
|
|
441
|
-
.replace(/"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"/gi, '"<UUID>"')
|
|
442
|
-
// Common ID fields (string values)
|
|
443
|
-
.replace(/"request_id":\s*"[^"]+"/g, '"request_id": "<ID>"')
|
|
444
|
-
.replace(/"requestId":\s*"[^"]+"/g, '"requestId": "<ID>"')
|
|
445
|
-
.replace(/"trace_id":\s*"[^"]+"/g, '"trace_id": "<ID>"')
|
|
446
|
-
// Numeric ID fields (normalize incrementing IDs) - both direct and escaped
|
|
447
|
-
.replace(/"id":\s*\d+/g, '"id": <NUMBER>')
|
|
448
|
-
.replace(/"Id":\s*\d+/g, '"Id": <NUMBER>')
|
|
449
|
-
.replace(/\\"id\\":\s*\d+/g, '\\"id\\": <NUMBER>')
|
|
450
|
-
.replace(/\\"Id\\":\s*\d+/g, '\\"Id\\": <NUMBER>')
|
|
451
|
-
// Counter/sequence fields - both direct and escaped (for nested JSON)
|
|
452
|
-
.replace(/"total_items":\s*\d+/g, '"total_items": <NUMBER>')
|
|
453
|
-
.replace(/\\"total_items\\":\s*\d+/g, '\\"total_items\\": <NUMBER>')
|
|
454
|
-
.replace(/"count":\s*\d+/g, '"count": <NUMBER>')
|
|
455
|
-
.replace(/\\"count\\":\s*\d+/g, '\\"count\\": <NUMBER>')
|
|
456
|
-
.replace(/"invocation_count":\s*\d+/g, '"invocation_count": <NUMBER>')
|
|
457
|
-
.replace(/\\"invocation_count\\":\s*\d+/g, '\\"invocation_count\\": <NUMBER>')
|
|
458
|
-
.replace(/"sequence":\s*\d+/g, '"sequence": <NUMBER>')
|
|
459
|
-
.replace(/\\"sequence\\":\s*\d+/g, '\\"sequence\\": <NUMBER>')
|
|
460
|
-
.replace(/"index":\s*\d+/g, '"index": <NUMBER>')
|
|
461
|
-
.replace(/\\"index\\":\s*\d+/g, '\\"index\\": <NUMBER>')
|
|
462
|
-
// Additional accumulation-related counter fields (defense-in-depth)
|
|
463
|
-
.replace(/"total_observations":\s*\d+/g, '"total_observations": <NUMBER>')
|
|
464
|
-
.replace(/\\"total_observations\\":\s*\d+/g, '\\"total_observations\\": <NUMBER>')
|
|
465
|
-
.replace(/"observations_count":\s*\d+/g, '"observations_count": <NUMBER>')
|
|
466
|
-
.replace(/\\"observations_count\\":\s*\d+/g, '\\"observations_count\\": <NUMBER>')
|
|
467
|
-
.replace(/"total_records":\s*\d+/g, '"total_records": <NUMBER>')
|
|
468
|
-
.replace(/\\"total_records\\":\s*\d+/g, '\\"total_records\\": <NUMBER>')
|
|
469
|
-
.replace(/"records_added":\s*\d+/g, '"records_added": <NUMBER>')
|
|
470
|
-
.replace(/\\"records_added\\":\s*\d+/g, '\\"records_added\\": <NUMBER>')
|
|
471
|
-
.replace(/"items_added":\s*\d+/g, '"items_added": <NUMBER>')
|
|
472
|
-
.replace(/\\"items_added\\":\s*\d+/g, '\\"items_added\\": <NUMBER>')
|
|
473
|
-
.replace(/"size":\s*\d+/g, '"size": <NUMBER>')
|
|
474
|
-
.replace(/\\"size\\":\s*\d+/g, '\\"size\\": <NUMBER>')
|
|
475
|
-
.replace(/"length":\s*\d+/g, '"length": <NUMBER>')
|
|
476
|
-
.replace(/\\"length\\":\s*\d+/g, '\\"length\\": <NUMBER>')
|
|
477
|
-
.replace(/"total":\s*\d+/g, '"total": <NUMBER>')
|
|
478
|
-
.replace(/\\"total\\":\s*\d+/g, '\\"total\\": <NUMBER>')
|
|
479
|
-
// String IDs
|
|
480
|
-
.replace(/"id":\s*"[^"]+"/g, '"id": "<ID>"')
|
|
481
|
-
// P2-1: Additional timestamp fields that vary between calls
|
|
482
|
-
.replace(/"(updated_at|created_at|modified_at)":\s*"[^"]+"/g, '"$1": "<TIMESTAMP>"')
|
|
483
|
-
// P2-1: Dynamic tokens/hashes that change per request
|
|
484
|
-
.replace(/"(nonce|token|hash|etag|session_id|correlation_id)":\s*"[^"]+"/g, '"$1": "<DYNAMIC>"'));
|
|
485
|
-
}
|
|
486
|
-
/**
|
|
487
|
-
* Detect if a tool may have side effects based on naming patterns.
|
|
488
|
-
*/
|
|
489
|
-
isDestructiveTool(tool) {
|
|
490
|
-
const name = tool.name.toLowerCase();
|
|
491
|
-
return this.DESTRUCTIVE_PATTERNS.some((p) => name.includes(p));
|
|
492
|
-
}
|
|
493
|
-
/**
|
|
494
|
-
* Check if a tool is expected to have state-dependent behavior.
|
|
495
|
-
* Stateful tools (search, list, add, store, etc.) legitimately return different
|
|
496
|
-
* results as underlying data changes - this is NOT a rug pull.
|
|
497
|
-
*
|
|
498
|
-
* Uses word-boundary matching to prevent false positives:
|
|
499
|
-
* - "add_observations" matches "add" ✓
|
|
500
|
-
* - "address_validator" does NOT match "add" ✓
|
|
501
|
-
*/
|
|
502
|
-
isStatefulTool(tool) {
|
|
503
|
-
const toolName = tool.name.toLowerCase();
|
|
504
|
-
// Exclude tools that are ALSO destructive - they should get strict exact comparison
|
|
505
|
-
// e.g., "get_and_delete" matches both "get" (stateful) and "delete" (destructive)
|
|
506
|
-
if (this.isDestructiveTool(tool)) {
|
|
507
|
-
return false;
|
|
508
|
-
}
|
|
509
|
-
// Use word-boundary matching: pattern must be at start/end or bounded by _ or -
|
|
510
|
-
// This prevents "address_validator" from matching "add"
|
|
511
|
-
return this.STATEFUL_TOOL_PATTERNS.some((pattern) => {
|
|
512
|
-
const wordBoundaryRegex = new RegExp(`(^|_|-)${pattern}($|_|-)`);
|
|
513
|
-
return wordBoundaryRegex.test(toolName);
|
|
514
|
-
});
|
|
515
|
-
}
|
|
516
|
-
/**
|
|
517
|
-
* Issue #69: Check if a tool creates new resources that legitimately vary per invocation.
|
|
518
|
-
* Resource-creating tools return different IDs, creation timestamps, etc.
|
|
519
|
-
* for each new resource - this is expected behavior, NOT a rug pull.
|
|
520
|
-
*
|
|
521
|
-
* Unlike isStatefulTool(), this DOES include patterns that overlap with DESTRUCTIVE_PATTERNS
|
|
522
|
-
* because resource-creating tools need intelligent variance classification, not exact comparison.
|
|
523
|
-
*
|
|
524
|
-
* Uses word-boundary matching like isStatefulTool() to prevent false matches.
|
|
525
|
-
* - "create_billing_product" matches "create" ✓
|
|
526
|
-
* - "recreate_view" does NOT match "create" ✓ (must be at word boundary)
|
|
527
|
-
*/
|
|
528
|
-
isResourceCreatingTool(tool) {
|
|
529
|
-
const toolName = tool.name.toLowerCase();
|
|
530
|
-
return this.RESOURCE_CREATING_PATTERNS.some((pattern) => {
|
|
531
|
-
const wordBoundaryRegex = new RegExp(`(^|_|-)${pattern}($|_|-)`);
|
|
532
|
-
return wordBoundaryRegex.test(toolName);
|
|
533
|
-
});
|
|
534
|
-
}
|
|
535
|
-
/**
|
|
536
|
-
* Issue #69: Classify variance between two responses to reduce false positives.
|
|
537
|
-
* Returns LEGITIMATE for expected variance (IDs, timestamps), SUSPICIOUS for
|
|
538
|
-
* schema changes, and BEHAVIORAL for semantic changes (promotional keywords, errors).
|
|
539
|
-
*/
|
|
540
|
-
classifyVariance(_tool, baseline, current) {
|
|
541
|
-
// 1. Schema comparison - structural changes are SUSPICIOUS
|
|
542
|
-
const schemaMatch = this.compareSchemas(baseline, current);
|
|
543
|
-
if (!schemaMatch) {
|
|
544
|
-
return {
|
|
545
|
-
type: "SUSPICIOUS",
|
|
546
|
-
confidence: "high",
|
|
547
|
-
reasons: ["Schema/field structure changed between invocations"],
|
|
548
|
-
suspiciousPatterns: ["schema_change"],
|
|
549
|
-
};
|
|
550
|
-
}
|
|
551
|
-
// 2. Content change detection - promotional/error keywords are BEHAVIORAL
|
|
552
|
-
const contentChange = this.detectStatefulContentChange(baseline, current);
|
|
553
|
-
if (contentChange.detected) {
|
|
554
|
-
return {
|
|
555
|
-
type: "BEHAVIORAL",
|
|
556
|
-
confidence: "high",
|
|
557
|
-
reasons: [`Behavioral change detected: ${contentChange.reason}`],
|
|
558
|
-
suspiciousPatterns: [contentChange.reason || "content_change"],
|
|
559
|
-
};
|
|
560
|
-
}
|
|
561
|
-
// 3. After normalization, if responses match = LEGITIMATE
|
|
562
|
-
const normalizedBaseline = this.normalizeResponse(baseline);
|
|
563
|
-
const normalizedCurrent = this.normalizeResponse(current);
|
|
564
|
-
if (normalizedBaseline === normalizedCurrent) {
|
|
565
|
-
return {
|
|
566
|
-
type: "LEGITIMATE",
|
|
567
|
-
confidence: "high",
|
|
568
|
-
reasons: ["All differences normalized (IDs, timestamps, counters)"],
|
|
569
|
-
};
|
|
570
|
-
}
|
|
571
|
-
// 4. Check for legitimate field variance (any _id, _at, token fields)
|
|
572
|
-
const variedFields = this.findVariedFields(baseline, current);
|
|
573
|
-
const unexplainedFields = variedFields.filter((f) => !this.isLegitimateFieldVariance(f));
|
|
574
|
-
if (unexplainedFields.length === 0) {
|
|
575
|
-
return {
|
|
576
|
-
type: "LEGITIMATE",
|
|
577
|
-
confidence: "high",
|
|
578
|
-
reasons: [
|
|
579
|
-
`Variance only in legitimate fields: ${variedFields.join(", ")}`,
|
|
580
|
-
],
|
|
581
|
-
variedFields,
|
|
582
|
-
};
|
|
583
|
-
}
|
|
584
|
-
// 5. Some unexplained variance - flag as suspicious with low confidence
|
|
585
|
-
return {
|
|
586
|
-
type: "SUSPICIOUS",
|
|
587
|
-
confidence: "low",
|
|
588
|
-
reasons: [
|
|
589
|
-
`Unexplained variance in fields: ${unexplainedFields.join(", ")}`,
|
|
590
|
-
],
|
|
591
|
-
variedFields,
|
|
592
|
-
suspiciousPatterns: ["unclassified_variance"],
|
|
593
|
-
};
|
|
594
|
-
}
|
|
595
|
-
/**
|
|
596
|
-
* Issue #69: Check if a field name represents legitimate variance.
|
|
597
|
-
* Fields containing IDs, timestamps, tokens, etc. are expected to vary.
|
|
598
|
-
*/
|
|
599
|
-
isLegitimateFieldVariance(field) {
|
|
600
|
-
const fieldLower = field.toLowerCase();
|
|
601
|
-
// ID fields - any field ending in _id or containing "id" at word boundary
|
|
602
|
-
if (fieldLower.endsWith("_id") || fieldLower.endsWith("id"))
|
|
603
|
-
return true;
|
|
604
|
-
if (fieldLower.includes("_id_") || fieldLower.startsWith("id_"))
|
|
605
|
-
return true;
|
|
606
|
-
// Timestamp fields
|
|
607
|
-
if (fieldLower.endsWith("_at") || fieldLower.endsWith("at"))
|
|
608
|
-
return true;
|
|
609
|
-
if (fieldLower.includes("time") ||
|
|
610
|
-
fieldLower.includes("date") ||
|
|
611
|
-
fieldLower.includes("timestamp"))
|
|
612
|
-
return true;
|
|
613
|
-
// Token/session fields
|
|
614
|
-
if (fieldLower.includes("token") ||
|
|
615
|
-
fieldLower.includes("cursor") ||
|
|
616
|
-
fieldLower.includes("nonce"))
|
|
617
|
-
return true;
|
|
618
|
-
if (fieldLower.includes("session") || fieldLower.includes("correlation"))
|
|
619
|
-
return true;
|
|
620
|
-
// Pagination fields
|
|
621
|
-
if (fieldLower.includes("offset") ||
|
|
622
|
-
fieldLower.includes("page") ||
|
|
623
|
-
fieldLower.includes("next"))
|
|
624
|
-
return true;
|
|
625
|
-
// Counter/accumulation fields
|
|
626
|
-
if (fieldLower.includes("count") ||
|
|
627
|
-
fieldLower.includes("total") ||
|
|
628
|
-
fieldLower.includes("size"))
|
|
629
|
-
return true;
|
|
630
|
-
if (fieldLower.includes("length") || fieldLower.includes("index"))
|
|
631
|
-
return true;
|
|
632
|
-
// Array content fields (search results, items)
|
|
633
|
-
if (fieldLower.includes("results") ||
|
|
634
|
-
fieldLower.includes("items") ||
|
|
635
|
-
fieldLower.includes("data"))
|
|
636
|
-
return true;
|
|
637
|
-
// Hash/version fields
|
|
638
|
-
if (fieldLower.includes("hash") ||
|
|
639
|
-
fieldLower.includes("etag") ||
|
|
640
|
-
fieldLower.includes("version"))
|
|
641
|
-
return true;
|
|
642
|
-
return false;
|
|
643
|
-
}
|
|
644
|
-
/**
|
|
645
|
-
* Issue #69: Find which fields differ between two responses.
|
|
646
|
-
* Returns field paths that have different values.
|
|
647
|
-
*/
|
|
648
|
-
findVariedFields(obj1, obj2, prefix = "") {
|
|
649
|
-
const varied = [];
|
|
650
|
-
// Handle primitives
|
|
651
|
-
if (typeof obj1 !== "object" || obj1 === null) {
|
|
652
|
-
if (obj1 !== obj2) {
|
|
653
|
-
return [prefix || "value"];
|
|
654
|
-
}
|
|
655
|
-
return [];
|
|
656
|
-
}
|
|
657
|
-
if (typeof obj2 !== "object" || obj2 === null) {
|
|
658
|
-
return [prefix || "value"];
|
|
659
|
-
}
|
|
660
|
-
// Handle arrays - just note if length or content differs
|
|
661
|
-
if (Array.isArray(obj1) || Array.isArray(obj2)) {
|
|
662
|
-
const arr1 = Array.isArray(obj1) ? obj1 : [];
|
|
663
|
-
const arr2 = Array.isArray(obj2) ? obj2 : [];
|
|
664
|
-
if (JSON.stringify(arr1) !== JSON.stringify(arr2)) {
|
|
665
|
-
return [prefix || "array"];
|
|
666
|
-
}
|
|
667
|
-
return [];
|
|
668
|
-
}
|
|
669
|
-
// Handle objects
|
|
670
|
-
const allKeys = new Set([
|
|
671
|
-
...Object.keys(obj1),
|
|
672
|
-
...Object.keys(obj2),
|
|
673
|
-
]);
|
|
674
|
-
for (const key of allKeys) {
|
|
675
|
-
const val1 = obj1[key];
|
|
676
|
-
const val2 = obj2[key];
|
|
677
|
-
const fieldPath = prefix ? `${prefix}.${key}` : key;
|
|
678
|
-
if (JSON.stringify(val1) !== JSON.stringify(val2)) {
|
|
679
|
-
// If both are objects, recurse to find specific field
|
|
680
|
-
if (typeof val1 === "object" &&
|
|
681
|
-
val1 !== null &&
|
|
682
|
-
typeof val2 === "object" &&
|
|
683
|
-
val2 !== null) {
|
|
684
|
-
const nestedVaried = this.findVariedFields(val1, val2, fieldPath);
|
|
685
|
-
varied.push(...nestedVaried);
|
|
686
|
-
}
|
|
687
|
-
else {
|
|
688
|
-
varied.push(fieldPath);
|
|
689
|
-
}
|
|
690
|
-
}
|
|
691
|
-
}
|
|
692
|
-
return varied;
|
|
693
|
-
}
|
|
694
|
-
/**
|
|
695
|
-
* Compare response schemas (field names) rather than full content.
|
|
696
|
-
* Stateful tools may have different values but should have consistent fields.
|
|
697
|
-
*
|
|
698
|
-
* For stateful tools, allows schema growth (empty arrays → populated arrays)
|
|
699
|
-
* but flags when baseline fields disappear (suspicious behavior).
|
|
700
|
-
*/
|
|
701
|
-
compareSchemas(response1, response2) {
|
|
702
|
-
const fields1 = this.extractFieldNames(response1).sort();
|
|
703
|
-
const fields2 = this.extractFieldNames(response2).sort();
|
|
704
|
-
// Edge case: empty baseline with populated later response is suspicious
|
|
705
|
-
// An attacker could start with {} then switch to content with malicious fields
|
|
706
|
-
if (fields1.length === 0 && fields2.length > 0) {
|
|
707
|
-
return false; // Flag as schema mismatch
|
|
708
|
-
}
|
|
709
|
-
// Check for exact match (handles non-array cases)
|
|
710
|
-
const exactMatch = fields1.join(",") === fields2.join(",");
|
|
711
|
-
if (exactMatch)
|
|
712
|
-
return true;
|
|
713
|
-
// For stateful tools, allow schema to grow (empty arrays → populated)
|
|
714
|
-
// Baseline (fields1) can be a subset of later responses (fields2)
|
|
715
|
-
// But fields2 cannot have FEWER fields than baseline (that's suspicious)
|
|
716
|
-
const set2 = new Set(fields2);
|
|
717
|
-
const baselineIsSubset = fields1.every((f) => set2.has(f));
|
|
718
|
-
return baselineIsSubset;
|
|
719
|
-
}
|
|
720
|
-
/**
|
|
721
|
-
* Extract all field names from an object recursively.
|
|
722
|
-
* Handles arrays by sampling multiple elements to detect heterogeneous schemas.
|
|
723
|
-
*/
|
|
724
|
-
extractFieldNames(obj, prefix = "") {
|
|
725
|
-
if (obj === null || obj === undefined || typeof obj !== "object")
|
|
726
|
-
return [];
|
|
727
|
-
const fields = [];
|
|
728
|
-
// Handle arrays: sample multiple elements to detect heterogeneous schemas
|
|
729
|
-
// An attacker could hide malicious fields in non-first array elements
|
|
730
|
-
if (Array.isArray(obj)) {
|
|
731
|
-
const samplesToCheck = Math.min(obj.length, 3); // Check up to 3 elements
|
|
732
|
-
const seenFields = new Set();
|
|
733
|
-
for (let i = 0; i < samplesToCheck; i++) {
|
|
734
|
-
if (typeof obj[i] === "object" && obj[i] !== null) {
|
|
735
|
-
const itemFields = this.extractFieldNames(obj[i], `${prefix}[]`);
|
|
736
|
-
itemFields.forEach((f) => seenFields.add(f));
|
|
737
|
-
}
|
|
738
|
-
}
|
|
739
|
-
fields.push(...seenFields);
|
|
740
|
-
return fields;
|
|
741
|
-
}
|
|
742
|
-
// Handle objects
|
|
743
|
-
for (const [key, value] of Object.entries(obj)) {
|
|
744
|
-
const fieldPath = prefix ? `${prefix}.${key}` : key;
|
|
745
|
-
fields.push(fieldPath);
|
|
746
|
-
if (typeof value === "object" && value !== null) {
|
|
747
|
-
fields.push(...this.extractFieldNames(value, fieldPath));
|
|
748
|
-
}
|
|
749
|
-
}
|
|
750
|
-
return fields;
|
|
751
|
-
}
|
|
752
|
-
/**
|
|
753
|
-
* Secondary detection for stateful tools that pass schema comparison.
|
|
754
|
-
* Catches rug pulls that change content semantically while keeping schema intact.
|
|
755
|
-
*
|
|
756
|
-
* Examples detected:
|
|
757
|
-
* - Weather data → "Rate limit exceeded, upgrade to premium"
|
|
758
|
-
* - Stock prices → "Subscribe for $9.99/month to continue"
|
|
759
|
-
* - Search results → "Error: Service unavailable"
|
|
760
|
-
*/
|
|
761
|
-
detectStatefulContentChange(baseline, current) {
|
|
762
|
-
// Convert to strings for content analysis
|
|
763
|
-
const baselineText = this.extractTextContent(baseline);
|
|
764
|
-
const currentText = this.extractTextContent(current);
|
|
765
|
-
// Skip if both are empty or identical
|
|
766
|
-
if (!baselineText && !currentText)
|
|
767
|
-
return { detected: false, reason: null };
|
|
768
|
-
if (baselineText === currentText)
|
|
769
|
-
return { detected: false, reason: null };
|
|
770
|
-
// Check 1: Error keywords appearing in later responses (not present in baseline)
|
|
771
|
-
if (this.hasErrorKeywords(currentText) &&
|
|
772
|
-
!this.hasErrorKeywords(baselineText)) {
|
|
773
|
-
return { detected: true, reason: "error_keywords_appeared" };
|
|
774
|
-
}
|
|
775
|
-
// Check 2: Promotional/payment keywords (rug pull monetization pattern)
|
|
776
|
-
if (this.hasPromotionalKeywords(currentText) &&
|
|
777
|
-
!this.hasPromotionalKeywords(baselineText)) {
|
|
778
|
-
return { detected: true, reason: "promotional_keywords_appeared" };
|
|
779
|
-
}
|
|
780
|
-
// Check 3: Suspicious links injected (URLs not present in baseline)
|
|
781
|
-
if (this.hasSuspiciousLinks(currentText) &&
|
|
782
|
-
!this.hasSuspiciousLinks(baselineText)) {
|
|
783
|
-
return { detected: true, reason: "suspicious_links_injected" };
|
|
784
|
-
}
|
|
785
|
-
// Check 4: Significant length DECREASE only (response becoming much shorter)
|
|
786
|
-
// This catches cases where helpful responses shrink to terse error messages
|
|
787
|
-
// We don't flag length increase because stateful tools legitimately accumulate data
|
|
788
|
-
if (baselineText.length > 20) {
|
|
789
|
-
// Only check if baseline has meaningful content
|
|
790
|
-
const lengthRatio = currentText.length / baselineText.length;
|
|
791
|
-
if (lengthRatio < 0.3) {
|
|
792
|
-
// Response shrunk to <30% of original
|
|
793
|
-
return { detected: true, reason: "significant_length_decrease" };
|
|
794
|
-
}
|
|
795
|
-
}
|
|
796
|
-
return { detected: false, reason: null };
|
|
797
|
-
}
|
|
798
|
-
/**
|
|
799
|
-
* Extract text content from a response for semantic analysis.
|
|
800
|
-
*/
|
|
801
|
-
extractTextContent(obj) {
|
|
802
|
-
if (typeof obj === "string")
|
|
803
|
-
return obj;
|
|
804
|
-
if (typeof obj !== "object" || !obj)
|
|
805
|
-
return "";
|
|
806
|
-
return JSON.stringify(obj);
|
|
807
|
-
}
|
|
808
|
-
/**
|
|
809
|
-
* Check for error-related keywords that indicate service degradation.
|
|
810
|
-
*/
|
|
811
|
-
hasErrorKeywords(text) {
|
|
812
|
-
const patterns = [
|
|
813
|
-
/\berror\b/i,
|
|
814
|
-
/\bfail(ed|ure)?\b/i,
|
|
815
|
-
/\bunavailable\b/i,
|
|
816
|
-
/\brate\s*limit/i,
|
|
817
|
-
/\bdenied\b/i,
|
|
818
|
-
/\bexpired\b/i,
|
|
819
|
-
/\btimeout\b/i,
|
|
820
|
-
/\bblocked\b/i,
|
|
821
|
-
];
|
|
822
|
-
return patterns.some((p) => p.test(text));
|
|
823
|
-
}
|
|
824
|
-
/**
|
|
825
|
-
* Check for promotional/monetization keywords that indicate a monetization rug pull.
|
|
826
|
-
* Enhanced to catch CH4-style rug pulls with limited-time offers, referral codes, etc.
|
|
827
|
-
*
|
|
828
|
-
* Combined into single regex for O(text_length) performance instead of O(18 * text_length).
|
|
829
|
-
*/
|
|
830
|
-
hasPromotionalKeywords(text) {
|
|
831
|
-
// Single combined regex with alternation - matches all 18 original patterns
|
|
832
|
-
// Word-boundary patterns: upgrade, premium, discount, exclusive, subscription variants,
|
|
833
|
-
// multi-word phrases (pro plan, buy now, limited time/offer, free trial, etc.)
|
|
834
|
-
// Non-word patterns: price ($X.XX), percentage (N% off/discount)
|
|
835
|
-
const PROMO_PATTERN = /\b(?:upgrade|premium|discount|exclusive|subscri(?:be|ption)|pro\s*plan|buy\s*now|limited\s*(?:time|offer)|free\s*trial|special\s*offer|referral\s*code|promo\s*code|act\s*now|don't\s*miss|for\s*a\s*fee|pay(?:ment)?\s*(?:required|needed|now))\b|\$\d+(?:\.\d{2})?|\b\d+%\s*(?:off|discount)\b/i;
|
|
836
|
-
return PROMO_PATTERN.test(text);
|
|
837
|
-
}
|
|
838
|
-
/**
|
|
839
|
-
* Check for suspicious URL/link injection that wasn't present initially.
|
|
840
|
-
* Rug pulls often inject links to external malicious or monetization pages.
|
|
841
|
-
*/
|
|
842
|
-
hasSuspiciousLinks(text) {
|
|
843
|
-
const patterns = [
|
|
844
|
-
// HTTP(S) URLs
|
|
845
|
-
/https?:\/\/[^\s]+/i,
|
|
846
|
-
// Markdown links
|
|
847
|
-
/\[.{0,50}?\]\(.{0,200}?\)/,
|
|
848
|
-
// URL shorteners
|
|
849
|
-
/\b(bit\.ly|tinyurl|t\.co|goo\.gl|ow\.ly|buff\.ly)\b/i,
|
|
850
|
-
// Click-bait action patterns
|
|
851
|
-
/\bclick\s*(here|now|this)\b/i,
|
|
852
|
-
/\bvisit\s*our\s*(website|site|page)\b/i,
|
|
853
|
-
/\b(sign\s*up|register)\s*(here|now|at)\b/i,
|
|
854
|
-
];
|
|
855
|
-
return patterns.some((p) => p.test(text));
|
|
856
|
-
}
|
|
857
321
|
determineTemporalStatus(rugPullsDetected, results) {
|
|
858
322
|
if (rugPullsDetected > 0) {
|
|
859
323
|
return "FAIL";
|
|
@@ -913,7 +377,7 @@ export class TemporalAssessor extends BaseAssessor {
|
|
|
913
377
|
const mutated = tool.definitionEvidence?.mutatedDescription
|
|
914
378
|
? `"${tool.definitionEvidence.mutatedDescription.substring(0, 100)}..."`
|
|
915
379
|
: "unknown";
|
|
916
|
-
recommendations.push(`${tool.tool}: Description changed at invocation ${tool.definitionMutationAt}. Baseline: ${baseline}
|
|
380
|
+
recommendations.push(`${tool.tool}: Description changed at invocation ${tool.definitionMutationAt}. Baseline: ${baseline} -> Mutated: ${mutated}`);
|
|
917
381
|
}
|
|
918
382
|
recommendations.push("Review tool source code for global state that mutates __doc__, description, or tool metadata based on call count.");
|
|
919
383
|
}
|