@bryan-thompson/inspector-assessment-client 1.28.0 → 1.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/dist/assets/{OAuthCallback-JnKCxulS.js → OAuthCallback-BbE88qbF.js} +1 -1
  2. package/dist/assets/{OAuthDebugCallback-C2zSlEIQ.js → OAuthDebugCallback-CfRYq1JG.js} +1 -1
  3. package/dist/assets/{index-C3xZdIFQ.js → index-CsUB73MT.js} +4 -4
  4. package/dist/index.html +1 -1
  5. package/lib/lib/assessment/configTypes.d.ts +6 -0
  6. package/lib/lib/assessment/configTypes.d.ts.map +1 -1
  7. package/lib/lib/assessment/configTypes.js +5 -0
  8. package/lib/lib/assessment/resultTypes.d.ts +8 -0
  9. package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
  10. package/lib/lib/moduleScoring.d.ts +11 -0
  11. package/lib/lib/moduleScoring.d.ts.map +1 -1
  12. package/lib/lib/moduleScoring.js +11 -0
  13. package/lib/lib/securityPatterns.d.ts +1 -1
  14. package/lib/lib/securityPatterns.js +1 -1
  15. package/lib/services/assessment/modules/TemporalAssessor.d.ts +5 -129
  16. package/lib/services/assessment/modules/TemporalAssessor.d.ts.map +1 -1
  17. package/lib/services/assessment/modules/TemporalAssessor.js +18 -554
  18. package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts +10 -70
  19. package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts.map +1 -1
  20. package/lib/services/assessment/modules/ToolAnnotationAssessor.js +32 -625
  21. package/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts +65 -0
  22. package/lib/services/assessment/modules/annotations/AlignmentChecker.d.ts.map +1 -0
  23. package/lib/services/assessment/modules/annotations/AlignmentChecker.js +289 -0
  24. package/lib/services/assessment/modules/annotations/ClaudeIntegration.d.ts +22 -0
  25. package/lib/services/assessment/modules/annotations/ClaudeIntegration.d.ts.map +1 -0
  26. package/lib/services/assessment/modules/annotations/ClaudeIntegration.js +139 -0
  27. package/lib/services/assessment/modules/annotations/EventEmitter.d.ts +20 -0
  28. package/lib/services/assessment/modules/annotations/EventEmitter.d.ts.map +1 -0
  29. package/lib/services/assessment/modules/annotations/EventEmitter.js +100 -0
  30. package/lib/services/assessment/modules/annotations/ExplanationGenerator.d.ts +25 -0
  31. package/lib/services/assessment/modules/annotations/ExplanationGenerator.d.ts.map +1 -0
  32. package/lib/services/assessment/modules/annotations/ExplanationGenerator.js +122 -0
  33. package/lib/services/assessment/modules/annotations/index.d.ts +5 -0
  34. package/lib/services/assessment/modules/annotations/index.d.ts.map +1 -1
  35. package/lib/services/assessment/modules/annotations/index.js +8 -0
  36. package/lib/services/assessment/modules/annotations/types.d.ts +33 -0
  37. package/lib/services/assessment/modules/annotations/types.d.ts.map +1 -0
  38. package/lib/services/assessment/modules/annotations/types.js +7 -0
  39. package/lib/services/assessment/modules/securityTests/SafeResponseDetector.d.ts +3 -0
  40. package/lib/services/assessment/modules/securityTests/SafeResponseDetector.d.ts.map +1 -1
  41. package/lib/services/assessment/modules/securityTests/SafeResponseDetector.js +14 -1
  42. package/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.d.ts +29 -0
  43. package/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.d.ts.map +1 -1
  44. package/lib/services/assessment/modules/securityTests/SecurityPatternLibrary.js +71 -0
  45. package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts.map +1 -1
  46. package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.js +24 -0
  47. package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts +66 -0
  48. package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts.map +1 -1
  49. package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.js +231 -3
  50. package/lib/services/assessment/modules/temporal/MutationDetector.d.ts +75 -0
  51. package/lib/services/assessment/modules/temporal/MutationDetector.d.ts.map +1 -0
  52. package/lib/services/assessment/modules/temporal/MutationDetector.js +147 -0
  53. package/lib/services/assessment/modules/temporal/VarianceClassifier.d.ts +112 -0
  54. package/lib/services/assessment/modules/temporal/VarianceClassifier.d.ts.map +1 -0
  55. package/lib/services/assessment/modules/temporal/VarianceClassifier.js +427 -0
  56. package/lib/services/assessment/modules/temporal/index.d.ts +10 -0
  57. package/lib/services/assessment/modules/temporal/index.d.ts.map +1 -0
  58. package/lib/services/assessment/modules/temporal/index.js +9 -0
  59. package/lib/services/assessment/orchestratorHelpers.d.ts.map +1 -1
  60. package/lib/services/assessment/orchestratorHelpers.js +5 -3
  61. package/package.json +1 -1
@@ -5,103 +5,25 @@
5
5
  *
6
6
  * This addresses a critical gap: standard assessments call tools with many different
7
7
  * payloads but never call the same tool repeatedly with identical payloads.
8
+ *
9
+ * Refactored in Issue #106 to extract MutationDetector and VarianceClassifier
10
+ * into focused helper modules for maintainability.
8
11
  */
9
12
  import { BaseAssessor } from "./BaseAssessor.js";
13
+ import { MutationDetector, VarianceClassifier, } from "./temporal/index.js";
10
14
  // Security: Maximum response size to prevent memory exhaustion attacks
11
15
  const MAX_RESPONSE_SIZE = 1_000_000; // 1MB
12
16
  export class TemporalAssessor extends BaseAssessor {
13
17
  invocationsPerTool;
14
- // Patterns that suggest a tool may have side effects
15
- DESTRUCTIVE_PATTERNS = [
16
- "create",
17
- "write",
18
- "delete",
19
- "remove",
20
- "update",
21
- "insert",
22
- "post",
23
- "put",
24
- "send",
25
- "submit",
26
- "execute",
27
- "run",
28
- // P2-3: Additional destructive patterns
29
- "drop",
30
- "truncate",
31
- "clear",
32
- "purge",
33
- "destroy",
34
- "reset",
35
- ];
18
+ mutationDetector;
19
+ varianceClassifier;
36
20
  // P2-2: Per-invocation timeout to prevent long-running tools from blocking
37
21
  PER_INVOCATION_TIMEOUT = 10_000; // 10 seconds
38
- /**
39
- * Tool name patterns that are expected to have state-dependent responses.
40
- * These tools legitimately return different results based on data state,
41
- * which is NOT a rug pull vulnerability.
42
- *
43
- * Includes both:
44
- * - READ operations: search, list, query return more results after data stored
45
- * - ACCUMULATION operations: add, append, store return accumulated state (counts, IDs)
46
- *
47
- * NOTE: Does NOT include patterns already in DESTRUCTIVE_PATTERNS (create, write,
48
- * insert, etc.) - those need strict comparison to detect real rug pulls.
49
- *
50
- * Uses word-boundary matching to prevent false matches.
51
- * "add_observations" matches "add" but "address_validator" does not.
52
- */
53
- STATEFUL_TOOL_PATTERNS = [
54
- // READ operations - results depend on current data state
55
- "search",
56
- "list",
57
- "query",
58
- "find",
59
- "get",
60
- "fetch",
61
- "read",
62
- "browse",
63
- // ACCUMULATION operations (non-destructive) that return accumulated state
64
- // These legitimately return different counts/IDs as data accumulates
65
- // NOTE: "add" is NOT in DESTRUCTIVE_PATTERNS, unlike "insert", "create", "write"
66
- "add",
67
- "append",
68
- "store",
69
- "save",
70
- "log",
71
- "record",
72
- "push",
73
- "enqueue",
74
- ];
75
- /**
76
- * Issue #69: Patterns for resource-creating operations that legitimately return
77
- * different IDs/resources each invocation.
78
- *
79
- * These tools CREATE new resources, so they should use schema comparison + variance
80
- * classification rather than exact comparison. Unlike STATEFUL_TOOL_PATTERNS, these
81
- * may overlap with DESTRUCTIVE_PATTERNS (e.g., "create", "insert") but should still
82
- * use intelligent variance classification to avoid false positives.
83
- *
84
- * Examples:
85
- * - create_billing_product → new product_id each time (LEGITIMATE variance)
86
- * - generate_report → new report_id each time (LEGITIMATE variance)
87
- * - insert_record → new record_id each time (LEGITIMATE variance)
88
- */
89
- RESOURCE_CREATING_PATTERNS = [
90
- "create",
91
- "new",
92
- "insert",
93
- "generate",
94
- "register",
95
- "allocate",
96
- "provision",
97
- "spawn",
98
- "instantiate",
99
- "init",
100
- "make",
101
- ];
102
22
  constructor(config) {
103
23
  super(config);
104
24
  this.invocationsPerTool = config.temporalInvocations ?? 25;
25
+ this.mutationDetector = new MutationDetector();
26
+ this.varianceClassifier = new VarianceClassifier(this.mutationDetector);
105
27
  }
106
28
  async assess(context) {
107
29
  const results = [];
@@ -155,7 +77,7 @@ export class TemporalAssessor extends BaseAssessor {
155
77
  const definitionSnapshots = [];
156
78
  const payload = this.generateSafePayload(tool);
157
79
  // Reduce invocations for potentially destructive tools
158
- const isDestructive = this.isDestructiveTool(tool);
80
+ const isDestructive = this.varianceClassifier.isDestructiveTool(tool);
159
81
  const invocations = isDestructive
160
82
  ? Math.min(5, this.invocationsPerTool)
161
83
  : this.invocationsPerTool;
@@ -221,7 +143,7 @@ export class TemporalAssessor extends BaseAssessor {
221
143
  // Analyze responses for temporal behavior changes
222
144
  const result = this.analyzeResponses(tool, responses);
223
145
  // Analyze definitions for mutation (rug pull via description change)
224
- const definitionMutation = this.detectDefinitionMutation(definitionSnapshots);
146
+ const definitionMutation = this.mutationDetector.detectDefinitionMutation(definitionSnapshots);
225
147
  return {
226
148
  ...result,
227
149
  reducedInvocations: isDestructive,
@@ -242,35 +164,6 @@ export class TemporalAssessor extends BaseAssessor {
242
164
  severity: definitionMutation !== null || result.vulnerable ? "HIGH" : "NONE",
243
165
  };
244
166
  }
245
- /**
246
- * Detect mutations in tool definition across invocation snapshots.
247
- * DVMCP Challenge 4: Tool descriptions that mutate after N calls.
248
- */
249
- detectDefinitionMutation(snapshots) {
250
- if (snapshots.length < 2)
251
- return null;
252
- const baseline = snapshots[0];
253
- for (let i = 1; i < snapshots.length; i++) {
254
- const current = snapshots[i];
255
- // Check if description changed
256
- const descriptionChanged = baseline.description !== current.description;
257
- // Check if schema changed (deep comparison)
258
- const schemaChanged = JSON.stringify(baseline.inputSchema) !==
259
- JSON.stringify(current.inputSchema);
260
- if (descriptionChanged || schemaChanged) {
261
- return {
262
- detectedAt: current.invocation,
263
- baselineDescription: baseline.description,
264
- mutatedDescription: descriptionChanged
265
- ? current.description
266
- : undefined,
267
- baselineSchema: schemaChanged ? baseline.inputSchema : undefined,
268
- mutatedSchema: schemaChanged ? current.inputSchema : undefined,
269
- };
270
- }
271
- }
272
- return null;
273
- }
274
167
  analyzeResponses(tool, responses) {
275
168
  if (responses.length === 0) {
276
169
  return {
@@ -284,7 +177,7 @@ export class TemporalAssessor extends BaseAssessor {
284
177
  severity: "NONE",
285
178
  };
286
179
  }
287
- const baseline = this.normalizeResponse(responses[0].response);
180
+ const baseline = this.varianceClassifier.normalizeResponse(responses[0].response);
288
181
  const deviations = [];
289
182
  const errors = [];
290
183
  // Issue #69: Track variance details for transparency
@@ -293,8 +186,8 @@ export class TemporalAssessor extends BaseAssessor {
293
186
  // 1. Stateful tools (search, list, etc.) - use schema comparison
294
187
  // 2. Resource-creating tools (create, insert, etc.) - use variance classification
295
188
  // 3. All other tools - use exact comparison
296
- const isStateful = this.isStatefulTool(tool);
297
- const isResourceCreating = this.isResourceCreatingTool(tool);
189
+ const isStateful = this.varianceClassifier.isStatefulTool(tool);
190
+ const isResourceCreating = this.varianceClassifier.isResourceCreatingTool(tool);
298
191
  if (isStateful) {
299
192
  this.logger.info(`${tool.name} classified as stateful - using schema comparison`);
300
193
  }
@@ -309,11 +202,11 @@ export class TemporalAssessor extends BaseAssessor {
309
202
  else if (isStateful) {
310
203
  // Original stateful tool logic: schema comparison + behavioral content check
311
204
  // Content variance is allowed as long as schema is consistent
312
- let isDifferent = !this.compareSchemas(responses[0].response, responses[i].response);
205
+ let isDifferent = !this.varianceClassifier.compareSchemas(responses[0].response, responses[i].response);
313
206
  // Secondary detection: Check for content semantic changes (rug pull patterns)
314
207
  // This catches cases where schema is same but content shifts from helpful to harmful
315
208
  if (!isDifferent) {
316
- const contentChange = this.detectStatefulContentChange(responses[0].response, responses[i].response);
209
+ const contentChange = this.mutationDetector.detectStatefulContentChange(responses[0].response, responses[i].response);
317
210
  if (contentChange.detected) {
318
211
  isDifferent = true;
319
212
  this.logger.info(`${tool.name}: Content semantic change detected at invocation ${i + 1} - ${contentChange.reason}`);
@@ -326,7 +219,7 @@ export class TemporalAssessor extends BaseAssessor {
326
219
  else if (isResourceCreating) {
327
220
  // Issue #69: Use variance classification for resource-creating tools
328
221
  // These need intelligent classification to distinguish ID variance from rug pulls
329
- const classification = this.classifyVariance(tool, responses[0].response, responses[i].response);
222
+ const classification = this.varianceClassifier.classifyVariance(responses[0].response, responses[i].response);
330
223
  varianceDetails.push({
331
224
  invocation: i + 1,
332
225
  classification,
@@ -340,7 +233,7 @@ export class TemporalAssessor extends BaseAssessor {
340
233
  }
341
234
  else {
342
235
  // Exact comparison for non-stateful, non-resource-creating tools
343
- const normalized = this.normalizeResponse(responses[i].response);
236
+ const normalized = this.varianceClassifier.normalizeResponse(responses[i].response);
344
237
  if (normalized !== baseline) {
345
238
  deviations.push(i + 1); // 1-indexed
346
239
  }
@@ -425,435 +318,6 @@ export class TemporalAssessor extends BaseAssessor {
425
318
  }
426
319
  return payload;
427
320
  }
428
- /**
429
- * Normalize response for comparison by removing naturally varying data.
430
- * Prevents false positives from timestamps, UUIDs, request IDs, counters, etc.
431
- * Handles both direct JSON and nested JSON strings (e.g., content[].text).
432
- */
433
- normalizeResponse(response) {
434
- const str = JSON.stringify(response);
435
- return (str
436
- // ISO timestamps (bounded quantifier to prevent ReDoS)
437
- .replace(/"\d{4}-\d{2}-\d{2}T[\d:.]{1,30}Z?"/g, '"<TIMESTAMP>"')
438
- // Unix timestamps (13 digits)
439
- .replace(/"\d{13}"/g, '"<TIMESTAMP>"')
440
- // UUIDs
441
- .replace(/"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"/gi, '"<UUID>"')
442
- // Common ID fields (string values)
443
- .replace(/"request_id":\s*"[^"]+"/g, '"request_id": "<ID>"')
444
- .replace(/"requestId":\s*"[^"]+"/g, '"requestId": "<ID>"')
445
- .replace(/"trace_id":\s*"[^"]+"/g, '"trace_id": "<ID>"')
446
- // Numeric ID fields (normalize incrementing IDs) - both direct and escaped
447
- .replace(/"id":\s*\d+/g, '"id": <NUMBER>')
448
- .replace(/"Id":\s*\d+/g, '"Id": <NUMBER>')
449
- .replace(/\\"id\\":\s*\d+/g, '\\"id\\": <NUMBER>')
450
- .replace(/\\"Id\\":\s*\d+/g, '\\"Id\\": <NUMBER>')
451
- // Counter/sequence fields - both direct and escaped (for nested JSON)
452
- .replace(/"total_items":\s*\d+/g, '"total_items": <NUMBER>')
453
- .replace(/\\"total_items\\":\s*\d+/g, '\\"total_items\\": <NUMBER>')
454
- .replace(/"count":\s*\d+/g, '"count": <NUMBER>')
455
- .replace(/\\"count\\":\s*\d+/g, '\\"count\\": <NUMBER>')
456
- .replace(/"invocation_count":\s*\d+/g, '"invocation_count": <NUMBER>')
457
- .replace(/\\"invocation_count\\":\s*\d+/g, '\\"invocation_count\\": <NUMBER>')
458
- .replace(/"sequence":\s*\d+/g, '"sequence": <NUMBER>')
459
- .replace(/\\"sequence\\":\s*\d+/g, '\\"sequence\\": <NUMBER>')
460
- .replace(/"index":\s*\d+/g, '"index": <NUMBER>')
461
- .replace(/\\"index\\":\s*\d+/g, '\\"index\\": <NUMBER>')
462
- // Additional accumulation-related counter fields (defense-in-depth)
463
- .replace(/"total_observations":\s*\d+/g, '"total_observations": <NUMBER>')
464
- .replace(/\\"total_observations\\":\s*\d+/g, '\\"total_observations\\": <NUMBER>')
465
- .replace(/"observations_count":\s*\d+/g, '"observations_count": <NUMBER>')
466
- .replace(/\\"observations_count\\":\s*\d+/g, '\\"observations_count\\": <NUMBER>')
467
- .replace(/"total_records":\s*\d+/g, '"total_records": <NUMBER>')
468
- .replace(/\\"total_records\\":\s*\d+/g, '\\"total_records\\": <NUMBER>')
469
- .replace(/"records_added":\s*\d+/g, '"records_added": <NUMBER>')
470
- .replace(/\\"records_added\\":\s*\d+/g, '\\"records_added\\": <NUMBER>')
471
- .replace(/"items_added":\s*\d+/g, '"items_added": <NUMBER>')
472
- .replace(/\\"items_added\\":\s*\d+/g, '\\"items_added\\": <NUMBER>')
473
- .replace(/"size":\s*\d+/g, '"size": <NUMBER>')
474
- .replace(/\\"size\\":\s*\d+/g, '\\"size\\": <NUMBER>')
475
- .replace(/"length":\s*\d+/g, '"length": <NUMBER>')
476
- .replace(/\\"length\\":\s*\d+/g, '\\"length\\": <NUMBER>')
477
- .replace(/"total":\s*\d+/g, '"total": <NUMBER>')
478
- .replace(/\\"total\\":\s*\d+/g, '\\"total\\": <NUMBER>')
479
- // String IDs
480
- .replace(/"id":\s*"[^"]+"/g, '"id": "<ID>"')
481
- // P2-1: Additional timestamp fields that vary between calls
482
- .replace(/"(updated_at|created_at|modified_at)":\s*"[^"]+"/g, '"$1": "<TIMESTAMP>"')
483
- // P2-1: Dynamic tokens/hashes that change per request
484
- .replace(/"(nonce|token|hash|etag|session_id|correlation_id)":\s*"[^"]+"/g, '"$1": "<DYNAMIC>"'));
485
- }
486
- /**
487
- * Detect if a tool may have side effects based on naming patterns.
488
- */
489
- isDestructiveTool(tool) {
490
- const name = tool.name.toLowerCase();
491
- return this.DESTRUCTIVE_PATTERNS.some((p) => name.includes(p));
492
- }
493
- /**
494
- * Check if a tool is expected to have state-dependent behavior.
495
- * Stateful tools (search, list, add, store, etc.) legitimately return different
496
- * results as underlying data changes - this is NOT a rug pull.
497
- *
498
- * Uses word-boundary matching to prevent false positives:
499
- * - "add_observations" matches "add" ✓
500
- * - "address_validator" does NOT match "add" ✓
501
- */
502
- isStatefulTool(tool) {
503
- const toolName = tool.name.toLowerCase();
504
- // Exclude tools that are ALSO destructive - they should get strict exact comparison
505
- // e.g., "get_and_delete" matches both "get" (stateful) and "delete" (destructive)
506
- if (this.isDestructiveTool(tool)) {
507
- return false;
508
- }
509
- // Use word-boundary matching: pattern must be at start/end or bounded by _ or -
510
- // This prevents "address_validator" from matching "add"
511
- return this.STATEFUL_TOOL_PATTERNS.some((pattern) => {
512
- const wordBoundaryRegex = new RegExp(`(^|_|-)${pattern}($|_|-)`);
513
- return wordBoundaryRegex.test(toolName);
514
- });
515
- }
516
- /**
517
- * Issue #69: Check if a tool creates new resources that legitimately vary per invocation.
518
- * Resource-creating tools return different IDs, creation timestamps, etc.
519
- * for each new resource - this is expected behavior, NOT a rug pull.
520
- *
521
- * Unlike isStatefulTool(), this DOES include patterns that overlap with DESTRUCTIVE_PATTERNS
522
- * because resource-creating tools need intelligent variance classification, not exact comparison.
523
- *
524
- * Uses word-boundary matching like isStatefulTool() to prevent false matches.
525
- * - "create_billing_product" matches "create" ✓
526
- * - "recreate_view" does NOT match "create" ✓ (must be at word boundary)
527
- */
528
- isResourceCreatingTool(tool) {
529
- const toolName = tool.name.toLowerCase();
530
- return this.RESOURCE_CREATING_PATTERNS.some((pattern) => {
531
- const wordBoundaryRegex = new RegExp(`(^|_|-)${pattern}($|_|-)`);
532
- return wordBoundaryRegex.test(toolName);
533
- });
534
- }
535
- /**
536
- * Issue #69: Classify variance between two responses to reduce false positives.
537
- * Returns LEGITIMATE for expected variance (IDs, timestamps), SUSPICIOUS for
538
- * schema changes, and BEHAVIORAL for semantic changes (promotional keywords, errors).
539
- */
540
- classifyVariance(_tool, baseline, current) {
541
- // 1. Schema comparison - structural changes are SUSPICIOUS
542
- const schemaMatch = this.compareSchemas(baseline, current);
543
- if (!schemaMatch) {
544
- return {
545
- type: "SUSPICIOUS",
546
- confidence: "high",
547
- reasons: ["Schema/field structure changed between invocations"],
548
- suspiciousPatterns: ["schema_change"],
549
- };
550
- }
551
- // 2. Content change detection - promotional/error keywords are BEHAVIORAL
552
- const contentChange = this.detectStatefulContentChange(baseline, current);
553
- if (contentChange.detected) {
554
- return {
555
- type: "BEHAVIORAL",
556
- confidence: "high",
557
- reasons: [`Behavioral change detected: ${contentChange.reason}`],
558
- suspiciousPatterns: [contentChange.reason || "content_change"],
559
- };
560
- }
561
- // 3. After normalization, if responses match = LEGITIMATE
562
- const normalizedBaseline = this.normalizeResponse(baseline);
563
- const normalizedCurrent = this.normalizeResponse(current);
564
- if (normalizedBaseline === normalizedCurrent) {
565
- return {
566
- type: "LEGITIMATE",
567
- confidence: "high",
568
- reasons: ["All differences normalized (IDs, timestamps, counters)"],
569
- };
570
- }
571
- // 4. Check for legitimate field variance (any _id, _at, token fields)
572
- const variedFields = this.findVariedFields(baseline, current);
573
- const unexplainedFields = variedFields.filter((f) => !this.isLegitimateFieldVariance(f));
574
- if (unexplainedFields.length === 0) {
575
- return {
576
- type: "LEGITIMATE",
577
- confidence: "high",
578
- reasons: [
579
- `Variance only in legitimate fields: ${variedFields.join(", ")}`,
580
- ],
581
- variedFields,
582
- };
583
- }
584
- // 5. Some unexplained variance - flag as suspicious with low confidence
585
- return {
586
- type: "SUSPICIOUS",
587
- confidence: "low",
588
- reasons: [
589
- `Unexplained variance in fields: ${unexplainedFields.join(", ")}`,
590
- ],
591
- variedFields,
592
- suspiciousPatterns: ["unclassified_variance"],
593
- };
594
- }
595
- /**
596
- * Issue #69: Check if a field name represents legitimate variance.
597
- * Fields containing IDs, timestamps, tokens, etc. are expected to vary.
598
- */
599
- isLegitimateFieldVariance(field) {
600
- const fieldLower = field.toLowerCase();
601
- // ID fields - any field ending in _id or containing "id" at word boundary
602
- if (fieldLower.endsWith("_id") || fieldLower.endsWith("id"))
603
- return true;
604
- if (fieldLower.includes("_id_") || fieldLower.startsWith("id_"))
605
- return true;
606
- // Timestamp fields
607
- if (fieldLower.endsWith("_at") || fieldLower.endsWith("at"))
608
- return true;
609
- if (fieldLower.includes("time") ||
610
- fieldLower.includes("date") ||
611
- fieldLower.includes("timestamp"))
612
- return true;
613
- // Token/session fields
614
- if (fieldLower.includes("token") ||
615
- fieldLower.includes("cursor") ||
616
- fieldLower.includes("nonce"))
617
- return true;
618
- if (fieldLower.includes("session") || fieldLower.includes("correlation"))
619
- return true;
620
- // Pagination fields
621
- if (fieldLower.includes("offset") ||
622
- fieldLower.includes("page") ||
623
- fieldLower.includes("next"))
624
- return true;
625
- // Counter/accumulation fields
626
- if (fieldLower.includes("count") ||
627
- fieldLower.includes("total") ||
628
- fieldLower.includes("size"))
629
- return true;
630
- if (fieldLower.includes("length") || fieldLower.includes("index"))
631
- return true;
632
- // Array content fields (search results, items)
633
- if (fieldLower.includes("results") ||
634
- fieldLower.includes("items") ||
635
- fieldLower.includes("data"))
636
- return true;
637
- // Hash/version fields
638
- if (fieldLower.includes("hash") ||
639
- fieldLower.includes("etag") ||
640
- fieldLower.includes("version"))
641
- return true;
642
- return false;
643
- }
644
- /**
645
- * Issue #69: Find which fields differ between two responses.
646
- * Returns field paths that have different values.
647
- */
648
- findVariedFields(obj1, obj2, prefix = "") {
649
- const varied = [];
650
- // Handle primitives
651
- if (typeof obj1 !== "object" || obj1 === null) {
652
- if (obj1 !== obj2) {
653
- return [prefix || "value"];
654
- }
655
- return [];
656
- }
657
- if (typeof obj2 !== "object" || obj2 === null) {
658
- return [prefix || "value"];
659
- }
660
- // Handle arrays - just note if length or content differs
661
- if (Array.isArray(obj1) || Array.isArray(obj2)) {
662
- const arr1 = Array.isArray(obj1) ? obj1 : [];
663
- const arr2 = Array.isArray(obj2) ? obj2 : [];
664
- if (JSON.stringify(arr1) !== JSON.stringify(arr2)) {
665
- return [prefix || "array"];
666
- }
667
- return [];
668
- }
669
- // Handle objects
670
- const allKeys = new Set([
671
- ...Object.keys(obj1),
672
- ...Object.keys(obj2),
673
- ]);
674
- for (const key of allKeys) {
675
- const val1 = obj1[key];
676
- const val2 = obj2[key];
677
- const fieldPath = prefix ? `${prefix}.${key}` : key;
678
- if (JSON.stringify(val1) !== JSON.stringify(val2)) {
679
- // If both are objects, recurse to find specific field
680
- if (typeof val1 === "object" &&
681
- val1 !== null &&
682
- typeof val2 === "object" &&
683
- val2 !== null) {
684
- const nestedVaried = this.findVariedFields(val1, val2, fieldPath);
685
- varied.push(...nestedVaried);
686
- }
687
- else {
688
- varied.push(fieldPath);
689
- }
690
- }
691
- }
692
- return varied;
693
- }
694
- /**
695
- * Compare response schemas (field names) rather than full content.
696
- * Stateful tools may have different values but should have consistent fields.
697
- *
698
- * For stateful tools, allows schema growth (empty arrays → populated arrays)
699
- * but flags when baseline fields disappear (suspicious behavior).
700
- */
701
- compareSchemas(response1, response2) {
702
- const fields1 = this.extractFieldNames(response1).sort();
703
- const fields2 = this.extractFieldNames(response2).sort();
704
- // Edge case: empty baseline with populated later response is suspicious
705
- // An attacker could start with {} then switch to content with malicious fields
706
- if (fields1.length === 0 && fields2.length > 0) {
707
- return false; // Flag as schema mismatch
708
- }
709
- // Check for exact match (handles non-array cases)
710
- const exactMatch = fields1.join(",") === fields2.join(",");
711
- if (exactMatch)
712
- return true;
713
- // For stateful tools, allow schema to grow (empty arrays → populated)
714
- // Baseline (fields1) can be a subset of later responses (fields2)
715
- // But fields2 cannot have FEWER fields than baseline (that's suspicious)
716
- const set2 = new Set(fields2);
717
- const baselineIsSubset = fields1.every((f) => set2.has(f));
718
- return baselineIsSubset;
719
- }
720
- /**
721
- * Extract all field names from an object recursively.
722
- * Handles arrays by sampling multiple elements to detect heterogeneous schemas.
723
- */
724
- extractFieldNames(obj, prefix = "") {
725
- if (obj === null || obj === undefined || typeof obj !== "object")
726
- return [];
727
- const fields = [];
728
- // Handle arrays: sample multiple elements to detect heterogeneous schemas
729
- // An attacker could hide malicious fields in non-first array elements
730
- if (Array.isArray(obj)) {
731
- const samplesToCheck = Math.min(obj.length, 3); // Check up to 3 elements
732
- const seenFields = new Set();
733
- for (let i = 0; i < samplesToCheck; i++) {
734
- if (typeof obj[i] === "object" && obj[i] !== null) {
735
- const itemFields = this.extractFieldNames(obj[i], `${prefix}[]`);
736
- itemFields.forEach((f) => seenFields.add(f));
737
- }
738
- }
739
- fields.push(...seenFields);
740
- return fields;
741
- }
742
- // Handle objects
743
- for (const [key, value] of Object.entries(obj)) {
744
- const fieldPath = prefix ? `${prefix}.${key}` : key;
745
- fields.push(fieldPath);
746
- if (typeof value === "object" && value !== null) {
747
- fields.push(...this.extractFieldNames(value, fieldPath));
748
- }
749
- }
750
- return fields;
751
- }
752
- /**
753
- * Secondary detection for stateful tools that pass schema comparison.
754
- * Catches rug pulls that change content semantically while keeping schema intact.
755
- *
756
- * Examples detected:
757
- * - Weather data → "Rate limit exceeded, upgrade to premium"
758
- * - Stock prices → "Subscribe for $9.99/month to continue"
759
- * - Search results → "Error: Service unavailable"
760
- */
761
- detectStatefulContentChange(baseline, current) {
762
- // Convert to strings for content analysis
763
- const baselineText = this.extractTextContent(baseline);
764
- const currentText = this.extractTextContent(current);
765
- // Skip if both are empty or identical
766
- if (!baselineText && !currentText)
767
- return { detected: false, reason: null };
768
- if (baselineText === currentText)
769
- return { detected: false, reason: null };
770
- // Check 1: Error keywords appearing in later responses (not present in baseline)
771
- if (this.hasErrorKeywords(currentText) &&
772
- !this.hasErrorKeywords(baselineText)) {
773
- return { detected: true, reason: "error_keywords_appeared" };
774
- }
775
- // Check 2: Promotional/payment keywords (rug pull monetization pattern)
776
- if (this.hasPromotionalKeywords(currentText) &&
777
- !this.hasPromotionalKeywords(baselineText)) {
778
- return { detected: true, reason: "promotional_keywords_appeared" };
779
- }
780
- // Check 3: Suspicious links injected (URLs not present in baseline)
781
- if (this.hasSuspiciousLinks(currentText) &&
782
- !this.hasSuspiciousLinks(baselineText)) {
783
- return { detected: true, reason: "suspicious_links_injected" };
784
- }
785
- // Check 4: Significant length DECREASE only (response becoming much shorter)
786
- // This catches cases where helpful responses shrink to terse error messages
787
- // We don't flag length increase because stateful tools legitimately accumulate data
788
- if (baselineText.length > 20) {
789
- // Only check if baseline has meaningful content
790
- const lengthRatio = currentText.length / baselineText.length;
791
- if (lengthRatio < 0.3) {
792
- // Response shrunk to <30% of original
793
- return { detected: true, reason: "significant_length_decrease" };
794
- }
795
- }
796
- return { detected: false, reason: null };
797
- }
798
- /**
799
- * Extract text content from a response for semantic analysis.
800
- */
801
- extractTextContent(obj) {
802
- if (typeof obj === "string")
803
- return obj;
804
- if (typeof obj !== "object" || !obj)
805
- return "";
806
- return JSON.stringify(obj);
807
- }
808
- /**
809
- * Check for error-related keywords that indicate service degradation.
810
- */
811
- hasErrorKeywords(text) {
812
- const patterns = [
813
- /\berror\b/i,
814
- /\bfail(ed|ure)?\b/i,
815
- /\bunavailable\b/i,
816
- /\brate\s*limit/i,
817
- /\bdenied\b/i,
818
- /\bexpired\b/i,
819
- /\btimeout\b/i,
820
- /\bblocked\b/i,
821
- ];
822
- return patterns.some((p) => p.test(text));
823
- }
824
- /**
825
- * Check for promotional/monetization keywords that indicate a monetization rug pull.
826
- * Enhanced to catch CH4-style rug pulls with limited-time offers, referral codes, etc.
827
- *
828
- * Combined into single regex for O(text_length) performance instead of O(18 * text_length).
829
- */
830
- hasPromotionalKeywords(text) {
831
- // Single combined regex with alternation - matches all 18 original patterns
832
- // Word-boundary patterns: upgrade, premium, discount, exclusive, subscription variants,
833
- // multi-word phrases (pro plan, buy now, limited time/offer, free trial, etc.)
834
- // Non-word patterns: price ($X.XX), percentage (N% off/discount)
835
- const PROMO_PATTERN = /\b(?:upgrade|premium|discount|exclusive|subscri(?:be|ption)|pro\s*plan|buy\s*now|limited\s*(?:time|offer)|free\s*trial|special\s*offer|referral\s*code|promo\s*code|act\s*now|don't\s*miss|for\s*a\s*fee|pay(?:ment)?\s*(?:required|needed|now))\b|\$\d+(?:\.\d{2})?|\b\d+%\s*(?:off|discount)\b/i;
836
- return PROMO_PATTERN.test(text);
837
- }
838
- /**
839
- * Check for suspicious URL/link injection that wasn't present initially.
840
- * Rug pulls often inject links to external malicious or monetization pages.
841
- */
842
- hasSuspiciousLinks(text) {
843
- const patterns = [
844
- // HTTP(S) URLs
845
- /https?:\/\/[^\s]+/i,
846
- // Markdown links
847
- /\[.{0,50}?\]\(.{0,200}?\)/,
848
- // URL shorteners
849
- /\b(bit\.ly|tinyurl|t\.co|goo\.gl|ow\.ly|buff\.ly)\b/i,
850
- // Click-bait action patterns
851
- /\bclick\s*(here|now|this)\b/i,
852
- /\bvisit\s*our\s*(website|site|page)\b/i,
853
- /\b(sign\s*up|register)\s*(here|now|at)\b/i,
854
- ];
855
- return patterns.some((p) => p.test(text));
856
- }
857
321
  determineTemporalStatus(rugPullsDetected, results) {
858
322
  if (rugPullsDetected > 0) {
859
323
  return "FAIL";
@@ -913,7 +377,7 @@ export class TemporalAssessor extends BaseAssessor {
913
377
  const mutated = tool.definitionEvidence?.mutatedDescription
914
378
  ? `"${tool.definitionEvidence.mutatedDescription.substring(0, 100)}..."`
915
379
  : "unknown";
916
- recommendations.push(`${tool.tool}: Description changed at invocation ${tool.definitionMutationAt}. Baseline: ${baseline} Mutated: ${mutated}`);
380
+ recommendations.push(`${tool.tool}: Description changed at invocation ${tool.definitionMutationAt}. Baseline: ${baseline} -> Mutated: ${mutated}`);
917
381
  }
918
382
  recommendations.push("Review tool source code for global state that mutates __doc__, description, or tool metadata based on call count.");
919
383
  }