@bryan-thompson/inspector-assessment-client 1.25.1 → 1.25.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/dist/assets/{OAuthCallback-CkzX_H4T.js → OAuthCallback-Dl4GYls3.js} +1 -1
  2. package/dist/assets/{OAuthDebugCallback-jZEkm74B.js → OAuthDebugCallback-BdJ38Z-r.js} +1 -1
  3. package/dist/assets/{index-Df9Sx1jt.css → index-cHhcEXbr.css} +4 -0
  4. package/dist/assets/{index-BVx1dGJT.js → index-pfUiTdQb.js} +4 -4
  5. package/dist/index.html +2 -2
  6. package/lib/lib/assessment/configTypes.d.ts +3 -0
  7. package/lib/lib/assessment/configTypes.d.ts.map +1 -1
  8. package/lib/lib/assessment/configTypes.js +11 -6
  9. package/lib/lib/assessment/coreTypes.d.ts +65 -0
  10. package/lib/lib/assessment/coreTypes.d.ts.map +1 -1
  11. package/lib/lib/assessment/extendedTypes.d.ts +127 -0
  12. package/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
  13. package/lib/lib/assessment/resultTypes.d.ts +45 -0
  14. package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
  15. package/lib/services/assessment/AssessmentOrchestrator.d.ts +4 -12
  16. package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
  17. package/lib/services/assessment/AssessmentOrchestrator.js +49 -238
  18. package/lib/services/assessment/TestDataGenerator.d.ts +9 -1
  19. package/lib/services/assessment/TestDataGenerator.d.ts.map +1 -1
  20. package/lib/services/assessment/TestDataGenerator.js +32 -6
  21. package/lib/services/assessment/TestScenarioEngine.d.ts +9 -1
  22. package/lib/services/assessment/TestScenarioEngine.d.ts.map +1 -1
  23. package/lib/services/assessment/TestScenarioEngine.js +17 -14
  24. package/lib/services/assessment/ToolClassifier.d.ts +154 -27
  25. package/lib/services/assessment/ToolClassifier.d.ts.map +1 -1
  26. package/lib/services/assessment/ToolClassifier.js +171 -318
  27. package/lib/services/assessment/config/annotationPatterns.d.ts +3 -1
  28. package/lib/services/assessment/config/annotationPatterns.d.ts.map +1 -1
  29. package/lib/services/assessment/config/annotationPatterns.js +5 -2
  30. package/lib/services/assessment/config/architecturePatterns.d.ts +101 -0
  31. package/lib/services/assessment/config/architecturePatterns.d.ts.map +1 -0
  32. package/lib/services/assessment/config/architecturePatterns.js +248 -0
  33. package/lib/services/assessment/config/performanceConfig.d.ts +122 -0
  34. package/lib/services/assessment/config/performanceConfig.d.ts.map +1 -0
  35. package/lib/services/assessment/config/performanceConfig.js +154 -0
  36. package/lib/services/assessment/config/sanitizationPatterns.d.ts +63 -0
  37. package/lib/services/assessment/config/sanitizationPatterns.d.ts.map +1 -0
  38. package/lib/services/assessment/config/sanitizationPatterns.js +223 -0
  39. package/lib/services/assessment/lib/claudeCodeBridge.d.ts +3 -1
  40. package/lib/services/assessment/lib/claudeCodeBridge.d.ts.map +1 -1
  41. package/lib/services/assessment/lib/claudeCodeBridge.js +5 -3
  42. package/lib/services/assessment/lib/concurrencyLimit.d.ts +6 -2
  43. package/lib/services/assessment/lib/concurrencyLimit.d.ts.map +1 -1
  44. package/lib/services/assessment/lib/concurrencyLimit.js +13 -6
  45. package/lib/services/assessment/lib/errors.d.ts +90 -0
  46. package/lib/services/assessment/lib/errors.d.ts.map +1 -0
  47. package/lib/services/assessment/lib/errors.js +136 -0
  48. package/lib/services/assessment/lib/timeoutUtils.d.ts +69 -0
  49. package/lib/services/assessment/lib/timeoutUtils.d.ts.map +1 -0
  50. package/lib/services/assessment/lib/timeoutUtils.js +103 -0
  51. package/lib/services/assessment/modules/BaseAssessor.d.ts +43 -8
  52. package/lib/services/assessment/modules/BaseAssessor.d.ts.map +1 -1
  53. package/lib/services/assessment/modules/BaseAssessor.js +103 -34
  54. package/lib/services/assessment/modules/DeveloperExperienceAssessor.d.ts +38 -1
  55. package/lib/services/assessment/modules/DeveloperExperienceAssessor.d.ts.map +1 -1
  56. package/lib/services/assessment/modules/DeveloperExperienceAssessor.js +185 -19
  57. package/lib/services/assessment/modules/DocumentationAssessor.d.ts +5 -0
  58. package/lib/services/assessment/modules/DocumentationAssessor.d.ts.map +1 -1
  59. package/lib/services/assessment/modules/DocumentationAssessor.js +11 -0
  60. package/lib/services/assessment/modules/ErrorHandlingAssessor.js +1 -1
  61. package/lib/services/assessment/modules/FunctionalityAssessor.d.ts.map +1 -1
  62. package/lib/services/assessment/modules/FunctionalityAssessor.js +6 -3
  63. package/lib/services/assessment/modules/MCPSpecComplianceAssessor.d.ts +3 -0
  64. package/lib/services/assessment/modules/MCPSpecComplianceAssessor.d.ts.map +1 -1
  65. package/lib/services/assessment/modules/MCPSpecComplianceAssessor.js +14 -2
  66. package/lib/services/assessment/modules/ManifestValidationAssessor.d.ts.map +1 -1
  67. package/lib/services/assessment/modules/ManifestValidationAssessor.js +7 -2
  68. package/lib/services/assessment/modules/PromptAssessor.d.ts +1 -0
  69. package/lib/services/assessment/modules/PromptAssessor.d.ts.map +1 -1
  70. package/lib/services/assessment/modules/PromptAssessor.js +26 -16
  71. package/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts.map +1 -1
  72. package/lib/services/assessment/modules/ProtocolComplianceAssessor.js +6 -2
  73. package/lib/services/assessment/modules/ProtocolConformanceAssessor.d.ts +5 -0
  74. package/lib/services/assessment/modules/ProtocolConformanceAssessor.d.ts.map +1 -1
  75. package/lib/services/assessment/modules/ProtocolConformanceAssessor.js +15 -0
  76. package/lib/services/assessment/modules/ResourceAssessor.d.ts.map +1 -1
  77. package/lib/services/assessment/modules/ResourceAssessor.js +8 -2
  78. package/lib/services/assessment/modules/SecurityAssessor.d.ts +3 -171
  79. package/lib/services/assessment/modules/SecurityAssessor.d.ts.map +1 -1
  80. package/lib/services/assessment/modules/SecurityAssessor.js +25 -1480
  81. package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts +27 -28
  82. package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts.map +1 -1
  83. package/lib/services/assessment/modules/ToolAnnotationAssessor.js +340 -863
  84. package/lib/services/assessment/modules/UsabilityAssessor.d.ts +5 -0
  85. package/lib/services/assessment/modules/UsabilityAssessor.d.ts.map +1 -1
  86. package/lib/services/assessment/modules/UsabilityAssessor.js +11 -0
  87. package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.d.ts +57 -0
  88. package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.d.ts.map +1 -0
  89. package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.js +176 -0
  90. package/lib/services/assessment/modules/annotations/ArchitectureDetector.d.ts +67 -0
  91. package/lib/services/assessment/modules/annotations/ArchitectureDetector.d.ts.map +1 -0
  92. package/lib/services/assessment/modules/annotations/ArchitectureDetector.js +239 -0
  93. package/lib/services/assessment/modules/annotations/BehaviorInference.d.ts +46 -0
  94. package/lib/services/assessment/modules/annotations/BehaviorInference.d.ts.map +1 -0
  95. package/lib/services/assessment/modules/annotations/BehaviorInference.js +394 -0
  96. package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.d.ts +64 -0
  97. package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.d.ts.map +1 -0
  98. package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.js +304 -0
  99. package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts +43 -0
  100. package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts.map +1 -0
  101. package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.js +276 -0
  102. package/lib/services/assessment/modules/annotations/SchemaAnalyzer.d.ts +122 -0
  103. package/lib/services/assessment/modules/annotations/SchemaAnalyzer.d.ts.map +1 -0
  104. package/lib/services/assessment/modules/annotations/SchemaAnalyzer.js +388 -0
  105. package/lib/services/assessment/modules/annotations/index.d.ts +13 -0
  106. package/lib/services/assessment/modules/annotations/index.d.ts.map +1 -0
  107. package/lib/services/assessment/modules/annotations/index.js +15 -0
  108. package/lib/services/assessment/modules/index.d.ts +10 -0
  109. package/lib/services/assessment/modules/index.d.ts.map +1 -1
  110. package/lib/services/assessment/modules/index.js +13 -0
  111. package/lib/services/assessment/modules/securityTests/SanitizationDetector.d.ts +125 -0
  112. package/lib/services/assessment/modules/securityTests/SanitizationDetector.d.ts.map +1 -0
  113. package/lib/services/assessment/modules/securityTests/SanitizationDetector.js +345 -0
  114. package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.d.ts +33 -0
  115. package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.d.ts.map +1 -0
  116. package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.js +128 -0
  117. package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts +67 -0
  118. package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts.map +1 -0
  119. package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.js +372 -0
  120. package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts +178 -0
  121. package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts.map +1 -0
  122. package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.js +1207 -0
  123. package/lib/services/assessment/modules/securityTests/index.d.ts +8 -0
  124. package/lib/services/assessment/modules/securityTests/index.d.ts.map +1 -0
  125. package/lib/services/assessment/modules/securityTests/index.js +7 -0
  126. package/lib/services/assessment/orchestratorHelpers.d.ts +83 -0
  127. package/lib/services/assessment/orchestratorHelpers.d.ts.map +1 -0
  128. package/lib/services/assessment/orchestratorHelpers.js +212 -0
  129. package/lib/services/assessment/tool-classifier-patterns.d.ts +85 -0
  130. package/lib/services/assessment/tool-classifier-patterns.d.ts.map +1 -0
  131. package/lib/services/assessment/tool-classifier-patterns.js +365 -0
  132. package/package.json +1 -1
@@ -7,413 +7,25 @@
7
7
  * - destructiveHint presence and accuracy
8
8
  * - Tool behavior inference from name patterns
9
9
  * - Annotation misalignment detection
10
+ * - Description poisoning detection (Issue #8)
10
11
  *
11
12
  * Reference: Anthropic MCP Directory Policy #17
12
- */
13
- import { BaseAssessor } from "./BaseAssessor.js";
14
- import { getDefaultCompiledPatterns, matchToolPattern, detectPersistenceModel, checkDescriptionForImmediatePersistence, } from "../config/annotationPatterns.js";
15
- /**
16
- * High-confidence deception detection patterns
17
- * These patterns detect obvious misalignment between annotations and tool names
18
- * where keywords appear ANYWHERE in the tool name (not just as prefixes)
19
- */
20
- /** Keywords that contradict readOnlyHint=true (these tools modify state) */
21
- const READONLY_CONTRADICTION_KEYWORDS = [
22
- // Execution keywords - tools that execute code/commands are never read-only
23
- "exec",
24
- "execute",
25
- "run",
26
- "shell",
27
- "command",
28
- "cmd",
29
- "spawn",
30
- "invoke",
31
- // Write/modify keywords
32
- "write",
33
- "create",
34
- "delete",
35
- "remove",
36
- "modify",
37
- "update",
38
- "edit",
39
- "change",
40
- "set",
41
- "put",
42
- "patch",
43
- // Deployment/installation keywords
44
- "install",
45
- "deploy",
46
- "upload",
47
- "push",
48
- // Communication keywords (sending data)
49
- "send",
50
- "post",
51
- "submit",
52
- "publish",
53
- // Destructive keywords
54
- "destroy",
55
- "drop",
56
- "purge",
57
- "wipe",
58
- "clear",
59
- "truncate",
60
- "reset",
61
- "kill",
62
- "terminate",
63
- ];
64
- /**
65
- * Suffixes that exempt "run" from readOnlyHint contradiction detection.
66
- * Tools matching "run" + these suffixes are legitimately read-only (fetch analysis data).
67
- * Issue #18: browser-tools-mcp uses runAccessibilityAudit, runSEOAudit, etc.
68
- */
69
- const RUN_READONLY_EXEMPT_SUFFIXES = [
70
- "audit", // runAccessibilityAudit, runPerformanceAudit, runSEOAudit
71
- "check", // runHealthCheck, runSecurityCheck
72
- "mode", // runAuditMode, runDebuggerMode
73
- "test", // runTest, runUnitTest (analysis, not execution)
74
- "scan", // runSecurityScan, runVulnerabilityScan
75
- "analyze", // runAnalyze, runCodeAnalyze
76
- "report", // runReport, runStatusReport
77
- "status", // runStatus, runHealthStatus
78
- "validate", // runValidate, runSchemaValidate
79
- "verify", // runVerify, runIntegrityVerify
80
- "inspect", // runInspect, runCodeInspect
81
- "lint", // runLint, runEslint
82
- "benchmark", // runBenchmark, runPerfBenchmark
83
- "diagnostic", // runDiagnostic
84
- ];
85
- /** Keywords that contradict destructiveHint=false (these tools delete/destroy data) */
86
- const DESTRUCTIVE_CONTRADICTION_KEYWORDS = [
87
- "delete",
88
- "remove",
89
- "drop",
90
- "destroy",
91
- "purge",
92
- "wipe",
93
- "erase",
94
- "truncate",
95
- "clear",
96
- "reset",
97
- "kill",
98
- "terminate",
99
- "revoke",
100
- "cancel",
101
- "force",
102
- ];
103
- /**
104
- * Check if a tool name contains any of the given keywords (case-insensitive)
105
- * Uses word segment matching to avoid false positives (e.g., "put" in "output")
106
- * Issue #25: Substring matching caused false positives for words like "output", "input", "compute"
107
13
  *
108
- * Handles: camelCase (putFile), snake_case (put_file), kebab-case (put-file), PascalCase (PutFile)
14
+ * This module orchestrates annotation assessment by coordinating:
15
+ * - BehaviorInference: Infers expected behavior from tool names
16
+ * - AnnotationDeceptionDetector: Detects keyword-based misalignments
17
+ * - DescriptionPoisoningDetector: Detects malicious content in descriptions
109
18
  */
110
- function containsKeyword(toolName, keywords) {
111
- // Normalize camelCase/PascalCase by inserting separator before uppercase letters
112
- // "putFile" "put_File", "updateUser" → "update_User", "GetOutput" → "Get_Output"
113
- const normalized = toolName.replace(/([a-z])([A-Z])/g, "$1_$2").toLowerCase();
114
- // Split by common separators (underscore, hyphen)
115
- const segments = normalized.split(/[_-]/);
116
- for (const keyword of keywords) {
117
- for (const segment of segments) {
118
- // Match if segment equals keyword or starts with keyword
119
- // This handles: "exec" matches "exec" segment, "exec_command" segment starts with "exec"
120
- if (segment === keyword || segment.startsWith(keyword)) {
121
- return keyword;
122
- }
123
- }
124
- }
125
- return null;
126
- }
127
- /**
128
- * Check if a tool name with "run" keyword is exempt from readOnlyHint contradiction.
129
- * Tools like "runAccessibilityAudit" are genuinely read-only (fetch analysis data).
130
- * Issue #18: Prevents false positives for analysis/audit tools.
131
- */
132
- function isRunKeywordExempt(toolName) {
133
- const lowerName = toolName.toLowerCase();
134
- // Only applies when "run" is detected
135
- if (!lowerName.includes("run")) {
136
- return false;
137
- }
138
- // Check if any exempt suffix is present
139
- return RUN_READONLY_EXEMPT_SUFFIXES.some((suffix) => lowerName.includes(suffix));
140
- }
141
- /**
142
- * Type guard for confidence levels that warrant event emission or status changes.
143
- * Uses positive check for acceptable levels (safer than !== "low" if new levels added).
144
- */
145
- function isActionableConfidence(confidence) {
146
- return confidence === "high" || confidence === "medium";
147
- }
148
- /**
149
- * Detect high-confidence annotation deception
150
- * Returns misalignment info if obvious deception detected, null otherwise
151
- */
152
- function detectAnnotationDeception(toolName, annotations) {
153
- // Check readOnlyHint=true contradiction
154
- if (annotations.readOnlyHint === true) {
155
- const keyword = containsKeyword(toolName, READONLY_CONTRADICTION_KEYWORDS);
156
- if (keyword) {
157
- // Issue #18: Skip deception flagging for "run" + analysis suffix combinations
158
- // Tools like "runAccessibilityAudit" are genuinely read-only
159
- if (keyword === "run" && isRunKeywordExempt(toolName)) {
160
- // Tool matches "run" but has an analysis suffix - not deceptive
161
- // Fall through to normal pattern-based inference
162
- }
163
- else {
164
- return {
165
- field: "readOnlyHint",
166
- matchedKeyword: keyword,
167
- reason: `Tool name contains '${keyword}' but claims readOnlyHint=true - this is likely deceptive`,
168
- };
169
- }
170
- }
171
- }
172
- // Check destructiveHint=false contradiction
173
- if (annotations.destructiveHint === false) {
174
- const keyword = containsKeyword(toolName, DESTRUCTIVE_CONTRADICTION_KEYWORDS);
175
- if (keyword) {
176
- return {
177
- field: "destructiveHint",
178
- matchedKeyword: keyword,
179
- reason: `Tool name contains '${keyword}' but claims destructiveHint=false - this is likely deceptive`,
180
- };
181
- }
182
- }
183
- return null;
184
- }
185
- const DESCRIPTION_POISONING_PATTERNS = [
186
- // Hidden instruction tags
187
- {
188
- name: "hidden_tag",
189
- pattern: /<HIDDEN>[\s\S]*?<\/HIDDEN>/gi,
190
- severity: "HIGH",
191
- category: "hidden_instructions",
192
- },
193
- {
194
- name: "important_tag",
195
- pattern: /<IMPORTANT>[\s\S]*?<\/IMPORTANT>/gi,
196
- severity: "HIGH",
197
- category: "hidden_instructions",
198
- },
199
- {
200
- name: "system_tag",
201
- pattern: /<SYSTEM>[\s\S]*?<\/SYSTEM>/gi,
202
- severity: "HIGH",
203
- category: "hidden_instructions",
204
- },
205
- {
206
- name: "instruction_tag",
207
- pattern: /<INSTRUCTION>[\s\S]*?<\/INSTRUCTION>/gi,
208
- severity: "HIGH",
209
- category: "hidden_instructions",
210
- },
211
- // Override commands
212
- {
213
- name: "ignore_instructions",
214
- pattern: /ignore\s+(all\s+)?(previous|prior|above)\s+instructions/i,
215
- severity: "HIGH",
216
- category: "override",
217
- },
218
- {
219
- name: "disregard_instructions",
220
- pattern: /disregard\s+(all\s+)?(previous|prior)\s+instructions/i,
221
- severity: "HIGH",
222
- category: "override",
223
- },
224
- {
225
- name: "you_are_now",
226
- pattern: /you\s+are\s+now\s+(a|an|the)/i,
227
- severity: "MEDIUM",
228
- category: "override",
229
- },
230
- {
231
- name: "system_override",
232
- pattern: /system:\s*override/i,
233
- severity: "HIGH",
234
- category: "override",
235
- },
236
- // Secrecy/concealment
237
- {
238
- name: "do_not_mention",
239
- pattern: /do\s+not\s+(mention|tell|inform|reveal)/i,
240
- severity: "HIGH",
241
- category: "concealment",
242
- },
243
- {
244
- name: "secretly",
245
- pattern: /\bsecretly\b/i,
246
- severity: "MEDIUM",
247
- category: "concealment",
248
- },
249
- {
250
- name: "hide_from_user",
251
- pattern: /hide\s+(this\s+)?from\s+(the\s+)?user/i,
252
- severity: "HIGH",
253
- category: "concealment",
254
- },
255
- // Data exfiltration
256
- {
257
- name: "return_keys",
258
- pattern: /return\s+(all\s+)?(api\s*)?keys/i,
259
- severity: "HIGH",
260
- category: "exfiltration",
261
- },
262
- {
263
- name: "include_credentials",
264
- // More specific: requires directive context (in/with/when/to) to reduce false positives
265
- // from legitimate documentation like "does not include credential storage"
266
- pattern: /include\s+(the\s+|all\s+|your\s+)?(api.?key|credential|password|secret)s?\s*(in|with|when|to)/i,
267
- severity: "HIGH",
268
- category: "exfiltration",
269
- },
270
- {
271
- name: "reveal_secrets",
272
- pattern: /reveal\s+(all\s+)?(secrets|credentials|api\s*keys)/i,
273
- severity: "HIGH",
274
- category: "exfiltration",
275
- },
276
- {
277
- name: "access_internal_secrets",
278
- pattern: /access\s+(the\s+)?internal\s+(api\s*)?(key|secret|credential|password|token)/i,
279
- severity: "HIGH",
280
- category: "exfiltration",
281
- },
282
- // Delimiter injection
283
- {
284
- name: "system_codeblock",
285
- pattern: /```system[\s\S]*?```/gi,
286
- severity: "HIGH",
287
- category: "delimiter",
288
- },
289
- {
290
- name: "inst_tags",
291
- pattern: /\[INST\][\s\S]*?\[\/INST\]/gi,
292
- severity: "HIGH",
293
- category: "delimiter",
294
- },
295
- {
296
- name: "chatml_system",
297
- pattern: /<\|im_start\|>system/gi,
298
- severity: "HIGH",
299
- category: "delimiter",
300
- },
301
- {
302
- name: "llama_sys",
303
- pattern: /<<SYS>>/gi,
304
- severity: "HIGH",
305
- category: "delimiter",
306
- },
307
- {
308
- name: "user_assistant_block",
309
- pattern: /\[USER\][\s\S]*?\[ASSISTANT\]/gi,
310
- severity: "HIGH",
311
- category: "delimiter",
312
- },
313
- // Role/persona injection (Warning #4)
314
- {
315
- name: "act_as",
316
- pattern: /act\s+(like|as)\s+(a|an|the)/i,
317
- severity: "MEDIUM",
318
- category: "override",
319
- },
320
- {
321
- name: "pretend_to_be",
322
- pattern: /pretend\s+(to\s+be|you\s*'?re)/i,
323
- severity: "MEDIUM",
324
- category: "override",
325
- },
326
- {
327
- name: "roleplay_as",
328
- pattern: /role\s*play\s+(as|like)/i,
329
- severity: "MEDIUM",
330
- category: "override",
331
- },
332
- {
333
- name: "new_task",
334
- pattern: /new\s+(task|instruction|objective):\s*/i,
335
- severity: "HIGH",
336
- category: "override",
337
- },
338
- // Encoding bypass detection (Warning #1)
339
- {
340
- name: "base64_encoded_block",
341
- pattern: /[A-Za-z0-9+/]{50,}={0,2}/g, // Large Base64 strings (50+ chars)
342
- severity: "MEDIUM",
343
- category: "encoding_bypass",
344
- },
345
- {
346
- name: "unicode_escape_sequence",
347
- pattern: /(?:\\u[0-9a-fA-F]{4}){3,}/gi, // 3+ consecutive Unicode escapes
348
- severity: "MEDIUM",
349
- category: "encoding_bypass",
350
- },
351
- {
352
- name: "html_entity_block",
353
- pattern: /(?:&#x?[0-9a-fA-F]+;){3,}/gi, // 3+ consecutive HTML entities
354
- severity: "MEDIUM",
355
- category: "encoding_bypass",
356
- },
357
- // Typoglycemia/evasion patterns (Warning #2)
358
- {
359
- name: "ignore_instructions_typo",
360
- pattern: /ign[o0]r[e3]?\s+(all\s+)?(pr[e3]v[i1][o0]us|pr[i1][o0]r|ab[o0]v[e3])\s+[i1]nstruct[i1][o0]ns?/i,
361
- severity: "HIGH",
362
- category: "override",
363
- },
364
- {
365
- name: "disregard_typo",
366
- pattern: /d[i1]sr[e3]g[a4]rd\s+(all\s+)?(pr[e3]v[i1][o0]us|pr[i1][o0]r)\s+[i1]nstruct[i1][o0]ns?/i,
367
- severity: "HIGH",
368
- category: "override",
369
- },
370
- // DVMCP-specific patterns (CTF testbed coverage)
371
- {
372
- name: "override_auth_protocol",
373
- pattern: /override[-_]?auth[-_]?protocol/gi,
374
- severity: "HIGH",
375
- category: "auth_bypass",
376
- },
377
- {
378
- name: "internal_resource_uri",
379
- pattern: /(?:internal|system|admin|secret|company|confidential):\/\/[a-z_\-/]+/gi,
380
- severity: "HIGH",
381
- category: "resource_access",
382
- },
383
- {
384
- name: "get_secrets_call",
385
- pattern: /get[-_]?secrets?\s*\(/gi,
386
- severity: "HIGH",
387
- category: "secret_access",
388
- },
389
- {
390
- name: "master_password",
391
- pattern: /master[-_]?password/gi,
392
- severity: "HIGH",
393
- category: "credential_exposure",
394
- },
395
- {
396
- name: "access_confidential",
397
- pattern: /access\s+(the\s+)?confidential/gi,
398
- severity: "HIGH",
399
- category: "exfiltration",
400
- },
401
- {
402
- name: "hidden_trigger_phrase",
403
- pattern: /if\s+(the\s+)?(query|input|text)\s+contains\s+(the\s+)?(exact\s+)?(phrase|word)/gi,
404
- severity: "HIGH",
405
- category: "hidden_trigger",
406
- },
407
- ];
408
- // NOTE: Pattern arrays moved to config/annotationPatterns.ts for configurability
409
- // The patterns are now loaded from getDefaultCompiledPatterns() or custom config
19
+ import { BaseAssessor } from "./BaseAssessor.js";
20
+ import { getDefaultCompiledPatterns, detectPersistenceModel, } from "../config/annotationPatterns.js";
21
+ // Import from extracted modules
22
+ import { scanDescriptionForPoisoning, detectAnnotationDeception, isActionableConfidence, inferBehavior, detectArchitecture, } from "./annotations/index.js";
410
23
  export class ToolAnnotationAssessor extends BaseAssessor {
411
24
  claudeBridge;
412
25
  compiledPatterns;
413
26
  persistenceContext;
414
27
  constructor(config) {
415
28
  super(config);
416
- // Initialize with default patterns (can be overridden via setPatterns)
417
29
  this.compiledPatterns = getDefaultCompiledPatterns();
418
30
  }
419
31
  /**
@@ -454,20 +66,46 @@ export class ToolAnnotationAssessor extends BaseAssessor {
454
66
  let missingAnnotationsCount = 0;
455
67
  let misalignedAnnotationsCount = 0;
456
68
  let poisonedDescriptionsCount = 0;
457
- // Track annotation sources
458
69
  const annotationSourceCounts = {
459
70
  mcp: 0,
460
71
  sourceCode: 0,
461
72
  inferred: 0,
462
73
  none: 0,
463
74
  };
464
- // Detect server persistence model from tool names (Three-Tier Classification)
75
+ // Extended metadata counters (Issue #54)
76
+ const extendedMetadataCounts = {
77
+ toolsWithRateLimits: 0,
78
+ toolsWithPermissions: 0,
79
+ toolsWithReturnSchema: 0,
80
+ toolsWithBulkSupport: 0,
81
+ };
82
+ // Detect server persistence model from tool names
465
83
  const toolNames = context.tools.map((t) => t.name);
466
84
  this.persistenceContext = detectPersistenceModel(toolNames);
467
85
  this.log(`Persistence model detected: ${this.persistenceContext.model} (confidence: ${this.persistenceContext.confidence})`);
468
- for (const indicator of this.persistenceContext.indicators) {
469
- this.log(` - ${indicator}`);
470
- }
86
+ // Issue #57: Detect server architecture
87
+ const architectureContext = {
88
+ tools: context.tools.map((t) => ({
89
+ name: t.name,
90
+ description: t.description,
91
+ inputSchema: t.inputSchema,
92
+ })),
93
+ transportType: context.transportConfig?.type,
94
+ sourceCodeFiles: context.sourceCodeFiles,
95
+ packageJson: context.packageJson && typeof context.packageJson === "object"
96
+ ? context.packageJson
97
+ : undefined,
98
+ };
99
+ const architectureAnalysis = detectArchitecture(architectureContext);
100
+ this.log(`Architecture detected: ${architectureAnalysis.serverType} server, databases: ${architectureAnalysis.databaseBackends.join(", ") || "none"}, network: ${architectureAnalysis.requiresNetworkAccess}`);
101
+ // Issue #57: Behavior inference metrics tracking
102
+ const behaviorInferenceMetrics = {
103
+ namePatternMatches: 0,
104
+ descriptionMatches: 0,
105
+ schemaMatches: 0,
106
+ aggregatedConfidenceSum: 0,
107
+ toolCount: 0,
108
+ };
471
109
  const useClaudeInference = this.isClaudeEnabled();
472
110
  if (useClaudeInference) {
473
111
  this.log("Claude Code integration enabled - using semantic behavior inference");
@@ -479,7 +117,6 @@ export class ToolAnnotationAssessor extends BaseAssessor {
479
117
  if (useClaudeInference) {
480
118
  const enhancedResult = await this.enhanceWithClaudeInference(tool, result);
481
119
  toolResults.push(enhancedResult);
482
- // Count based on Claude analysis if high confidence
483
120
  if (enhancedResult.claudeInference &&
484
121
  enhancedResult.claudeInference.confidence >= 70 &&
485
122
  enhancedResult.claudeInference.misalignmentDetected) {
@@ -490,7 +127,6 @@ export class ToolAnnotationAssessor extends BaseAssessor {
490
127
  }
491
128
  }
492
129
  else {
493
- // Standard pattern-based result
494
130
  const inferredBehavior = result.inferredBehavior ?? {
495
131
  expectedReadOnly: false,
496
132
  expectedDestructive: false,
@@ -501,7 +137,7 @@ export class ToolAnnotationAssessor extends BaseAssessor {
501
137
  claudeInference: {
502
138
  expectedReadOnly: inferredBehavior.expectedReadOnly,
503
139
  expectedDestructive: inferredBehavior.expectedDestructive,
504
- confidence: 50, // Lower confidence for pattern-based
140
+ confidence: 50,
505
141
  reasoning: inferredBehavior.reason,
506
142
  suggestedAnnotations: {
507
143
  readOnlyHint: inferredBehavior.expectedReadOnly,
@@ -536,7 +172,43 @@ export class ToolAnnotationAssessor extends BaseAssessor {
536
172
  else {
537
173
  annotationSourceCounts.none++;
538
174
  }
539
- // Track and emit poisoned description detection (Issue #8)
175
+ // Track extended metadata (Issue #54)
176
+ if (latestResult.extendedMetadata) {
177
+ if (latestResult.extendedMetadata.rateLimit) {
178
+ extendedMetadataCounts.toolsWithRateLimits++;
179
+ }
180
+ if (latestResult.extendedMetadata.permissions) {
181
+ extendedMetadataCounts.toolsWithPermissions++;
182
+ }
183
+ if (latestResult.extendedMetadata.returnSchema?.hasSchema) {
184
+ extendedMetadataCounts.toolsWithReturnSchema++;
185
+ }
186
+ if (latestResult.extendedMetadata.bulkOperations) {
187
+ extendedMetadataCounts.toolsWithBulkSupport++;
188
+ }
189
+ }
190
+ // Issue #57: Track behavior inference metrics
191
+ if (latestResult.inferredBehavior) {
192
+ behaviorInferenceMetrics.toolCount++;
193
+ // Check if name pattern was primary signal
194
+ if (latestResult.inferredBehavior.reason.includes("pattern") ||
195
+ latestResult.inferredBehavior.confidence === "high") {
196
+ behaviorInferenceMetrics.namePatternMatches++;
197
+ }
198
+ // Check if description was a factor
199
+ if (latestResult.inferredBehavior.reason.includes("Description") ||
200
+ latestResult.inferredBehavior.reason.includes("description")) {
201
+ behaviorInferenceMetrics.descriptionMatches++;
202
+ }
203
+ // Calculate confidence contribution
204
+ const confVal = latestResult.inferredBehavior.confidence === "high"
205
+ ? 90
206
+ : latestResult.inferredBehavior.confidence === "medium"
207
+ ? 70
208
+ : 40;
209
+ behaviorInferenceMetrics.aggregatedConfidenceSum += confVal;
210
+ }
211
+ // Emit poisoned description event
540
212
  if (latestResult.descriptionPoisoning?.detected) {
541
213
  poisonedDescriptionsCount++;
542
214
  this.log(`POISONED DESCRIPTION DETECTED: ${tool.name} contains suspicious patterns`);
@@ -550,143 +222,18 @@ export class ToolAnnotationAssessor extends BaseAssessor {
550
222
  });
551
223
  }
552
224
  }
553
- // Emit annotation_missing event with tool details
554
- if (!latestResult.hasAnnotations) {
555
- if (context.onProgress && latestResult.inferredBehavior) {
556
- const annotations = this.extractAnnotations(tool);
557
- context.onProgress({
558
- type: "annotation_missing",
559
- tool: tool.name,
560
- title: annotations.title,
561
- description: tool.description,
562
- parameters: this.extractToolParams(tool.inputSchema),
563
- inferredBehavior: {
564
- expectedReadOnly: latestResult.inferredBehavior.expectedReadOnly,
565
- expectedDestructive: latestResult.inferredBehavior.expectedDestructive,
566
- reason: latestResult.inferredBehavior.reason,
567
- },
568
- });
569
- }
570
- }
571
- // Emit annotation_aligned event when annotations correctly match behavior
572
- if (latestResult.hasAnnotations &&
573
- latestResult.alignmentStatus === "ALIGNED") {
574
- if (context.onProgress) {
575
- const annotations = latestResult.annotations;
576
- const inferredConfidence = latestResult.inferredBehavior?.confidence ?? "medium";
577
- context.onProgress({
578
- type: "annotation_aligned",
579
- tool: tool.name,
580
- confidence: inferredConfidence,
581
- annotations: {
582
- readOnlyHint: annotations?.readOnlyHint,
583
- destructiveHint: annotations?.destructiveHint,
584
- openWorldHint: annotations?.openWorldHint,
585
- idempotentHint: annotations?.idempotentHint,
586
- },
587
- });
588
- }
589
- }
590
- // Emit appropriate event based on alignment status
591
- if (context.onProgress && latestResult.inferredBehavior) {
592
- const annotations = latestResult.annotations;
593
- const inferred = latestResult.inferredBehavior;
594
- const confidence = latestResult.claudeInference?.confidence ?? 50;
595
- const toolParams = this.extractToolParams(tool.inputSchema);
596
- const toolAnnotations = this.extractAnnotations(tool);
597
- const alignmentStatus = latestResult.alignmentStatus;
598
- // Check readOnlyHint mismatch
599
- // Only emit events when inference is confident enough to contradict explicit annotations
600
- if (annotations?.readOnlyHint !== undefined &&
601
- annotations.readOnlyHint !== inferred.expectedReadOnly) {
602
- if (alignmentStatus === "REVIEW_RECOMMENDED") {
603
- // Emit review_recommended for ambiguous cases
604
- context.onProgress({
605
- type: "annotation_review_recommended",
606
- tool: tool.name,
607
- title: toolAnnotations.title,
608
- description: tool.description,
609
- parameters: toolParams,
610
- field: "readOnlyHint",
611
- actual: annotations.readOnlyHint,
612
- inferred: inferred.expectedReadOnly,
613
- confidence: inferred.confidence,
614
- isAmbiguous: inferred.isAmbiguous,
615
- reason: inferred.reason,
616
- });
617
- }
618
- else if (!inferred.isAmbiguous &&
619
- isActionableConfidence(inferred.confidence)) {
620
- // Emit misaligned only for medium/high-confidence mismatches
621
- // When inference is low-confidence/ambiguous, trust explicit annotation
622
- context.onProgress({
623
- type: "annotation_misaligned",
624
- tool: tool.name,
625
- title: toolAnnotations.title,
626
- description: tool.description,
627
- parameters: toolParams,
628
- field: "readOnlyHint",
629
- actual: annotations.readOnlyHint,
630
- expected: inferred.expectedReadOnly,
631
- confidence,
632
- reason: `Tool has readOnlyHint=${annotations.readOnlyHint}, but ${inferred.reason}`,
633
- });
634
- }
635
- // When inference is ambiguous/low-confidence, trust explicit annotation - no event emitted
636
- }
637
- // Check destructiveHint mismatch
638
- // Only emit events when inference is confident enough to contradict explicit annotations
639
- if (annotations?.destructiveHint !== undefined &&
640
- annotations.destructiveHint !== inferred.expectedDestructive) {
641
- if (alignmentStatus === "REVIEW_RECOMMENDED") {
642
- // Emit review_recommended for ambiguous cases
643
- context.onProgress({
644
- type: "annotation_review_recommended",
645
- tool: tool.name,
646
- title: toolAnnotations.title,
647
- description: tool.description,
648
- parameters: toolParams,
649
- field: "destructiveHint",
650
- actual: annotations.destructiveHint,
651
- inferred: inferred.expectedDestructive,
652
- confidence: inferred.confidence,
653
- isAmbiguous: inferred.isAmbiguous,
654
- reason: inferred.reason,
655
- });
656
- }
657
- else if (!inferred.isAmbiguous &&
658
- isActionableConfidence(inferred.confidence)) {
659
- // Emit misaligned only for medium/high-confidence mismatches
660
- // When inference is low-confidence/ambiguous, trust explicit annotation
661
- context.onProgress({
662
- type: "annotation_misaligned",
663
- tool: tool.name,
664
- title: toolAnnotations.title,
665
- description: tool.description,
666
- parameters: toolParams,
667
- field: "destructiveHint",
668
- actual: annotations.destructiveHint,
669
- expected: inferred.expectedDestructive,
670
- confidence,
671
- reason: `Tool has destructiveHint=${annotations.destructiveHint}, but ${inferred.reason}`,
672
- });
673
- }
674
- // When inference is ambiguous/low-confidence, trust explicit annotation - no event emitted
675
- }
676
- }
225
+ // Emit annotation events
226
+ this.emitAnnotationEvents(context, tool, latestResult);
677
227
  }
678
228
  const status = this.determineAnnotationStatus(toolResults, context.tools.length);
679
229
  const explanation = this.generateExplanation(annotatedCount, missingAnnotationsCount, misalignedAnnotationsCount, context.tools.length);
680
230
  const recommendations = this.generateRecommendations(toolResults);
681
- // Calculate new metrics and alignment breakdown
682
231
  const { metrics, alignmentBreakdown } = this.calculateMetrics(toolResults, context.tools.length);
683
232
  this.log(`Assessment complete: ${annotatedCount}/${context.tools.length} tools annotated, ${misalignedAnnotationsCount} misaligned, ${alignmentBreakdown.reviewRecommended} need review, ${poisonedDescriptionsCount} poisoned`);
684
- // Return enhanced assessment if Claude was used
685
233
  if (useClaudeInference) {
686
234
  const highConfidenceMisalignments = toolResults.filter((r) => r.claudeInference &&
687
235
  r.claudeInference.confidence >= 70 &&
688
236
  r.claudeInference.misalignmentDetected);
689
- this.log(`Claude inference found ${highConfidenceMisalignments.length} high-confidence misalignments`);
690
237
  return {
691
238
  toolResults,
692
239
  annotatedCount,
@@ -699,6 +246,18 @@ export class ToolAnnotationAssessor extends BaseAssessor {
699
246
  alignmentBreakdown,
700
247
  annotationSources: annotationSourceCounts,
701
248
  poisonedDescriptionsDetected: poisonedDescriptionsCount,
249
+ extendedMetadataMetrics: extendedMetadataCounts,
250
+ // Issue #57: Architecture and behavior inference
251
+ architectureAnalysis,
252
+ behaviorInferenceMetrics: {
253
+ namePatternMatches: behaviorInferenceMetrics.namePatternMatches,
254
+ descriptionMatches: behaviorInferenceMetrics.descriptionMatches,
255
+ schemaMatches: behaviorInferenceMetrics.schemaMatches,
256
+ aggregatedConfidenceAvg: behaviorInferenceMetrics.toolCount > 0
257
+ ? Math.round(behaviorInferenceMetrics.aggregatedConfidenceSum /
258
+ behaviorInferenceMetrics.toolCount)
259
+ : 0,
260
+ },
702
261
  claudeEnhanced: true,
703
262
  highConfidenceMisalignments,
704
263
  };
@@ -715,8 +274,110 @@ export class ToolAnnotationAssessor extends BaseAssessor {
715
274
  alignmentBreakdown,
716
275
  annotationSources: annotationSourceCounts,
717
276
  poisonedDescriptionsDetected: poisonedDescriptionsCount,
277
+ extendedMetadataMetrics: extendedMetadataCounts,
278
+ // Issue #57: Architecture and behavior inference
279
+ architectureAnalysis,
280
+ behaviorInferenceMetrics: {
281
+ namePatternMatches: behaviorInferenceMetrics.namePatternMatches,
282
+ descriptionMatches: behaviorInferenceMetrics.descriptionMatches,
283
+ schemaMatches: behaviorInferenceMetrics.schemaMatches,
284
+ aggregatedConfidenceAvg: behaviorInferenceMetrics.toolCount > 0
285
+ ? Math.round(behaviorInferenceMetrics.aggregatedConfidenceSum /
286
+ behaviorInferenceMetrics.toolCount)
287
+ : 0,
288
+ },
718
289
  };
719
290
  }
291
+ /**
292
+ * Emit annotation-related progress events
293
+ */
294
+ emitAnnotationEvents(context, tool, result) {
295
+ if (!context.onProgress || !result.inferredBehavior)
296
+ return;
297
+ const annotations = result.annotations;
298
+ const inferred = result.inferredBehavior;
299
+ const confidence = result.claudeInference?.confidence ?? 50;
300
+ const toolParams = this.extractToolParams(tool.inputSchema);
301
+ const toolAnnotations = this.extractAnnotations(tool);
302
+ // Emit missing annotation event
303
+ if (!result.hasAnnotations) {
304
+ context.onProgress({
305
+ type: "annotation_missing",
306
+ tool: tool.name,
307
+ title: toolAnnotations.title,
308
+ description: tool.description,
309
+ parameters: toolParams,
310
+ inferredBehavior: {
311
+ expectedReadOnly: inferred.expectedReadOnly,
312
+ expectedDestructive: inferred.expectedDestructive,
313
+ reason: inferred.reason,
314
+ },
315
+ });
316
+ return;
317
+ }
318
+ // Emit aligned event
319
+ if (result.alignmentStatus === "ALIGNED") {
320
+ context.onProgress({
321
+ type: "annotation_aligned",
322
+ tool: tool.name,
323
+ confidence: inferred.confidence ?? "medium",
324
+ annotations: {
325
+ readOnlyHint: annotations?.readOnlyHint,
326
+ destructiveHint: annotations?.destructiveHint,
327
+ openWorldHint: annotations?.openWorldHint,
328
+ idempotentHint: annotations?.idempotentHint,
329
+ },
330
+ });
331
+ return;
332
+ }
333
+ // Check readOnlyHint mismatch
334
+ if (annotations?.readOnlyHint !== undefined &&
335
+ annotations.readOnlyHint !== inferred.expectedReadOnly) {
336
+ this.emitMismatchEvent(context, tool, toolParams, toolAnnotations, "readOnlyHint", annotations.readOnlyHint, inferred.expectedReadOnly, confidence, inferred, result.alignmentStatus);
337
+ }
338
+ // Check destructiveHint mismatch
339
+ if (annotations?.destructiveHint !== undefined &&
340
+ annotations.destructiveHint !== inferred.expectedDestructive) {
341
+ this.emitMismatchEvent(context, tool, toolParams, toolAnnotations, "destructiveHint", annotations.destructiveHint, inferred.expectedDestructive, confidence, inferred, result.alignmentStatus);
342
+ }
343
+ }
344
+ /**
345
+ * Emit mismatch event (misaligned or review_recommended)
346
+ */
347
+ emitMismatchEvent(context, tool, toolParams, toolAnnotations, field, actual, expected, confidence, inferred, alignmentStatus) {
348
+ if (!context.onProgress)
349
+ return;
350
+ if (alignmentStatus === "REVIEW_RECOMMENDED") {
351
+ context.onProgress({
352
+ type: "annotation_review_recommended",
353
+ tool: tool.name,
354
+ title: toolAnnotations.title,
355
+ description: tool.description,
356
+ parameters: toolParams,
357
+ field,
358
+ actual,
359
+ inferred: expected,
360
+ confidence: inferred.confidence,
361
+ isAmbiguous: inferred.isAmbiguous,
362
+ reason: inferred.reason,
363
+ });
364
+ }
365
+ else if (!inferred.isAmbiguous &&
366
+ isActionableConfidence(inferred.confidence)) {
367
+ context.onProgress({
368
+ type: "annotation_misaligned",
369
+ tool: tool.name,
370
+ title: toolAnnotations.title,
371
+ description: tool.description,
372
+ parameters: toolParams,
373
+ field,
374
+ actual,
375
+ expected,
376
+ confidence,
377
+ reason: `Tool has ${field}=${actual}, but ${inferred.reason}`,
378
+ });
379
+ }
380
+ }
720
381
  /**
721
382
  * Enhance tool assessment with Claude inference
722
383
  */
@@ -751,7 +412,6 @@ export class ToolAnnotationAssessor extends BaseAssessor {
751
412
  }
752
413
  : undefined;
753
414
  const inference = await this.claudeBridge.inferToolBehavior(tool, currentAnnotations);
754
- // Handle null result (Claude unavailable or error)
755
415
  if (!inference) {
756
416
  return {
757
417
  ...baseResult,
@@ -762,15 +422,12 @@ export class ToolAnnotationAssessor extends BaseAssessor {
762
422
  reasoning: "Claude inference unavailable. Using pattern-based analysis.",
763
423
  suggestedAnnotations: {},
764
424
  misalignmentDetected: false,
765
- misalignmentDetails: undefined,
766
425
  source: "pattern-based",
767
426
  },
768
427
  };
769
428
  }
770
- // Merge Claude inference with pattern-based findings
771
429
  const updatedIssues = [...baseResult.issues];
772
430
  const updatedRecommendations = [...baseResult.recommendations];
773
- // Add Claude-detected misalignment if high confidence
774
431
  if (inference.misalignmentDetected && inference.confidence >= 70) {
775
432
  const misalignmentMsg = inference.misalignmentDetails
776
433
  ? `Claude analysis (${inference.confidence}% confidence): ${inference.misalignmentDetails}`
@@ -778,7 +435,6 @@ export class ToolAnnotationAssessor extends BaseAssessor {
778
435
  if (!updatedIssues.some((i) => i.includes("Claude analysis"))) {
779
436
  updatedIssues.push(misalignmentMsg);
780
437
  }
781
- // Add specific recommendations based on Claude inference
782
438
  if (inference.suggestedAnnotations) {
783
439
  const { readOnlyHint, destructiveHint, idempotentHint } = inference.suggestedAnnotations;
784
440
  if (readOnlyHint !== undefined &&
@@ -812,7 +468,6 @@ export class ToolAnnotationAssessor extends BaseAssessor {
812
468
  }
813
469
  catch (error) {
814
470
  this.logError(`Claude inference failed for ${tool.name}`, error);
815
- // Fall back to pattern-based (use inferredBehavior from top of function)
816
471
  return {
817
472
  ...baseResult,
818
473
  claudeInference: {
@@ -830,112 +485,32 @@ export class ToolAnnotationAssessor extends BaseAssessor {
830
485
  };
831
486
  }
832
487
  }
833
- /**
834
- * Generate enhanced explanation with Claude analysis
835
- */
836
- generateEnhancedExplanation(annotatedCount, missingCount, highConfidenceMisalignments, totalTools) {
837
- const parts = [];
838
- if (totalTools === 0) {
839
- return "No tools found to assess for annotations.";
840
- }
841
- parts.push(`Tool annotation coverage: ${annotatedCount}/${totalTools} tools have annotations.`);
842
- if (missingCount > 0) {
843
- parts.push(`${missingCount} tool(s) are missing required annotations (readOnlyHint, destructiveHint).`);
844
- }
845
- if (highConfidenceMisalignments > 0) {
846
- parts.push(`Claude analysis identified ${highConfidenceMisalignments} high-confidence annotation misalignment(s).`);
847
- }
848
- parts.push("Analysis enhanced with Claude semantic behavior inference.");
849
- return parts.join(" ");
850
- }
851
- /**
852
- * Generate enhanced recommendations with Claude analysis
853
- */
854
- generateEnhancedRecommendations(results) {
855
- const recommendations = [];
856
- // Prioritize Claude high-confidence misalignments
857
- const claudeMisalignments = results.filter((r) => r.claudeInference &&
858
- r.claudeInference.source === "claude-inferred" &&
859
- r.claudeInference.confidence >= 70 &&
860
- r.claudeInference.misalignmentDetected);
861
- if (claudeMisalignments.length > 0) {
862
- recommendations.push("HIGH CONFIDENCE: Claude analysis identified the following annotation issues:");
863
- for (const result of claudeMisalignments.slice(0, 5)) {
864
- if (result.claudeInference) {
865
- recommendations.push(` - ${result.toolName}: ${result.claudeInference.reasoning}`);
866
- }
867
- }
868
- }
869
- // Collect Claude suggestions
870
- const claudeSuggestions = results
871
- .filter((r) => r.claudeInference &&
872
- r.claudeInference.source === "claude-inferred" &&
873
- r.claudeInference.confidence >= 60)
874
- .flatMap((r) => r.recommendations.filter((rec) => rec.includes("Claude")));
875
- if (claudeSuggestions.length > 0) {
876
- recommendations.push(...claudeSuggestions.slice(0, 5));
877
- }
878
- // Add pattern-based recommendations for remaining tools
879
- const patternRecs = new Set();
880
- for (const result of results) {
881
- for (const rec of result.recommendations) {
882
- if (!rec.includes("Claude")) {
883
- patternRecs.add(rec);
884
- }
885
- }
886
- }
887
- const destructiveRecs = Array.from(patternRecs).filter((r) => r.includes("destructive"));
888
- const otherRecs = Array.from(patternRecs).filter((r) => !r.includes("destructive"));
889
- if (destructiveRecs.length > 0) {
890
- recommendations.push("PRIORITY: Potential destructive tools without proper hints:");
891
- recommendations.push(...destructiveRecs.slice(0, 3));
892
- }
893
- if (otherRecs.length > 0 && recommendations.length < 10) {
894
- recommendations.push(...otherRecs.slice(0, 3));
895
- }
896
- if (recommendations.length === 0) {
897
- recommendations.push("All tools have proper annotations. No action required.");
898
- }
899
- else {
900
- recommendations.push("Reference: MCP Directory Policy #17 requires tools to have readOnlyHint and destructiveHint annotations.");
901
- }
902
- return recommendations;
903
- }
904
488
  /**
905
489
  * Assess a single tool's annotations
906
- * Now includes alignment status with confidence-aware logic
907
- * Enhanced with high-confidence deception detection for obvious misalignments
908
490
  */
909
491
  assessTool(tool) {
910
492
  const issues = [];
911
493
  const recommendations = [];
912
- // Extract annotations from tool
913
494
  const annotations = this.extractAnnotations(tool);
914
495
  const hasAnnotations = annotations.readOnlyHint !== undefined ||
915
496
  annotations.destructiveHint !== undefined;
916
- // Infer expected behavior from tool name
917
- const inferredBehavior = this.inferBehavior(tool.name, tool.description);
918
- // Determine alignment status
497
+ const inferredBehavior = inferBehavior(tool.name, tool.description, this.compiledPatterns, this.persistenceContext);
919
498
  let alignmentStatus = "ALIGNED";
920
- // Check for missing annotations
921
499
  if (!hasAnnotations) {
922
500
  issues.push("Missing tool annotations (readOnlyHint, destructiveHint)");
923
501
  recommendations.push(`Add annotations to ${tool.name}: readOnlyHint=${inferredBehavior.expectedReadOnly}, destructiveHint=${inferredBehavior.expectedDestructive}`);
924
502
  alignmentStatus = "UNKNOWN";
925
503
  }
926
504
  else {
927
- // FIRST: Check for high-confidence deception (keywords anywhere in tool name)
928
- // This catches obvious cases like "vulnerable_system_exec_tool" + readOnlyHint=true
505
+ // Check for high-confidence deception
929
506
  const deception = detectAnnotationDeception(tool.name, {
930
507
  readOnlyHint: annotations.readOnlyHint,
931
508
  destructiveHint: annotations.destructiveHint,
932
509
  });
933
510
  if (deception) {
934
- // High-confidence deception detected - this is MISALIGNED, not REVIEW_RECOMMENDED
935
511
  alignmentStatus = "MISALIGNED";
936
512
  issues.push(`DECEPTIVE ANNOTATION: ${deception.reason}`);
937
513
  recommendations.push(`CRITICAL: Fix deceptive ${deception.field} for ${tool.name} - tool name contains '${deception.matchedKeyword}' which contradicts the annotation`);
938
- // Override inferred behavior to match the detected deception
939
514
  if (deception.field === "readOnlyHint") {
940
515
  inferredBehavior.expectedReadOnly = false;
941
516
  inferredBehavior.confidence = "high";
@@ -950,19 +525,14 @@ export class ToolAnnotationAssessor extends BaseAssessor {
950
525
  }
951
526
  }
952
527
  else {
953
- // Normal flow: Check for misaligned annotations with confidence-aware logic
528
+ // Check for misaligned annotations
954
529
  const readOnlyMismatch = annotations.readOnlyHint !== undefined &&
955
530
  annotations.readOnlyHint !== inferredBehavior.expectedReadOnly;
956
531
  const destructiveMismatch = annotations.destructiveHint !== undefined &&
957
532
  annotations.destructiveHint !== inferredBehavior.expectedDestructive;
958
533
  if (readOnlyMismatch || destructiveMismatch) {
959
- // Only flag misalignment for medium/high confidence inference
960
- // When confidence is low/ambiguous, trust the explicit annotation
961
- // Note: High-confidence deception detection (exec/install keywords)
962
- // is handled in the `deception` block above, not here
963
534
  if (!inferredBehavior.isAmbiguous &&
964
535
  isActionableConfidence(inferredBehavior.confidence)) {
965
- // Medium/high confidence mismatch: MISALIGNED
966
536
  alignmentStatus = "MISALIGNED";
967
537
  if (readOnlyMismatch) {
968
538
  issues.push(`Potentially misaligned readOnlyHint: set to ${annotations.readOnlyHint}, expected ${inferredBehavior.expectedReadOnly} based on tool name pattern`);
@@ -973,28 +543,27 @@ export class ToolAnnotationAssessor extends BaseAssessor {
973
543
  recommendations.push(`Verify destructiveHint for ${tool.name}: currently ${annotations.destructiveHint}, tool name suggests ${inferredBehavior.expectedDestructive}`);
974
544
  }
975
545
  }
976
- // When inference is ambiguous/low confidence, trust the explicit annotation
977
- // and keep alignmentStatus as ALIGNED (no change needed)
978
546
  }
979
547
  }
980
548
  }
981
- // Check for destructive tools without explicit hint (only for high-confidence patterns)
549
+ // Check for destructive tools without explicit hint
982
550
  if (inferredBehavior.expectedDestructive &&
983
551
  isActionableConfidence(inferredBehavior.confidence) &&
984
552
  annotations.destructiveHint !== true) {
985
553
  issues.push("Tool appears destructive but destructiveHint is not set to true");
986
554
  recommendations.push(`Set destructiveHint=true for ${tool.name} - this tool appears to perform destructive operations`);
987
- // Only upgrade to MISALIGNED if we have high confidence
988
555
  if (inferredBehavior.confidence === "high") {
989
556
  alignmentStatus = "MISALIGNED";
990
557
  }
991
558
  }
992
- // Scan for description poisoning (Issue #8)
993
- const descriptionPoisoning = this.scanDescriptionForPoisoning(tool);
559
+ // Scan for description poisoning
560
+ const descriptionPoisoning = scanDescriptionForPoisoning(tool);
994
561
  if (descriptionPoisoning.detected) {
995
562
  issues.push(`Tool description contains suspicious patterns: ${descriptionPoisoning.patterns.map((p) => p.name).join(", ")}`);
996
563
  recommendations.push(`Review ${tool.name} description for potential prompt injection or hidden instructions`);
997
564
  }
565
+ // Extract extended metadata (Issue #54)
566
+ const extendedMetadata = this.extractExtendedMetadata(tool);
998
567
  return {
999
568
  toolName: tool.name,
1000
569
  hasAnnotations,
@@ -1005,63 +574,15 @@ export class ToolAnnotationAssessor extends BaseAssessor {
1005
574
  issues,
1006
575
  recommendations,
1007
576
  descriptionPoisoning,
1008
- };
1009
- }
1010
- /**
1011
- * Scan tool description for poisoning patterns (Issue #8)
1012
- * Detects hidden instructions, override commands, concealment, and exfiltration attempts
1013
- */
1014
- scanDescriptionForPoisoning(tool) {
1015
- const description = tool.description || "";
1016
- const matches = [];
1017
- for (const patternDef of DESCRIPTION_POISONING_PATTERNS) {
1018
- // Create a fresh regex to reset lastIndex
1019
- const regex = new RegExp(patternDef.pattern.source, patternDef.pattern.flags);
1020
- // Loop to find all matches (not just first)
1021
- let match;
1022
- while ((match = regex.exec(description)) !== null) {
1023
- matches.push({
1024
- name: patternDef.name,
1025
- pattern: patternDef.pattern.toString(),
1026
- severity: patternDef.severity,
1027
- category: patternDef.category,
1028
- evidence: match[0].substring(0, 100) + (match[0].length > 100 ? "..." : ""),
1029
- });
1030
- // Prevent infinite loop for patterns without 'g' flag
1031
- if (!regex.global)
1032
- break;
1033
- }
1034
- }
1035
- // Determine overall risk level based on highest severity match
1036
- let riskLevel = "NONE";
1037
- if (matches.some((m) => m.severity === "HIGH")) {
1038
- riskLevel = "HIGH";
1039
- }
1040
- else if (matches.some((m) => m.severity === "MEDIUM")) {
1041
- riskLevel = "MEDIUM";
1042
- }
1043
- else if (matches.length > 0) {
1044
- riskLevel = "LOW";
1045
- }
1046
- return {
1047
- detected: matches.length > 0,
1048
- patterns: matches,
1049
- riskLevel,
577
+ extendedMetadata,
1050
578
  };
1051
579
  }
1052
580
  /**
1053
581
  * Extract annotations from a tool
1054
- * MCP SDK may have annotations in different locations
1055
- *
1056
- * Priority order:
1057
- * 1. tool.annotations (MCP 2024-11 spec) - "mcp" source
1058
- * 2. Direct properties on tool - "mcp" source
1059
- * 3. tool.metadata - "mcp" source
1060
- * 4. No annotations found - "none" source
1061
582
  */
1062
583
  extractAnnotations(tool) {
1063
584
  const toolAny = tool;
1064
- // Priority 1: Check annotations object (MCP 2024-11 spec) - primary source
585
+ // Priority 1: Check annotations object (MCP 2024-11 spec)
1065
586
  if (toolAny.annotations) {
1066
587
  const hasAnnotations = toolAny.annotations.readOnlyHint !== undefined ||
1067
588
  toolAny.annotations.destructiveHint !== undefined;
@@ -1077,7 +598,7 @@ export class ToolAnnotationAssessor extends BaseAssessor {
1077
598
  };
1078
599
  }
1079
600
  }
1080
- // Priority 2: Check direct properties on tool object
601
+ // Priority 2: Check direct properties
1081
602
  if (toolAny.readOnlyHint !== undefined ||
1082
603
  toolAny.destructiveHint !== undefined) {
1083
604
  return {
@@ -1090,7 +611,7 @@ export class ToolAnnotationAssessor extends BaseAssessor {
1090
611
  source: "mcp",
1091
612
  };
1092
613
  }
1093
- // Priority 3: Check metadata (some servers use this)
614
+ // Priority 3: Check metadata
1094
615
  if (toolAny.metadata) {
1095
616
  const hasMetadataAnnotations = toolAny.metadata.readOnlyHint !== undefined ||
1096
617
  toolAny.metadata.destructiveHint !== undefined;
@@ -1106,7 +627,6 @@ export class ToolAnnotationAssessor extends BaseAssessor {
1106
627
  };
1107
628
  }
1108
629
  }
1109
- // No annotations found from MCP protocol
1110
630
  return {
1111
631
  title: toolAny.title,
1112
632
  description: tool.description,
@@ -1114,7 +634,64 @@ export class ToolAnnotationAssessor extends BaseAssessor {
1114
634
  };
1115
635
  }
1116
636
  /**
1117
- * Extract parameters from tool input schema for event emission
637
+ * Extract extended metadata from tool (Issue #54)
638
+ * Extracts rate limits, permissions, return schemas, and bulk operation support
639
+ */
640
+ extractExtendedMetadata(tool) {
641
+ const toolAny = tool;
642
+ const metadata = {};
643
+ // Rate limiting - check annotations, metadata, and direct props
644
+ const rateLimit = toolAny.rateLimit ||
645
+ toolAny.annotations?.rateLimit ||
646
+ toolAny.metadata?.rateLimit;
647
+ if (rateLimit && typeof rateLimit === "object") {
648
+ metadata.rateLimit = {
649
+ windowMs: rateLimit.windowMs,
650
+ maxRequests: rateLimit.maxRequests,
651
+ requestsPerMinute: rateLimit.requestsPerMinute,
652
+ requestsPerSecond: rateLimit.requestsPerSecond,
653
+ };
654
+ }
655
+ // Permissions - check requiredPermission, permissions, scopes
656
+ const permissions = toolAny.requiredPermission ||
657
+ toolAny.permissions ||
658
+ toolAny.annotations?.permissions ||
659
+ toolAny.metadata?.requiredPermission ||
660
+ toolAny.metadata?.permissions;
661
+ if (permissions) {
662
+ const required = Array.isArray(permissions) ? permissions : [permissions];
663
+ const scopes = toolAny.scopes ||
664
+ toolAny.annotations?.scopes ||
665
+ toolAny.metadata?.scopes;
666
+ metadata.permissions = {
667
+ required: required.filter((p) => typeof p === "string"),
668
+ scopes: Array.isArray(scopes)
669
+ ? scopes.filter((s) => typeof s === "string")
670
+ : undefined,
671
+ };
672
+ }
673
+ // Return schema - check outputSchema (MCP 2025-06-18 spec)
674
+ if (toolAny.outputSchema) {
675
+ metadata.returnSchema = {
676
+ hasSchema: true,
677
+ schema: toolAny.outputSchema,
678
+ };
679
+ }
680
+ // Bulk operations - check metadata for batch support
681
+ const bulkSupport = toolAny.supportsBulkOperations ||
682
+ toolAny.annotations?.supportsBulkOperations ||
683
+ toolAny.metadata?.supportsBulkOperations;
684
+ const maxBatchSize = toolAny.metadata?.maxBatchSize;
685
+ if (bulkSupport !== undefined || maxBatchSize !== undefined) {
686
+ metadata.bulkOperations = {
687
+ supported: !!bulkSupport,
688
+ maxBatchSize: typeof maxBatchSize === "number" ? maxBatchSize : undefined,
689
+ };
690
+ }
691
+ return Object.keys(metadata).length > 0 ? metadata : undefined;
692
+ }
693
+ /**
694
+ * Extract parameters from tool input schema
1118
695
  */
1119
696
  extractToolParams(schema) {
1120
697
  if (!schema || typeof schema !== "object")
@@ -1137,196 +714,34 @@ export class ToolAnnotationAssessor extends BaseAssessor {
1137
714
  });
1138
715
  }
1139
716
  /**
1140
- * Infer expected behavior from tool name and description
1141
- * Now returns confidence level and ambiguity flag for better handling
1142
- */
1143
- inferBehavior(toolName, description) {
1144
- const lowerDesc = (description || "").toLowerCase();
1145
- // Issue #18: Early check for run + analysis suffix pattern
1146
- // Tools like "runAccessibilityAudit" are genuinely read-only (fetch analysis data)
1147
- // Check this BEFORE pattern matching to override the generic "run_" write pattern
1148
- if (isRunKeywordExempt(toolName)) {
1149
- return {
1150
- expectedReadOnly: true,
1151
- expectedDestructive: false,
1152
- reason: `Tool name contains 'run' with analysis suffix (audit, check, scan, etc.) - this is a read-only analysis operation`,
1153
- confidence: "medium",
1154
- isAmbiguous: false,
1155
- };
1156
- }
1157
- // Use the configurable pattern matching system
1158
- const patternMatch = matchToolPattern(toolName, this.compiledPatterns);
1159
- // Handle pattern match results
1160
- switch (patternMatch.category) {
1161
- case "ambiguous":
1162
- // Ambiguous patterns - don't make strong assertions
1163
- return {
1164
- expectedReadOnly: false,
1165
- expectedDestructive: false,
1166
- reason: `Tool name matches ambiguous pattern '${patternMatch.pattern}' - behavior varies by implementation context`,
1167
- confidence: "low",
1168
- isAmbiguous: true,
1169
- };
1170
- case "destructive":
1171
- return {
1172
- expectedReadOnly: false,
1173
- expectedDestructive: true,
1174
- reason: `Tool name matches destructive pattern: ${patternMatch.pattern}`,
1175
- confidence: "high",
1176
- isAmbiguous: false,
1177
- };
1178
- case "readOnly":
1179
- return {
1180
- expectedReadOnly: true,
1181
- expectedDestructive: false,
1182
- reason: `Tool name matches read-only pattern: ${patternMatch.pattern}`,
1183
- confidence: "high",
1184
- isAmbiguous: false,
1185
- };
1186
- case "write": {
1187
- // CREATE operations are NEVER destructive - they only ADD new data
1188
- // Only UPDATE/MODIFY operations can be considered destructive when they modify existing data
1189
- const isCreateOperation = /^(create|add|insert|new|generate)[_-]/i.test(toolName);
1190
- if (isCreateOperation) {
1191
- return {
1192
- expectedReadOnly: false,
1193
- expectedDestructive: false,
1194
- reason: `Tool name matches create pattern: ${patternMatch.pattern} - create operations only add data and are not destructive`,
1195
- confidence: "high",
1196
- isAmbiguous: false,
1197
- };
1198
- }
1199
- // Three-Tier Classification: Check persistence model for UPDATE/MODIFY operations
1200
- // If immediate persistence detected, update operations should be marked destructive
1201
- const descriptionCheck = checkDescriptionForImmediatePersistence(description || "");
1202
- // Priority 1: Description explicitly indicates deferred persistence
1203
- if (descriptionCheck.indicatesDeferred) {
1204
- return {
1205
- expectedReadOnly: false,
1206
- expectedDestructive: false,
1207
- reason: `Tool name matches write pattern (${patternMatch.pattern}), description indicates deferred/in-memory operation`,
1208
- confidence: "medium",
1209
- isAmbiguous: false,
1210
- };
1211
- }
1212
- // Priority 2: Description explicitly indicates immediate persistence
1213
- if (descriptionCheck.indicatesImmediate) {
1214
- return {
1215
- expectedReadOnly: false,
1216
- expectedDestructive: true,
1217
- reason: `Tool name matches write pattern (${patternMatch.pattern}), description indicates immediate persistence to storage (${descriptionCheck.matchedPatterns.slice(0, 2).join(", ")})`,
1218
- confidence: "medium",
1219
- isAmbiguous: false,
1220
- };
1221
- }
1222
- // Priority 3: Server-level persistence model (no save operations = immediate)
1223
- if (this.persistenceContext?.model === "immediate") {
1224
- return {
1225
- expectedReadOnly: false,
1226
- expectedDestructive: true,
1227
- reason: `Tool name matches write pattern (${patternMatch.pattern}), server has no save operations → write operations likely persist immediately`,
1228
- confidence: "medium",
1229
- isAmbiguous: false,
1230
- };
1231
- }
1232
- // Priority 4: Server has save operations = deferred (in-memory until save)
1233
- if (this.persistenceContext?.model === "deferred") {
1234
- return {
1235
- expectedReadOnly: false,
1236
- expectedDestructive: false,
1237
- reason: `Tool name matches write pattern (${patternMatch.pattern}), server has save operations → write operations likely in-memory until explicit save`,
1238
- confidence: "medium",
1239
- isAmbiguous: false,
1240
- };
1241
- }
1242
- // Default: Unknown persistence model - conservative approach (not destructive)
1243
- return {
1244
- expectedReadOnly: false,
1245
- expectedDestructive: false,
1246
- reason: `Tool name matches write pattern: ${patternMatch.pattern}`,
1247
- confidence: "medium",
1248
- isAmbiguous: false,
1249
- };
1250
- }
1251
- case "unknown":
1252
- default:
1253
- // Fall through to description-based analysis
1254
- break;
1255
- }
1256
- // Check description for hints (medium confidence)
1257
- if (lowerDesc.includes("delete") || lowerDesc.includes("remove")) {
1258
- return {
1259
- expectedReadOnly: false,
1260
- expectedDestructive: true,
1261
- reason: "Description mentions delete/remove operations",
1262
- confidence: "medium",
1263
- isAmbiguous: false,
1264
- };
1265
- }
1266
- if (lowerDesc.includes("read") ||
1267
- lowerDesc.includes("get") ||
1268
- lowerDesc.includes("fetch")) {
1269
- return {
1270
- expectedReadOnly: true,
1271
- expectedDestructive: false,
1272
- reason: "Description suggests read-only operation",
1273
- confidence: "medium",
1274
- isAmbiguous: false,
1275
- };
1276
- }
1277
- // Default: assume write with low confidence (ambiguous)
1278
- return {
1279
- expectedReadOnly: false,
1280
- expectedDestructive: false,
1281
- reason: "Could not infer behavior from name pattern",
1282
- confidence: "low",
1283
- isAmbiguous: true,
1284
- };
1285
- }
1286
- /**
1287
- * Determine overall status using alignment status.
1288
- * Only MISALIGNED counts as failure; REVIEW_RECOMMENDED does not fail.
717
+ * Determine overall status
1289
718
  */
1290
719
  determineAnnotationStatus(results, totalTools) {
1291
720
  if (totalTools === 0)
1292
721
  return "PASS";
1293
722
  const annotatedCount = results.filter((r) => r.hasAnnotations).length;
1294
- // Check for poisoned descriptions (Issue #8) - critical security issue
1295
723
  const poisonedCount = results.filter((r) => r.descriptionPoisoning?.detected === true).length;
1296
- if (poisonedCount > 0) {
724
+ if (poisonedCount > 0)
1297
725
  return "FAIL";
1298
- }
1299
- // Only count actual MISALIGNED, not REVIEW_RECOMMENDED
1300
726
  const misalignedCount = results.filter((r) => r.alignmentStatus === "MISALIGNED").length;
1301
- // Count high-confidence destructive tools without proper hints
1302
727
  const destructiveWithoutHint = results.filter((r) => r.inferredBehavior?.expectedDestructive === true &&
1303
728
  r.inferredBehavior?.confidence === "high" &&
1304
729
  r.annotations?.destructiveHint !== true).length;
1305
- // Destructive tools without proper hints = FAIL (critical safety issue)
1306
- if (destructiveWithoutHint > 0) {
730
+ if (destructiveWithoutHint > 0)
1307
731
  return "FAIL";
1308
- }
1309
- // High-confidence misalignments = FAIL
1310
- if (misalignedCount > 0) {
732
+ if (misalignedCount > 0)
1311
733
  return "FAIL";
1312
- }
1313
- // All tools annotated = PASS
1314
- if (annotatedCount === totalTools) {
734
+ if (annotatedCount === totalTools)
1315
735
  return "PASS";
1316
- }
1317
- // Some annotations missing = NEED_MORE_INFO
1318
736
  const annotationRate = annotatedCount / totalTools;
1319
- if (annotationRate >= 0.8) {
737
+ if (annotationRate >= 0.8)
1320
738
  return "NEED_MORE_INFO";
1321
- }
1322
- // Mostly missing annotations = FAIL
1323
- if (annotationRate < 0.5) {
739
+ if (annotationRate < 0.5)
1324
740
  return "FAIL";
1325
- }
1326
741
  return "NEED_MORE_INFO";
1327
742
  }
1328
743
  /**
1329
- * Calculate metrics and alignment breakdown for the assessment
744
+ * Calculate metrics and alignment breakdown
1330
745
  */
1331
746
  calculateMetrics(results, totalTools) {
1332
747
  const alignmentBreakdown = {
@@ -1338,17 +753,13 @@ export class ToolAnnotationAssessor extends BaseAssessor {
1338
753
  };
1339
754
  const annotatedCount = results.filter((r) => r.hasAnnotations).length;
1340
755
  const metrics = {
1341
- // Coverage: percentage of tools with annotations
1342
756
  coverage: totalTools > 0 ? (annotatedCount / totalTools) * 100 : 100,
1343
- // Consistency: percentage without contradictions (not MISALIGNED)
1344
757
  consistency: totalTools > 0
1345
758
  ? ((totalTools - alignmentBreakdown.misaligned) / totalTools) * 100
1346
759
  : 100,
1347
- // Correctness: percentage of annotated tools that are ALIGNED
1348
760
  correctness: annotatedCount > 0
1349
761
  ? (alignmentBreakdown.aligned / annotatedCount) * 100
1350
762
  : 0,
1351
- // Review required: count of tools needing manual review
1352
763
  reviewRequired: alignmentBreakdown.reviewRecommended,
1353
764
  };
1354
765
  return { metrics, alignmentBreakdown };
@@ -1373,19 +784,35 @@ export class ToolAnnotationAssessor extends BaseAssessor {
1373
784
  }
1374
785
  return parts.join(" ");
1375
786
  }
787
+ /**
788
+ * Generate enhanced explanation with Claude analysis
789
+ */
790
+ generateEnhancedExplanation(annotatedCount, missingCount, highConfidenceMisalignments, totalTools) {
791
+ const parts = [];
792
+ if (totalTools === 0) {
793
+ return "No tools found to assess for annotations.";
794
+ }
795
+ parts.push(`Tool annotation coverage: ${annotatedCount}/${totalTools} tools have annotations.`);
796
+ if (missingCount > 0) {
797
+ parts.push(`${missingCount} tool(s) are missing required annotations (readOnlyHint, destructiveHint).`);
798
+ }
799
+ if (highConfidenceMisalignments > 0) {
800
+ parts.push(`Claude analysis identified ${highConfidenceMisalignments} high-confidence annotation misalignment(s).`);
801
+ }
802
+ parts.push("Analysis enhanced with Claude semantic behavior inference.");
803
+ return parts.join(" ");
804
+ }
1376
805
  /**
1377
806
  * Generate recommendations
1378
807
  */
1379
808
  generateRecommendations(results) {
1380
809
  const recommendations = [];
1381
- // Collect unique recommendations from all tools
1382
810
  const allRecs = new Set();
1383
811
  for (const result of results) {
1384
812
  for (const rec of result.recommendations) {
1385
813
  allRecs.add(rec);
1386
814
  }
1387
815
  }
1388
- // Prioritize destructive tool warnings
1389
816
  const destructiveRecs = Array.from(allRecs).filter((r) => r.includes("destructive"));
1390
817
  const otherRecs = Array.from(allRecs).filter((r) => !r.includes("destructive"));
1391
818
  if (destructiveRecs.length > 0) {
@@ -1403,4 +830,54 @@ export class ToolAnnotationAssessor extends BaseAssessor {
1403
830
  }
1404
831
  return recommendations;
1405
832
  }
833
+ /**
834
+ * Generate enhanced recommendations with Claude analysis
835
+ */
836
+ generateEnhancedRecommendations(results) {
837
+ const recommendations = [];
838
+ const claudeMisalignments = results.filter((r) => r.claudeInference &&
839
+ r.claudeInference.source === "claude-inferred" &&
840
+ r.claudeInference.confidence >= 70 &&
841
+ r.claudeInference.misalignmentDetected);
842
+ if (claudeMisalignments.length > 0) {
843
+ recommendations.push("HIGH CONFIDENCE: Claude analysis identified the following annotation issues:");
844
+ for (const result of claudeMisalignments.slice(0, 5)) {
845
+ if (result.claudeInference) {
846
+ recommendations.push(` - ${result.toolName}: ${result.claudeInference.reasoning}`);
847
+ }
848
+ }
849
+ }
850
+ const claudeSuggestions = results
851
+ .filter((r) => r.claudeInference &&
852
+ r.claudeInference.source === "claude-inferred" &&
853
+ r.claudeInference.confidence >= 60)
854
+ .flatMap((r) => r.recommendations.filter((rec) => rec.includes("Claude")));
855
+ if (claudeSuggestions.length > 0) {
856
+ recommendations.push(...claudeSuggestions.slice(0, 5));
857
+ }
858
+ const patternRecs = new Set();
859
+ for (const result of results) {
860
+ for (const rec of result.recommendations) {
861
+ if (!rec.includes("Claude")) {
862
+ patternRecs.add(rec);
863
+ }
864
+ }
865
+ }
866
+ const destructiveRecs = Array.from(patternRecs).filter((r) => r.includes("destructive"));
867
+ const otherRecs = Array.from(patternRecs).filter((r) => !r.includes("destructive"));
868
+ if (destructiveRecs.length > 0) {
869
+ recommendations.push("PRIORITY: Potential destructive tools without proper hints:");
870
+ recommendations.push(...destructiveRecs.slice(0, 3));
871
+ }
872
+ if (otherRecs.length > 0 && recommendations.length < 10) {
873
+ recommendations.push(...otherRecs.slice(0, 3));
874
+ }
875
+ if (recommendations.length === 0) {
876
+ recommendations.push("All tools have proper annotations. No action required.");
877
+ }
878
+ else {
879
+ recommendations.push("Reference: MCP Directory Policy #17 requires tools to have readOnlyHint and destructiveHint annotations.");
880
+ }
881
+ return recommendations;
882
+ }
1406
883
  }