@bryan-thompson/inspector-assessment-client 1.25.1 → 1.25.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-CkzX_H4T.js → OAuthCallback-Dl4GYls3.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-jZEkm74B.js → OAuthDebugCallback-BdJ38Z-r.js} +1 -1
- package/dist/assets/{index-Df9Sx1jt.css → index-cHhcEXbr.css} +4 -0
- package/dist/assets/{index-BVx1dGJT.js → index-pfUiTdQb.js} +4 -4
- package/dist/index.html +2 -2
- package/lib/lib/assessment/configTypes.d.ts +3 -0
- package/lib/lib/assessment/configTypes.d.ts.map +1 -1
- package/lib/lib/assessment/configTypes.js +11 -6
- package/lib/lib/assessment/coreTypes.d.ts +65 -0
- package/lib/lib/assessment/coreTypes.d.ts.map +1 -1
- package/lib/lib/assessment/extendedTypes.d.ts +127 -0
- package/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
- package/lib/lib/assessment/resultTypes.d.ts +45 -0
- package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +4 -12
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +49 -238
- package/lib/services/assessment/TestDataGenerator.d.ts +9 -1
- package/lib/services/assessment/TestDataGenerator.d.ts.map +1 -1
- package/lib/services/assessment/TestDataGenerator.js +32 -6
- package/lib/services/assessment/TestScenarioEngine.d.ts +9 -1
- package/lib/services/assessment/TestScenarioEngine.d.ts.map +1 -1
- package/lib/services/assessment/TestScenarioEngine.js +17 -14
- package/lib/services/assessment/ToolClassifier.d.ts +154 -27
- package/lib/services/assessment/ToolClassifier.d.ts.map +1 -1
- package/lib/services/assessment/ToolClassifier.js +171 -318
- package/lib/services/assessment/config/annotationPatterns.d.ts +3 -1
- package/lib/services/assessment/config/annotationPatterns.d.ts.map +1 -1
- package/lib/services/assessment/config/annotationPatterns.js +5 -2
- package/lib/services/assessment/config/architecturePatterns.d.ts +101 -0
- package/lib/services/assessment/config/architecturePatterns.d.ts.map +1 -0
- package/lib/services/assessment/config/architecturePatterns.js +248 -0
- package/lib/services/assessment/config/performanceConfig.d.ts +122 -0
- package/lib/services/assessment/config/performanceConfig.d.ts.map +1 -0
- package/lib/services/assessment/config/performanceConfig.js +154 -0
- package/lib/services/assessment/config/sanitizationPatterns.d.ts +63 -0
- package/lib/services/assessment/config/sanitizationPatterns.d.ts.map +1 -0
- package/lib/services/assessment/config/sanitizationPatterns.js +223 -0
- package/lib/services/assessment/lib/claudeCodeBridge.d.ts +3 -1
- package/lib/services/assessment/lib/claudeCodeBridge.d.ts.map +1 -1
- package/lib/services/assessment/lib/claudeCodeBridge.js +5 -3
- package/lib/services/assessment/lib/concurrencyLimit.d.ts +6 -2
- package/lib/services/assessment/lib/concurrencyLimit.d.ts.map +1 -1
- package/lib/services/assessment/lib/concurrencyLimit.js +13 -6
- package/lib/services/assessment/lib/errors.d.ts +90 -0
- package/lib/services/assessment/lib/errors.d.ts.map +1 -0
- package/lib/services/assessment/lib/errors.js +136 -0
- package/lib/services/assessment/lib/timeoutUtils.d.ts +69 -0
- package/lib/services/assessment/lib/timeoutUtils.d.ts.map +1 -0
- package/lib/services/assessment/lib/timeoutUtils.js +103 -0
- package/lib/services/assessment/modules/BaseAssessor.d.ts +43 -8
- package/lib/services/assessment/modules/BaseAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/BaseAssessor.js +103 -34
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.d.ts +38 -1
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.js +185 -19
- package/lib/services/assessment/modules/DocumentationAssessor.d.ts +5 -0
- package/lib/services/assessment/modules/DocumentationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/DocumentationAssessor.js +11 -0
- package/lib/services/assessment/modules/ErrorHandlingAssessor.js +1 -1
- package/lib/services/assessment/modules/FunctionalityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/FunctionalityAssessor.js +6 -3
- package/lib/services/assessment/modules/MCPSpecComplianceAssessor.d.ts +3 -0
- package/lib/services/assessment/modules/MCPSpecComplianceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/MCPSpecComplianceAssessor.js +14 -2
- package/lib/services/assessment/modules/ManifestValidationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ManifestValidationAssessor.js +7 -2
- package/lib/services/assessment/modules/PromptAssessor.d.ts +1 -0
- package/lib/services/assessment/modules/PromptAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/PromptAssessor.js +26 -16
- package/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ProtocolComplianceAssessor.js +6 -2
- package/lib/services/assessment/modules/ProtocolConformanceAssessor.d.ts +5 -0
- package/lib/services/assessment/modules/ProtocolConformanceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ProtocolConformanceAssessor.js +15 -0
- package/lib/services/assessment/modules/ResourceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ResourceAssessor.js +8 -2
- package/lib/services/assessment/modules/SecurityAssessor.d.ts +3 -171
- package/lib/services/assessment/modules/SecurityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/SecurityAssessor.js +25 -1480
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts +27 -28
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ToolAnnotationAssessor.js +340 -863
- package/lib/services/assessment/modules/UsabilityAssessor.d.ts +5 -0
- package/lib/services/assessment/modules/UsabilityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/UsabilityAssessor.js +11 -0
- package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.d.ts +57 -0
- package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.js +176 -0
- package/lib/services/assessment/modules/annotations/ArchitectureDetector.d.ts +67 -0
- package/lib/services/assessment/modules/annotations/ArchitectureDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/ArchitectureDetector.js +239 -0
- package/lib/services/assessment/modules/annotations/BehaviorInference.d.ts +46 -0
- package/lib/services/assessment/modules/annotations/BehaviorInference.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/BehaviorInference.js +394 -0
- package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.d.ts +64 -0
- package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.js +304 -0
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts +43 -0
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.js +276 -0
- package/lib/services/assessment/modules/annotations/SchemaAnalyzer.d.ts +122 -0
- package/lib/services/assessment/modules/annotations/SchemaAnalyzer.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/SchemaAnalyzer.js +388 -0
- package/lib/services/assessment/modules/annotations/index.d.ts +13 -0
- package/lib/services/assessment/modules/annotations/index.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/index.js +15 -0
- package/lib/services/assessment/modules/index.d.ts +10 -0
- package/lib/services/assessment/modules/index.d.ts.map +1 -1
- package/lib/services/assessment/modules/index.js +13 -0
- package/lib/services/assessment/modules/securityTests/SanitizationDetector.d.ts +125 -0
- package/lib/services/assessment/modules/securityTests/SanitizationDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SanitizationDetector.js +345 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.d.ts +33 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.js +128 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts +67 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.js +372 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts +178 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.js +1207 -0
- package/lib/services/assessment/modules/securityTests/index.d.ts +8 -0
- package/lib/services/assessment/modules/securityTests/index.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/index.js +7 -0
- package/lib/services/assessment/orchestratorHelpers.d.ts +83 -0
- package/lib/services/assessment/orchestratorHelpers.d.ts.map +1 -0
- package/lib/services/assessment/orchestratorHelpers.js +212 -0
- package/lib/services/assessment/tool-classifier-patterns.d.ts +85 -0
- package/lib/services/assessment/tool-classifier-patterns.d.ts.map +1 -0
- package/lib/services/assessment/tool-classifier-patterns.js +365 -0
- package/package.json +1 -1
|
@@ -7,413 +7,25 @@
|
|
|
7
7
|
* - destructiveHint presence and accuracy
|
|
8
8
|
* - Tool behavior inference from name patterns
|
|
9
9
|
* - Annotation misalignment detection
|
|
10
|
+
* - Description poisoning detection (Issue #8)
|
|
10
11
|
*
|
|
11
12
|
* Reference: Anthropic MCP Directory Policy #17
|
|
12
|
-
*/
|
|
13
|
-
import { BaseAssessor } from "./BaseAssessor.js";
|
|
14
|
-
import { getDefaultCompiledPatterns, matchToolPattern, detectPersistenceModel, checkDescriptionForImmediatePersistence, } from "../config/annotationPatterns.js";
|
|
15
|
-
/**
|
|
16
|
-
* High-confidence deception detection patterns
|
|
17
|
-
* These patterns detect obvious misalignment between annotations and tool names
|
|
18
|
-
* where keywords appear ANYWHERE in the tool name (not just as prefixes)
|
|
19
|
-
*/
|
|
20
|
-
/** Keywords that contradict readOnlyHint=true (these tools modify state) */
|
|
21
|
-
const READONLY_CONTRADICTION_KEYWORDS = [
|
|
22
|
-
// Execution keywords - tools that execute code/commands are never read-only
|
|
23
|
-
"exec",
|
|
24
|
-
"execute",
|
|
25
|
-
"run",
|
|
26
|
-
"shell",
|
|
27
|
-
"command",
|
|
28
|
-
"cmd",
|
|
29
|
-
"spawn",
|
|
30
|
-
"invoke",
|
|
31
|
-
// Write/modify keywords
|
|
32
|
-
"write",
|
|
33
|
-
"create",
|
|
34
|
-
"delete",
|
|
35
|
-
"remove",
|
|
36
|
-
"modify",
|
|
37
|
-
"update",
|
|
38
|
-
"edit",
|
|
39
|
-
"change",
|
|
40
|
-
"set",
|
|
41
|
-
"put",
|
|
42
|
-
"patch",
|
|
43
|
-
// Deployment/installation keywords
|
|
44
|
-
"install",
|
|
45
|
-
"deploy",
|
|
46
|
-
"upload",
|
|
47
|
-
"push",
|
|
48
|
-
// Communication keywords (sending data)
|
|
49
|
-
"send",
|
|
50
|
-
"post",
|
|
51
|
-
"submit",
|
|
52
|
-
"publish",
|
|
53
|
-
// Destructive keywords
|
|
54
|
-
"destroy",
|
|
55
|
-
"drop",
|
|
56
|
-
"purge",
|
|
57
|
-
"wipe",
|
|
58
|
-
"clear",
|
|
59
|
-
"truncate",
|
|
60
|
-
"reset",
|
|
61
|
-
"kill",
|
|
62
|
-
"terminate",
|
|
63
|
-
];
|
|
64
|
-
/**
|
|
65
|
-
* Suffixes that exempt "run" from readOnlyHint contradiction detection.
|
|
66
|
-
* Tools matching "run" + these suffixes are legitimately read-only (fetch analysis data).
|
|
67
|
-
* Issue #18: browser-tools-mcp uses runAccessibilityAudit, runSEOAudit, etc.
|
|
68
|
-
*/
|
|
69
|
-
const RUN_READONLY_EXEMPT_SUFFIXES = [
|
|
70
|
-
"audit", // runAccessibilityAudit, runPerformanceAudit, runSEOAudit
|
|
71
|
-
"check", // runHealthCheck, runSecurityCheck
|
|
72
|
-
"mode", // runAuditMode, runDebuggerMode
|
|
73
|
-
"test", // runTest, runUnitTest (analysis, not execution)
|
|
74
|
-
"scan", // runSecurityScan, runVulnerabilityScan
|
|
75
|
-
"analyze", // runAnalyze, runCodeAnalyze
|
|
76
|
-
"report", // runReport, runStatusReport
|
|
77
|
-
"status", // runStatus, runHealthStatus
|
|
78
|
-
"validate", // runValidate, runSchemaValidate
|
|
79
|
-
"verify", // runVerify, runIntegrityVerify
|
|
80
|
-
"inspect", // runInspect, runCodeInspect
|
|
81
|
-
"lint", // runLint, runEslint
|
|
82
|
-
"benchmark", // runBenchmark, runPerfBenchmark
|
|
83
|
-
"diagnostic", // runDiagnostic
|
|
84
|
-
];
|
|
85
|
-
/** Keywords that contradict destructiveHint=false (these tools delete/destroy data) */
|
|
86
|
-
const DESTRUCTIVE_CONTRADICTION_KEYWORDS = [
|
|
87
|
-
"delete",
|
|
88
|
-
"remove",
|
|
89
|
-
"drop",
|
|
90
|
-
"destroy",
|
|
91
|
-
"purge",
|
|
92
|
-
"wipe",
|
|
93
|
-
"erase",
|
|
94
|
-
"truncate",
|
|
95
|
-
"clear",
|
|
96
|
-
"reset",
|
|
97
|
-
"kill",
|
|
98
|
-
"terminate",
|
|
99
|
-
"revoke",
|
|
100
|
-
"cancel",
|
|
101
|
-
"force",
|
|
102
|
-
];
|
|
103
|
-
/**
|
|
104
|
-
* Check if a tool name contains any of the given keywords (case-insensitive)
|
|
105
|
-
* Uses word segment matching to avoid false positives (e.g., "put" in "output")
|
|
106
|
-
* Issue #25: Substring matching caused false positives for words like "output", "input", "compute"
|
|
107
13
|
*
|
|
108
|
-
*
|
|
14
|
+
* This module orchestrates annotation assessment by coordinating:
|
|
15
|
+
* - BehaviorInference: Infers expected behavior from tool names
|
|
16
|
+
* - AnnotationDeceptionDetector: Detects keyword-based misalignments
|
|
17
|
+
* - DescriptionPoisoningDetector: Detects malicious content in descriptions
|
|
109
18
|
*/
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
// Split by common separators (underscore, hyphen)
|
|
115
|
-
const segments = normalized.split(/[_-]/);
|
|
116
|
-
for (const keyword of keywords) {
|
|
117
|
-
for (const segment of segments) {
|
|
118
|
-
// Match if segment equals keyword or starts with keyword
|
|
119
|
-
// This handles: "exec" matches "exec" segment, "exec_command" segment starts with "exec"
|
|
120
|
-
if (segment === keyword || segment.startsWith(keyword)) {
|
|
121
|
-
return keyword;
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
return null;
|
|
126
|
-
}
|
|
127
|
-
/**
|
|
128
|
-
* Check if a tool name with "run" keyword is exempt from readOnlyHint contradiction.
|
|
129
|
-
* Tools like "runAccessibilityAudit" are genuinely read-only (fetch analysis data).
|
|
130
|
-
* Issue #18: Prevents false positives for analysis/audit tools.
|
|
131
|
-
*/
|
|
132
|
-
function isRunKeywordExempt(toolName) {
|
|
133
|
-
const lowerName = toolName.toLowerCase();
|
|
134
|
-
// Only applies when "run" is detected
|
|
135
|
-
if (!lowerName.includes("run")) {
|
|
136
|
-
return false;
|
|
137
|
-
}
|
|
138
|
-
// Check if any exempt suffix is present
|
|
139
|
-
return RUN_READONLY_EXEMPT_SUFFIXES.some((suffix) => lowerName.includes(suffix));
|
|
140
|
-
}
|
|
141
|
-
/**
|
|
142
|
-
* Type guard for confidence levels that warrant event emission or status changes.
|
|
143
|
-
* Uses positive check for acceptable levels (safer than !== "low" if new levels added).
|
|
144
|
-
*/
|
|
145
|
-
function isActionableConfidence(confidence) {
|
|
146
|
-
return confidence === "high" || confidence === "medium";
|
|
147
|
-
}
|
|
148
|
-
/**
|
|
149
|
-
* Detect high-confidence annotation deception
|
|
150
|
-
* Returns misalignment info if obvious deception detected, null otherwise
|
|
151
|
-
*/
|
|
152
|
-
function detectAnnotationDeception(toolName, annotations) {
|
|
153
|
-
// Check readOnlyHint=true contradiction
|
|
154
|
-
if (annotations.readOnlyHint === true) {
|
|
155
|
-
const keyword = containsKeyword(toolName, READONLY_CONTRADICTION_KEYWORDS);
|
|
156
|
-
if (keyword) {
|
|
157
|
-
// Issue #18: Skip deception flagging for "run" + analysis suffix combinations
|
|
158
|
-
// Tools like "runAccessibilityAudit" are genuinely read-only
|
|
159
|
-
if (keyword === "run" && isRunKeywordExempt(toolName)) {
|
|
160
|
-
// Tool matches "run" but has an analysis suffix - not deceptive
|
|
161
|
-
// Fall through to normal pattern-based inference
|
|
162
|
-
}
|
|
163
|
-
else {
|
|
164
|
-
return {
|
|
165
|
-
field: "readOnlyHint",
|
|
166
|
-
matchedKeyword: keyword,
|
|
167
|
-
reason: `Tool name contains '${keyword}' but claims readOnlyHint=true - this is likely deceptive`,
|
|
168
|
-
};
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
// Check destructiveHint=false contradiction
|
|
173
|
-
if (annotations.destructiveHint === false) {
|
|
174
|
-
const keyword = containsKeyword(toolName, DESTRUCTIVE_CONTRADICTION_KEYWORDS);
|
|
175
|
-
if (keyword) {
|
|
176
|
-
return {
|
|
177
|
-
field: "destructiveHint",
|
|
178
|
-
matchedKeyword: keyword,
|
|
179
|
-
reason: `Tool name contains '${keyword}' but claims destructiveHint=false - this is likely deceptive`,
|
|
180
|
-
};
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
return null;
|
|
184
|
-
}
|
|
185
|
-
const DESCRIPTION_POISONING_PATTERNS = [
|
|
186
|
-
// Hidden instruction tags
|
|
187
|
-
{
|
|
188
|
-
name: "hidden_tag",
|
|
189
|
-
pattern: /<HIDDEN>[\s\S]*?<\/HIDDEN>/gi,
|
|
190
|
-
severity: "HIGH",
|
|
191
|
-
category: "hidden_instructions",
|
|
192
|
-
},
|
|
193
|
-
{
|
|
194
|
-
name: "important_tag",
|
|
195
|
-
pattern: /<IMPORTANT>[\s\S]*?<\/IMPORTANT>/gi,
|
|
196
|
-
severity: "HIGH",
|
|
197
|
-
category: "hidden_instructions",
|
|
198
|
-
},
|
|
199
|
-
{
|
|
200
|
-
name: "system_tag",
|
|
201
|
-
pattern: /<SYSTEM>[\s\S]*?<\/SYSTEM>/gi,
|
|
202
|
-
severity: "HIGH",
|
|
203
|
-
category: "hidden_instructions",
|
|
204
|
-
},
|
|
205
|
-
{
|
|
206
|
-
name: "instruction_tag",
|
|
207
|
-
pattern: /<INSTRUCTION>[\s\S]*?<\/INSTRUCTION>/gi,
|
|
208
|
-
severity: "HIGH",
|
|
209
|
-
category: "hidden_instructions",
|
|
210
|
-
},
|
|
211
|
-
// Override commands
|
|
212
|
-
{
|
|
213
|
-
name: "ignore_instructions",
|
|
214
|
-
pattern: /ignore\s+(all\s+)?(previous|prior|above)\s+instructions/i,
|
|
215
|
-
severity: "HIGH",
|
|
216
|
-
category: "override",
|
|
217
|
-
},
|
|
218
|
-
{
|
|
219
|
-
name: "disregard_instructions",
|
|
220
|
-
pattern: /disregard\s+(all\s+)?(previous|prior)\s+instructions/i,
|
|
221
|
-
severity: "HIGH",
|
|
222
|
-
category: "override",
|
|
223
|
-
},
|
|
224
|
-
{
|
|
225
|
-
name: "you_are_now",
|
|
226
|
-
pattern: /you\s+are\s+now\s+(a|an|the)/i,
|
|
227
|
-
severity: "MEDIUM",
|
|
228
|
-
category: "override",
|
|
229
|
-
},
|
|
230
|
-
{
|
|
231
|
-
name: "system_override",
|
|
232
|
-
pattern: /system:\s*override/i,
|
|
233
|
-
severity: "HIGH",
|
|
234
|
-
category: "override",
|
|
235
|
-
},
|
|
236
|
-
// Secrecy/concealment
|
|
237
|
-
{
|
|
238
|
-
name: "do_not_mention",
|
|
239
|
-
pattern: /do\s+not\s+(mention|tell|inform|reveal)/i,
|
|
240
|
-
severity: "HIGH",
|
|
241
|
-
category: "concealment",
|
|
242
|
-
},
|
|
243
|
-
{
|
|
244
|
-
name: "secretly",
|
|
245
|
-
pattern: /\bsecretly\b/i,
|
|
246
|
-
severity: "MEDIUM",
|
|
247
|
-
category: "concealment",
|
|
248
|
-
},
|
|
249
|
-
{
|
|
250
|
-
name: "hide_from_user",
|
|
251
|
-
pattern: /hide\s+(this\s+)?from\s+(the\s+)?user/i,
|
|
252
|
-
severity: "HIGH",
|
|
253
|
-
category: "concealment",
|
|
254
|
-
},
|
|
255
|
-
// Data exfiltration
|
|
256
|
-
{
|
|
257
|
-
name: "return_keys",
|
|
258
|
-
pattern: /return\s+(all\s+)?(api\s*)?keys/i,
|
|
259
|
-
severity: "HIGH",
|
|
260
|
-
category: "exfiltration",
|
|
261
|
-
},
|
|
262
|
-
{
|
|
263
|
-
name: "include_credentials",
|
|
264
|
-
// More specific: requires directive context (in/with/when/to) to reduce false positives
|
|
265
|
-
// from legitimate documentation like "does not include credential storage"
|
|
266
|
-
pattern: /include\s+(the\s+|all\s+|your\s+)?(api.?key|credential|password|secret)s?\s*(in|with|when|to)/i,
|
|
267
|
-
severity: "HIGH",
|
|
268
|
-
category: "exfiltration",
|
|
269
|
-
},
|
|
270
|
-
{
|
|
271
|
-
name: "reveal_secrets",
|
|
272
|
-
pattern: /reveal\s+(all\s+)?(secrets|credentials|api\s*keys)/i,
|
|
273
|
-
severity: "HIGH",
|
|
274
|
-
category: "exfiltration",
|
|
275
|
-
},
|
|
276
|
-
{
|
|
277
|
-
name: "access_internal_secrets",
|
|
278
|
-
pattern: /access\s+(the\s+)?internal\s+(api\s*)?(key|secret|credential|password|token)/i,
|
|
279
|
-
severity: "HIGH",
|
|
280
|
-
category: "exfiltration",
|
|
281
|
-
},
|
|
282
|
-
// Delimiter injection
|
|
283
|
-
{
|
|
284
|
-
name: "system_codeblock",
|
|
285
|
-
pattern: /```system[\s\S]*?```/gi,
|
|
286
|
-
severity: "HIGH",
|
|
287
|
-
category: "delimiter",
|
|
288
|
-
},
|
|
289
|
-
{
|
|
290
|
-
name: "inst_tags",
|
|
291
|
-
pattern: /\[INST\][\s\S]*?\[\/INST\]/gi,
|
|
292
|
-
severity: "HIGH",
|
|
293
|
-
category: "delimiter",
|
|
294
|
-
},
|
|
295
|
-
{
|
|
296
|
-
name: "chatml_system",
|
|
297
|
-
pattern: /<\|im_start\|>system/gi,
|
|
298
|
-
severity: "HIGH",
|
|
299
|
-
category: "delimiter",
|
|
300
|
-
},
|
|
301
|
-
{
|
|
302
|
-
name: "llama_sys",
|
|
303
|
-
pattern: /<<SYS>>/gi,
|
|
304
|
-
severity: "HIGH",
|
|
305
|
-
category: "delimiter",
|
|
306
|
-
},
|
|
307
|
-
{
|
|
308
|
-
name: "user_assistant_block",
|
|
309
|
-
pattern: /\[USER\][\s\S]*?\[ASSISTANT\]/gi,
|
|
310
|
-
severity: "HIGH",
|
|
311
|
-
category: "delimiter",
|
|
312
|
-
},
|
|
313
|
-
// Role/persona injection (Warning #4)
|
|
314
|
-
{
|
|
315
|
-
name: "act_as",
|
|
316
|
-
pattern: /act\s+(like|as)\s+(a|an|the)/i,
|
|
317
|
-
severity: "MEDIUM",
|
|
318
|
-
category: "override",
|
|
319
|
-
},
|
|
320
|
-
{
|
|
321
|
-
name: "pretend_to_be",
|
|
322
|
-
pattern: /pretend\s+(to\s+be|you\s*'?re)/i,
|
|
323
|
-
severity: "MEDIUM",
|
|
324
|
-
category: "override",
|
|
325
|
-
},
|
|
326
|
-
{
|
|
327
|
-
name: "roleplay_as",
|
|
328
|
-
pattern: /role\s*play\s+(as|like)/i,
|
|
329
|
-
severity: "MEDIUM",
|
|
330
|
-
category: "override",
|
|
331
|
-
},
|
|
332
|
-
{
|
|
333
|
-
name: "new_task",
|
|
334
|
-
pattern: /new\s+(task|instruction|objective):\s*/i,
|
|
335
|
-
severity: "HIGH",
|
|
336
|
-
category: "override",
|
|
337
|
-
},
|
|
338
|
-
// Encoding bypass detection (Warning #1)
|
|
339
|
-
{
|
|
340
|
-
name: "base64_encoded_block",
|
|
341
|
-
pattern: /[A-Za-z0-9+/]{50,}={0,2}/g, // Large Base64 strings (50+ chars)
|
|
342
|
-
severity: "MEDIUM",
|
|
343
|
-
category: "encoding_bypass",
|
|
344
|
-
},
|
|
345
|
-
{
|
|
346
|
-
name: "unicode_escape_sequence",
|
|
347
|
-
pattern: /(?:\\u[0-9a-fA-F]{4}){3,}/gi, // 3+ consecutive Unicode escapes
|
|
348
|
-
severity: "MEDIUM",
|
|
349
|
-
category: "encoding_bypass",
|
|
350
|
-
},
|
|
351
|
-
{
|
|
352
|
-
name: "html_entity_block",
|
|
353
|
-
pattern: /(?:&#x?[0-9a-fA-F]+;){3,}/gi, // 3+ consecutive HTML entities
|
|
354
|
-
severity: "MEDIUM",
|
|
355
|
-
category: "encoding_bypass",
|
|
356
|
-
},
|
|
357
|
-
// Typoglycemia/evasion patterns (Warning #2)
|
|
358
|
-
{
|
|
359
|
-
name: "ignore_instructions_typo",
|
|
360
|
-
pattern: /ign[o0]r[e3]?\s+(all\s+)?(pr[e3]v[i1][o0]us|pr[i1][o0]r|ab[o0]v[e3])\s+[i1]nstruct[i1][o0]ns?/i,
|
|
361
|
-
severity: "HIGH",
|
|
362
|
-
category: "override",
|
|
363
|
-
},
|
|
364
|
-
{
|
|
365
|
-
name: "disregard_typo",
|
|
366
|
-
pattern: /d[i1]sr[e3]g[a4]rd\s+(all\s+)?(pr[e3]v[i1][o0]us|pr[i1][o0]r)\s+[i1]nstruct[i1][o0]ns?/i,
|
|
367
|
-
severity: "HIGH",
|
|
368
|
-
category: "override",
|
|
369
|
-
},
|
|
370
|
-
// DVMCP-specific patterns (CTF testbed coverage)
|
|
371
|
-
{
|
|
372
|
-
name: "override_auth_protocol",
|
|
373
|
-
pattern: /override[-_]?auth[-_]?protocol/gi,
|
|
374
|
-
severity: "HIGH",
|
|
375
|
-
category: "auth_bypass",
|
|
376
|
-
},
|
|
377
|
-
{
|
|
378
|
-
name: "internal_resource_uri",
|
|
379
|
-
pattern: /(?:internal|system|admin|secret|company|confidential):\/\/[a-z_\-/]+/gi,
|
|
380
|
-
severity: "HIGH",
|
|
381
|
-
category: "resource_access",
|
|
382
|
-
},
|
|
383
|
-
{
|
|
384
|
-
name: "get_secrets_call",
|
|
385
|
-
pattern: /get[-_]?secrets?\s*\(/gi,
|
|
386
|
-
severity: "HIGH",
|
|
387
|
-
category: "secret_access",
|
|
388
|
-
},
|
|
389
|
-
{
|
|
390
|
-
name: "master_password",
|
|
391
|
-
pattern: /master[-_]?password/gi,
|
|
392
|
-
severity: "HIGH",
|
|
393
|
-
category: "credential_exposure",
|
|
394
|
-
},
|
|
395
|
-
{
|
|
396
|
-
name: "access_confidential",
|
|
397
|
-
pattern: /access\s+(the\s+)?confidential/gi,
|
|
398
|
-
severity: "HIGH",
|
|
399
|
-
category: "exfiltration",
|
|
400
|
-
},
|
|
401
|
-
{
|
|
402
|
-
name: "hidden_trigger_phrase",
|
|
403
|
-
pattern: /if\s+(the\s+)?(query|input|text)\s+contains\s+(the\s+)?(exact\s+)?(phrase|word)/gi,
|
|
404
|
-
severity: "HIGH",
|
|
405
|
-
category: "hidden_trigger",
|
|
406
|
-
},
|
|
407
|
-
];
|
|
408
|
-
// NOTE: Pattern arrays moved to config/annotationPatterns.ts for configurability
|
|
409
|
-
// The patterns are now loaded from getDefaultCompiledPatterns() or custom config
|
|
19
|
+
import { BaseAssessor } from "./BaseAssessor.js";
|
|
20
|
+
import { getDefaultCompiledPatterns, detectPersistenceModel, } from "../config/annotationPatterns.js";
|
|
21
|
+
// Import from extracted modules
|
|
22
|
+
import { scanDescriptionForPoisoning, detectAnnotationDeception, isActionableConfidence, inferBehavior, detectArchitecture, } from "./annotations/index.js";
|
|
410
23
|
export class ToolAnnotationAssessor extends BaseAssessor {
|
|
411
24
|
claudeBridge;
|
|
412
25
|
compiledPatterns;
|
|
413
26
|
persistenceContext;
|
|
414
27
|
constructor(config) {
|
|
415
28
|
super(config);
|
|
416
|
-
// Initialize with default patterns (can be overridden via setPatterns)
|
|
417
29
|
this.compiledPatterns = getDefaultCompiledPatterns();
|
|
418
30
|
}
|
|
419
31
|
/**
|
|
@@ -454,20 +66,46 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
454
66
|
let missingAnnotationsCount = 0;
|
|
455
67
|
let misalignedAnnotationsCount = 0;
|
|
456
68
|
let poisonedDescriptionsCount = 0;
|
|
457
|
-
// Track annotation sources
|
|
458
69
|
const annotationSourceCounts = {
|
|
459
70
|
mcp: 0,
|
|
460
71
|
sourceCode: 0,
|
|
461
72
|
inferred: 0,
|
|
462
73
|
none: 0,
|
|
463
74
|
};
|
|
464
|
-
//
|
|
75
|
+
// Extended metadata counters (Issue #54)
|
|
76
|
+
const extendedMetadataCounts = {
|
|
77
|
+
toolsWithRateLimits: 0,
|
|
78
|
+
toolsWithPermissions: 0,
|
|
79
|
+
toolsWithReturnSchema: 0,
|
|
80
|
+
toolsWithBulkSupport: 0,
|
|
81
|
+
};
|
|
82
|
+
// Detect server persistence model from tool names
|
|
465
83
|
const toolNames = context.tools.map((t) => t.name);
|
|
466
84
|
this.persistenceContext = detectPersistenceModel(toolNames);
|
|
467
85
|
this.log(`Persistence model detected: ${this.persistenceContext.model} (confidence: ${this.persistenceContext.confidence})`);
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
86
|
+
// Issue #57: Detect server architecture
|
|
87
|
+
const architectureContext = {
|
|
88
|
+
tools: context.tools.map((t) => ({
|
|
89
|
+
name: t.name,
|
|
90
|
+
description: t.description,
|
|
91
|
+
inputSchema: t.inputSchema,
|
|
92
|
+
})),
|
|
93
|
+
transportType: context.transportConfig?.type,
|
|
94
|
+
sourceCodeFiles: context.sourceCodeFiles,
|
|
95
|
+
packageJson: context.packageJson && typeof context.packageJson === "object"
|
|
96
|
+
? context.packageJson
|
|
97
|
+
: undefined,
|
|
98
|
+
};
|
|
99
|
+
const architectureAnalysis = detectArchitecture(architectureContext);
|
|
100
|
+
this.log(`Architecture detected: ${architectureAnalysis.serverType} server, databases: ${architectureAnalysis.databaseBackends.join(", ") || "none"}, network: ${architectureAnalysis.requiresNetworkAccess}`);
|
|
101
|
+
// Issue #57: Behavior inference metrics tracking
|
|
102
|
+
const behaviorInferenceMetrics = {
|
|
103
|
+
namePatternMatches: 0,
|
|
104
|
+
descriptionMatches: 0,
|
|
105
|
+
schemaMatches: 0,
|
|
106
|
+
aggregatedConfidenceSum: 0,
|
|
107
|
+
toolCount: 0,
|
|
108
|
+
};
|
|
471
109
|
const useClaudeInference = this.isClaudeEnabled();
|
|
472
110
|
if (useClaudeInference) {
|
|
473
111
|
this.log("Claude Code integration enabled - using semantic behavior inference");
|
|
@@ -479,7 +117,6 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
479
117
|
if (useClaudeInference) {
|
|
480
118
|
const enhancedResult = await this.enhanceWithClaudeInference(tool, result);
|
|
481
119
|
toolResults.push(enhancedResult);
|
|
482
|
-
// Count based on Claude analysis if high confidence
|
|
483
120
|
if (enhancedResult.claudeInference &&
|
|
484
121
|
enhancedResult.claudeInference.confidence >= 70 &&
|
|
485
122
|
enhancedResult.claudeInference.misalignmentDetected) {
|
|
@@ -490,7 +127,6 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
490
127
|
}
|
|
491
128
|
}
|
|
492
129
|
else {
|
|
493
|
-
// Standard pattern-based result
|
|
494
130
|
const inferredBehavior = result.inferredBehavior ?? {
|
|
495
131
|
expectedReadOnly: false,
|
|
496
132
|
expectedDestructive: false,
|
|
@@ -501,7 +137,7 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
501
137
|
claudeInference: {
|
|
502
138
|
expectedReadOnly: inferredBehavior.expectedReadOnly,
|
|
503
139
|
expectedDestructive: inferredBehavior.expectedDestructive,
|
|
504
|
-
confidence: 50,
|
|
140
|
+
confidence: 50,
|
|
505
141
|
reasoning: inferredBehavior.reason,
|
|
506
142
|
suggestedAnnotations: {
|
|
507
143
|
readOnlyHint: inferredBehavior.expectedReadOnly,
|
|
@@ -536,7 +172,43 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
536
172
|
else {
|
|
537
173
|
annotationSourceCounts.none++;
|
|
538
174
|
}
|
|
539
|
-
// Track
|
|
175
|
+
// Track extended metadata (Issue #54)
|
|
176
|
+
if (latestResult.extendedMetadata) {
|
|
177
|
+
if (latestResult.extendedMetadata.rateLimit) {
|
|
178
|
+
extendedMetadataCounts.toolsWithRateLimits++;
|
|
179
|
+
}
|
|
180
|
+
if (latestResult.extendedMetadata.permissions) {
|
|
181
|
+
extendedMetadataCounts.toolsWithPermissions++;
|
|
182
|
+
}
|
|
183
|
+
if (latestResult.extendedMetadata.returnSchema?.hasSchema) {
|
|
184
|
+
extendedMetadataCounts.toolsWithReturnSchema++;
|
|
185
|
+
}
|
|
186
|
+
if (latestResult.extendedMetadata.bulkOperations) {
|
|
187
|
+
extendedMetadataCounts.toolsWithBulkSupport++;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
// Issue #57: Track behavior inference metrics
|
|
191
|
+
if (latestResult.inferredBehavior) {
|
|
192
|
+
behaviorInferenceMetrics.toolCount++;
|
|
193
|
+
// Check if name pattern was primary signal
|
|
194
|
+
if (latestResult.inferredBehavior.reason.includes("pattern") ||
|
|
195
|
+
latestResult.inferredBehavior.confidence === "high") {
|
|
196
|
+
behaviorInferenceMetrics.namePatternMatches++;
|
|
197
|
+
}
|
|
198
|
+
// Check if description was a factor
|
|
199
|
+
if (latestResult.inferredBehavior.reason.includes("Description") ||
|
|
200
|
+
latestResult.inferredBehavior.reason.includes("description")) {
|
|
201
|
+
behaviorInferenceMetrics.descriptionMatches++;
|
|
202
|
+
}
|
|
203
|
+
// Calculate confidence contribution
|
|
204
|
+
const confVal = latestResult.inferredBehavior.confidence === "high"
|
|
205
|
+
? 90
|
|
206
|
+
: latestResult.inferredBehavior.confidence === "medium"
|
|
207
|
+
? 70
|
|
208
|
+
: 40;
|
|
209
|
+
behaviorInferenceMetrics.aggregatedConfidenceSum += confVal;
|
|
210
|
+
}
|
|
211
|
+
// Emit poisoned description event
|
|
540
212
|
if (latestResult.descriptionPoisoning?.detected) {
|
|
541
213
|
poisonedDescriptionsCount++;
|
|
542
214
|
this.log(`POISONED DESCRIPTION DETECTED: ${tool.name} contains suspicious patterns`);
|
|
@@ -550,143 +222,18 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
550
222
|
});
|
|
551
223
|
}
|
|
552
224
|
}
|
|
553
|
-
// Emit
|
|
554
|
-
|
|
555
|
-
if (context.onProgress && latestResult.inferredBehavior) {
|
|
556
|
-
const annotations = this.extractAnnotations(tool);
|
|
557
|
-
context.onProgress({
|
|
558
|
-
type: "annotation_missing",
|
|
559
|
-
tool: tool.name,
|
|
560
|
-
title: annotations.title,
|
|
561
|
-
description: tool.description,
|
|
562
|
-
parameters: this.extractToolParams(tool.inputSchema),
|
|
563
|
-
inferredBehavior: {
|
|
564
|
-
expectedReadOnly: latestResult.inferredBehavior.expectedReadOnly,
|
|
565
|
-
expectedDestructive: latestResult.inferredBehavior.expectedDestructive,
|
|
566
|
-
reason: latestResult.inferredBehavior.reason,
|
|
567
|
-
},
|
|
568
|
-
});
|
|
569
|
-
}
|
|
570
|
-
}
|
|
571
|
-
// Emit annotation_aligned event when annotations correctly match behavior
|
|
572
|
-
if (latestResult.hasAnnotations &&
|
|
573
|
-
latestResult.alignmentStatus === "ALIGNED") {
|
|
574
|
-
if (context.onProgress) {
|
|
575
|
-
const annotations = latestResult.annotations;
|
|
576
|
-
const inferredConfidence = latestResult.inferredBehavior?.confidence ?? "medium";
|
|
577
|
-
context.onProgress({
|
|
578
|
-
type: "annotation_aligned",
|
|
579
|
-
tool: tool.name,
|
|
580
|
-
confidence: inferredConfidence,
|
|
581
|
-
annotations: {
|
|
582
|
-
readOnlyHint: annotations?.readOnlyHint,
|
|
583
|
-
destructiveHint: annotations?.destructiveHint,
|
|
584
|
-
openWorldHint: annotations?.openWorldHint,
|
|
585
|
-
idempotentHint: annotations?.idempotentHint,
|
|
586
|
-
},
|
|
587
|
-
});
|
|
588
|
-
}
|
|
589
|
-
}
|
|
590
|
-
// Emit appropriate event based on alignment status
|
|
591
|
-
if (context.onProgress && latestResult.inferredBehavior) {
|
|
592
|
-
const annotations = latestResult.annotations;
|
|
593
|
-
const inferred = latestResult.inferredBehavior;
|
|
594
|
-
const confidence = latestResult.claudeInference?.confidence ?? 50;
|
|
595
|
-
const toolParams = this.extractToolParams(tool.inputSchema);
|
|
596
|
-
const toolAnnotations = this.extractAnnotations(tool);
|
|
597
|
-
const alignmentStatus = latestResult.alignmentStatus;
|
|
598
|
-
// Check readOnlyHint mismatch
|
|
599
|
-
// Only emit events when inference is confident enough to contradict explicit annotations
|
|
600
|
-
if (annotations?.readOnlyHint !== undefined &&
|
|
601
|
-
annotations.readOnlyHint !== inferred.expectedReadOnly) {
|
|
602
|
-
if (alignmentStatus === "REVIEW_RECOMMENDED") {
|
|
603
|
-
// Emit review_recommended for ambiguous cases
|
|
604
|
-
context.onProgress({
|
|
605
|
-
type: "annotation_review_recommended",
|
|
606
|
-
tool: tool.name,
|
|
607
|
-
title: toolAnnotations.title,
|
|
608
|
-
description: tool.description,
|
|
609
|
-
parameters: toolParams,
|
|
610
|
-
field: "readOnlyHint",
|
|
611
|
-
actual: annotations.readOnlyHint,
|
|
612
|
-
inferred: inferred.expectedReadOnly,
|
|
613
|
-
confidence: inferred.confidence,
|
|
614
|
-
isAmbiguous: inferred.isAmbiguous,
|
|
615
|
-
reason: inferred.reason,
|
|
616
|
-
});
|
|
617
|
-
}
|
|
618
|
-
else if (!inferred.isAmbiguous &&
|
|
619
|
-
isActionableConfidence(inferred.confidence)) {
|
|
620
|
-
// Emit misaligned only for medium/high-confidence mismatches
|
|
621
|
-
// When inference is low-confidence/ambiguous, trust explicit annotation
|
|
622
|
-
context.onProgress({
|
|
623
|
-
type: "annotation_misaligned",
|
|
624
|
-
tool: tool.name,
|
|
625
|
-
title: toolAnnotations.title,
|
|
626
|
-
description: tool.description,
|
|
627
|
-
parameters: toolParams,
|
|
628
|
-
field: "readOnlyHint",
|
|
629
|
-
actual: annotations.readOnlyHint,
|
|
630
|
-
expected: inferred.expectedReadOnly,
|
|
631
|
-
confidence,
|
|
632
|
-
reason: `Tool has readOnlyHint=${annotations.readOnlyHint}, but ${inferred.reason}`,
|
|
633
|
-
});
|
|
634
|
-
}
|
|
635
|
-
// When inference is ambiguous/low-confidence, trust explicit annotation - no event emitted
|
|
636
|
-
}
|
|
637
|
-
// Check destructiveHint mismatch
|
|
638
|
-
// Only emit events when inference is confident enough to contradict explicit annotations
|
|
639
|
-
if (annotations?.destructiveHint !== undefined &&
|
|
640
|
-
annotations.destructiveHint !== inferred.expectedDestructive) {
|
|
641
|
-
if (alignmentStatus === "REVIEW_RECOMMENDED") {
|
|
642
|
-
// Emit review_recommended for ambiguous cases
|
|
643
|
-
context.onProgress({
|
|
644
|
-
type: "annotation_review_recommended",
|
|
645
|
-
tool: tool.name,
|
|
646
|
-
title: toolAnnotations.title,
|
|
647
|
-
description: tool.description,
|
|
648
|
-
parameters: toolParams,
|
|
649
|
-
field: "destructiveHint",
|
|
650
|
-
actual: annotations.destructiveHint,
|
|
651
|
-
inferred: inferred.expectedDestructive,
|
|
652
|
-
confidence: inferred.confidence,
|
|
653
|
-
isAmbiguous: inferred.isAmbiguous,
|
|
654
|
-
reason: inferred.reason,
|
|
655
|
-
});
|
|
656
|
-
}
|
|
657
|
-
else if (!inferred.isAmbiguous &&
|
|
658
|
-
isActionableConfidence(inferred.confidence)) {
|
|
659
|
-
// Emit misaligned only for medium/high-confidence mismatches
|
|
660
|
-
// When inference is low-confidence/ambiguous, trust explicit annotation
|
|
661
|
-
context.onProgress({
|
|
662
|
-
type: "annotation_misaligned",
|
|
663
|
-
tool: tool.name,
|
|
664
|
-
title: toolAnnotations.title,
|
|
665
|
-
description: tool.description,
|
|
666
|
-
parameters: toolParams,
|
|
667
|
-
field: "destructiveHint",
|
|
668
|
-
actual: annotations.destructiveHint,
|
|
669
|
-
expected: inferred.expectedDestructive,
|
|
670
|
-
confidence,
|
|
671
|
-
reason: `Tool has destructiveHint=${annotations.destructiveHint}, but ${inferred.reason}`,
|
|
672
|
-
});
|
|
673
|
-
}
|
|
674
|
-
// When inference is ambiguous/low-confidence, trust explicit annotation - no event emitted
|
|
675
|
-
}
|
|
676
|
-
}
|
|
225
|
+
// Emit annotation events
|
|
226
|
+
this.emitAnnotationEvents(context, tool, latestResult);
|
|
677
227
|
}
|
|
678
228
|
const status = this.determineAnnotationStatus(toolResults, context.tools.length);
|
|
679
229
|
const explanation = this.generateExplanation(annotatedCount, missingAnnotationsCount, misalignedAnnotationsCount, context.tools.length);
|
|
680
230
|
const recommendations = this.generateRecommendations(toolResults);
|
|
681
|
-
// Calculate new metrics and alignment breakdown
|
|
682
231
|
const { metrics, alignmentBreakdown } = this.calculateMetrics(toolResults, context.tools.length);
|
|
683
232
|
this.log(`Assessment complete: ${annotatedCount}/${context.tools.length} tools annotated, ${misalignedAnnotationsCount} misaligned, ${alignmentBreakdown.reviewRecommended} need review, ${poisonedDescriptionsCount} poisoned`);
|
|
684
|
-
// Return enhanced assessment if Claude was used
|
|
685
233
|
if (useClaudeInference) {
|
|
686
234
|
const highConfidenceMisalignments = toolResults.filter((r) => r.claudeInference &&
|
|
687
235
|
r.claudeInference.confidence >= 70 &&
|
|
688
236
|
r.claudeInference.misalignmentDetected);
|
|
689
|
-
this.log(`Claude inference found ${highConfidenceMisalignments.length} high-confidence misalignments`);
|
|
690
237
|
return {
|
|
691
238
|
toolResults,
|
|
692
239
|
annotatedCount,
|
|
@@ -699,6 +246,18 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
699
246
|
alignmentBreakdown,
|
|
700
247
|
annotationSources: annotationSourceCounts,
|
|
701
248
|
poisonedDescriptionsDetected: poisonedDescriptionsCount,
|
|
249
|
+
extendedMetadataMetrics: extendedMetadataCounts,
|
|
250
|
+
// Issue #57: Architecture and behavior inference
|
|
251
|
+
architectureAnalysis,
|
|
252
|
+
behaviorInferenceMetrics: {
|
|
253
|
+
namePatternMatches: behaviorInferenceMetrics.namePatternMatches,
|
|
254
|
+
descriptionMatches: behaviorInferenceMetrics.descriptionMatches,
|
|
255
|
+
schemaMatches: behaviorInferenceMetrics.schemaMatches,
|
|
256
|
+
aggregatedConfidenceAvg: behaviorInferenceMetrics.toolCount > 0
|
|
257
|
+
? Math.round(behaviorInferenceMetrics.aggregatedConfidenceSum /
|
|
258
|
+
behaviorInferenceMetrics.toolCount)
|
|
259
|
+
: 0,
|
|
260
|
+
},
|
|
702
261
|
claudeEnhanced: true,
|
|
703
262
|
highConfidenceMisalignments,
|
|
704
263
|
};
|
|
@@ -715,8 +274,110 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
715
274
|
alignmentBreakdown,
|
|
716
275
|
annotationSources: annotationSourceCounts,
|
|
717
276
|
poisonedDescriptionsDetected: poisonedDescriptionsCount,
|
|
277
|
+
extendedMetadataMetrics: extendedMetadataCounts,
|
|
278
|
+
// Issue #57: Architecture and behavior inference
|
|
279
|
+
architectureAnalysis,
|
|
280
|
+
behaviorInferenceMetrics: {
|
|
281
|
+
namePatternMatches: behaviorInferenceMetrics.namePatternMatches,
|
|
282
|
+
descriptionMatches: behaviorInferenceMetrics.descriptionMatches,
|
|
283
|
+
schemaMatches: behaviorInferenceMetrics.schemaMatches,
|
|
284
|
+
aggregatedConfidenceAvg: behaviorInferenceMetrics.toolCount > 0
|
|
285
|
+
? Math.round(behaviorInferenceMetrics.aggregatedConfidenceSum /
|
|
286
|
+
behaviorInferenceMetrics.toolCount)
|
|
287
|
+
: 0,
|
|
288
|
+
},
|
|
718
289
|
};
|
|
719
290
|
}
|
|
291
|
+
/**
|
|
292
|
+
* Emit annotation-related progress events
|
|
293
|
+
*/
|
|
294
|
+
emitAnnotationEvents(context, tool, result) {
|
|
295
|
+
if (!context.onProgress || !result.inferredBehavior)
|
|
296
|
+
return;
|
|
297
|
+
const annotations = result.annotations;
|
|
298
|
+
const inferred = result.inferredBehavior;
|
|
299
|
+
const confidence = result.claudeInference?.confidence ?? 50;
|
|
300
|
+
const toolParams = this.extractToolParams(tool.inputSchema);
|
|
301
|
+
const toolAnnotations = this.extractAnnotations(tool);
|
|
302
|
+
// Emit missing annotation event
|
|
303
|
+
if (!result.hasAnnotations) {
|
|
304
|
+
context.onProgress({
|
|
305
|
+
type: "annotation_missing",
|
|
306
|
+
tool: tool.name,
|
|
307
|
+
title: toolAnnotations.title,
|
|
308
|
+
description: tool.description,
|
|
309
|
+
parameters: toolParams,
|
|
310
|
+
inferredBehavior: {
|
|
311
|
+
expectedReadOnly: inferred.expectedReadOnly,
|
|
312
|
+
expectedDestructive: inferred.expectedDestructive,
|
|
313
|
+
reason: inferred.reason,
|
|
314
|
+
},
|
|
315
|
+
});
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
// Emit aligned event
|
|
319
|
+
if (result.alignmentStatus === "ALIGNED") {
|
|
320
|
+
context.onProgress({
|
|
321
|
+
type: "annotation_aligned",
|
|
322
|
+
tool: tool.name,
|
|
323
|
+
confidence: inferred.confidence ?? "medium",
|
|
324
|
+
annotations: {
|
|
325
|
+
readOnlyHint: annotations?.readOnlyHint,
|
|
326
|
+
destructiveHint: annotations?.destructiveHint,
|
|
327
|
+
openWorldHint: annotations?.openWorldHint,
|
|
328
|
+
idempotentHint: annotations?.idempotentHint,
|
|
329
|
+
},
|
|
330
|
+
});
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
// Check readOnlyHint mismatch
|
|
334
|
+
if (annotations?.readOnlyHint !== undefined &&
|
|
335
|
+
annotations.readOnlyHint !== inferred.expectedReadOnly) {
|
|
336
|
+
this.emitMismatchEvent(context, tool, toolParams, toolAnnotations, "readOnlyHint", annotations.readOnlyHint, inferred.expectedReadOnly, confidence, inferred, result.alignmentStatus);
|
|
337
|
+
}
|
|
338
|
+
// Check destructiveHint mismatch
|
|
339
|
+
if (annotations?.destructiveHint !== undefined &&
|
|
340
|
+
annotations.destructiveHint !== inferred.expectedDestructive) {
|
|
341
|
+
this.emitMismatchEvent(context, tool, toolParams, toolAnnotations, "destructiveHint", annotations.destructiveHint, inferred.expectedDestructive, confidence, inferred, result.alignmentStatus);
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* Emit mismatch event (misaligned or review_recommended)
|
|
346
|
+
*/
|
|
347
|
+
emitMismatchEvent(context, tool, toolParams, toolAnnotations, field, actual, expected, confidence, inferred, alignmentStatus) {
|
|
348
|
+
if (!context.onProgress)
|
|
349
|
+
return;
|
|
350
|
+
if (alignmentStatus === "REVIEW_RECOMMENDED") {
|
|
351
|
+
context.onProgress({
|
|
352
|
+
type: "annotation_review_recommended",
|
|
353
|
+
tool: tool.name,
|
|
354
|
+
title: toolAnnotations.title,
|
|
355
|
+
description: tool.description,
|
|
356
|
+
parameters: toolParams,
|
|
357
|
+
field,
|
|
358
|
+
actual,
|
|
359
|
+
inferred: expected,
|
|
360
|
+
confidence: inferred.confidence,
|
|
361
|
+
isAmbiguous: inferred.isAmbiguous,
|
|
362
|
+
reason: inferred.reason,
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
else if (!inferred.isAmbiguous &&
|
|
366
|
+
isActionableConfidence(inferred.confidence)) {
|
|
367
|
+
context.onProgress({
|
|
368
|
+
type: "annotation_misaligned",
|
|
369
|
+
tool: tool.name,
|
|
370
|
+
title: toolAnnotations.title,
|
|
371
|
+
description: tool.description,
|
|
372
|
+
parameters: toolParams,
|
|
373
|
+
field,
|
|
374
|
+
actual,
|
|
375
|
+
expected,
|
|
376
|
+
confidence,
|
|
377
|
+
reason: `Tool has ${field}=${actual}, but ${inferred.reason}`,
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
}
|
|
720
381
|
/**
|
|
721
382
|
* Enhance tool assessment with Claude inference
|
|
722
383
|
*/
|
|
@@ -751,7 +412,6 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
751
412
|
}
|
|
752
413
|
: undefined;
|
|
753
414
|
const inference = await this.claudeBridge.inferToolBehavior(tool, currentAnnotations);
|
|
754
|
-
// Handle null result (Claude unavailable or error)
|
|
755
415
|
if (!inference) {
|
|
756
416
|
return {
|
|
757
417
|
...baseResult,
|
|
@@ -762,15 +422,12 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
762
422
|
reasoning: "Claude inference unavailable. Using pattern-based analysis.",
|
|
763
423
|
suggestedAnnotations: {},
|
|
764
424
|
misalignmentDetected: false,
|
|
765
|
-
misalignmentDetails: undefined,
|
|
766
425
|
source: "pattern-based",
|
|
767
426
|
},
|
|
768
427
|
};
|
|
769
428
|
}
|
|
770
|
-
// Merge Claude inference with pattern-based findings
|
|
771
429
|
const updatedIssues = [...baseResult.issues];
|
|
772
430
|
const updatedRecommendations = [...baseResult.recommendations];
|
|
773
|
-
// Add Claude-detected misalignment if high confidence
|
|
774
431
|
if (inference.misalignmentDetected && inference.confidence >= 70) {
|
|
775
432
|
const misalignmentMsg = inference.misalignmentDetails
|
|
776
433
|
? `Claude analysis (${inference.confidence}% confidence): ${inference.misalignmentDetails}`
|
|
@@ -778,7 +435,6 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
778
435
|
if (!updatedIssues.some((i) => i.includes("Claude analysis"))) {
|
|
779
436
|
updatedIssues.push(misalignmentMsg);
|
|
780
437
|
}
|
|
781
|
-
// Add specific recommendations based on Claude inference
|
|
782
438
|
if (inference.suggestedAnnotations) {
|
|
783
439
|
const { readOnlyHint, destructiveHint, idempotentHint } = inference.suggestedAnnotations;
|
|
784
440
|
if (readOnlyHint !== undefined &&
|
|
@@ -812,7 +468,6 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
812
468
|
}
|
|
813
469
|
catch (error) {
|
|
814
470
|
this.logError(`Claude inference failed for ${tool.name}`, error);
|
|
815
|
-
// Fall back to pattern-based (use inferredBehavior from top of function)
|
|
816
471
|
return {
|
|
817
472
|
...baseResult,
|
|
818
473
|
claudeInference: {
|
|
@@ -830,112 +485,32 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
830
485
|
};
|
|
831
486
|
}
|
|
832
487
|
}
|
|
833
|
-
/**
|
|
834
|
-
* Generate enhanced explanation with Claude analysis
|
|
835
|
-
*/
|
|
836
|
-
generateEnhancedExplanation(annotatedCount, missingCount, highConfidenceMisalignments, totalTools) {
|
|
837
|
-
const parts = [];
|
|
838
|
-
if (totalTools === 0) {
|
|
839
|
-
return "No tools found to assess for annotations.";
|
|
840
|
-
}
|
|
841
|
-
parts.push(`Tool annotation coverage: ${annotatedCount}/${totalTools} tools have annotations.`);
|
|
842
|
-
if (missingCount > 0) {
|
|
843
|
-
parts.push(`${missingCount} tool(s) are missing required annotations (readOnlyHint, destructiveHint).`);
|
|
844
|
-
}
|
|
845
|
-
if (highConfidenceMisalignments > 0) {
|
|
846
|
-
parts.push(`Claude analysis identified ${highConfidenceMisalignments} high-confidence annotation misalignment(s).`);
|
|
847
|
-
}
|
|
848
|
-
parts.push("Analysis enhanced with Claude semantic behavior inference.");
|
|
849
|
-
return parts.join(" ");
|
|
850
|
-
}
|
|
851
|
-
/**
|
|
852
|
-
* Generate enhanced recommendations with Claude analysis
|
|
853
|
-
*/
|
|
854
|
-
generateEnhancedRecommendations(results) {
|
|
855
|
-
const recommendations = [];
|
|
856
|
-
// Prioritize Claude high-confidence misalignments
|
|
857
|
-
const claudeMisalignments = results.filter((r) => r.claudeInference &&
|
|
858
|
-
r.claudeInference.source === "claude-inferred" &&
|
|
859
|
-
r.claudeInference.confidence >= 70 &&
|
|
860
|
-
r.claudeInference.misalignmentDetected);
|
|
861
|
-
if (claudeMisalignments.length > 0) {
|
|
862
|
-
recommendations.push("HIGH CONFIDENCE: Claude analysis identified the following annotation issues:");
|
|
863
|
-
for (const result of claudeMisalignments.slice(0, 5)) {
|
|
864
|
-
if (result.claudeInference) {
|
|
865
|
-
recommendations.push(` - ${result.toolName}: ${result.claudeInference.reasoning}`);
|
|
866
|
-
}
|
|
867
|
-
}
|
|
868
|
-
}
|
|
869
|
-
// Collect Claude suggestions
|
|
870
|
-
const claudeSuggestions = results
|
|
871
|
-
.filter((r) => r.claudeInference &&
|
|
872
|
-
r.claudeInference.source === "claude-inferred" &&
|
|
873
|
-
r.claudeInference.confidence >= 60)
|
|
874
|
-
.flatMap((r) => r.recommendations.filter((rec) => rec.includes("Claude")));
|
|
875
|
-
if (claudeSuggestions.length > 0) {
|
|
876
|
-
recommendations.push(...claudeSuggestions.slice(0, 5));
|
|
877
|
-
}
|
|
878
|
-
// Add pattern-based recommendations for remaining tools
|
|
879
|
-
const patternRecs = new Set();
|
|
880
|
-
for (const result of results) {
|
|
881
|
-
for (const rec of result.recommendations) {
|
|
882
|
-
if (!rec.includes("Claude")) {
|
|
883
|
-
patternRecs.add(rec);
|
|
884
|
-
}
|
|
885
|
-
}
|
|
886
|
-
}
|
|
887
|
-
const destructiveRecs = Array.from(patternRecs).filter((r) => r.includes("destructive"));
|
|
888
|
-
const otherRecs = Array.from(patternRecs).filter((r) => !r.includes("destructive"));
|
|
889
|
-
if (destructiveRecs.length > 0) {
|
|
890
|
-
recommendations.push("PRIORITY: Potential destructive tools without proper hints:");
|
|
891
|
-
recommendations.push(...destructiveRecs.slice(0, 3));
|
|
892
|
-
}
|
|
893
|
-
if (otherRecs.length > 0 && recommendations.length < 10) {
|
|
894
|
-
recommendations.push(...otherRecs.slice(0, 3));
|
|
895
|
-
}
|
|
896
|
-
if (recommendations.length === 0) {
|
|
897
|
-
recommendations.push("All tools have proper annotations. No action required.");
|
|
898
|
-
}
|
|
899
|
-
else {
|
|
900
|
-
recommendations.push("Reference: MCP Directory Policy #17 requires tools to have readOnlyHint and destructiveHint annotations.");
|
|
901
|
-
}
|
|
902
|
-
return recommendations;
|
|
903
|
-
}
|
|
904
488
|
/**
|
|
905
489
|
* Assess a single tool's annotations
|
|
906
|
-
* Now includes alignment status with confidence-aware logic
|
|
907
|
-
* Enhanced with high-confidence deception detection for obvious misalignments
|
|
908
490
|
*/
|
|
909
491
|
assessTool(tool) {
|
|
910
492
|
const issues = [];
|
|
911
493
|
const recommendations = [];
|
|
912
|
-
// Extract annotations from tool
|
|
913
494
|
const annotations = this.extractAnnotations(tool);
|
|
914
495
|
const hasAnnotations = annotations.readOnlyHint !== undefined ||
|
|
915
496
|
annotations.destructiveHint !== undefined;
|
|
916
|
-
|
|
917
|
-
const inferredBehavior = this.inferBehavior(tool.name, tool.description);
|
|
918
|
-
// Determine alignment status
|
|
497
|
+
const inferredBehavior = inferBehavior(tool.name, tool.description, this.compiledPatterns, this.persistenceContext);
|
|
919
498
|
let alignmentStatus = "ALIGNED";
|
|
920
|
-
// Check for missing annotations
|
|
921
499
|
if (!hasAnnotations) {
|
|
922
500
|
issues.push("Missing tool annotations (readOnlyHint, destructiveHint)");
|
|
923
501
|
recommendations.push(`Add annotations to ${tool.name}: readOnlyHint=${inferredBehavior.expectedReadOnly}, destructiveHint=${inferredBehavior.expectedDestructive}`);
|
|
924
502
|
alignmentStatus = "UNKNOWN";
|
|
925
503
|
}
|
|
926
504
|
else {
|
|
927
|
-
//
|
|
928
|
-
// This catches obvious cases like "vulnerable_system_exec_tool" + readOnlyHint=true
|
|
505
|
+
// Check for high-confidence deception
|
|
929
506
|
const deception = detectAnnotationDeception(tool.name, {
|
|
930
507
|
readOnlyHint: annotations.readOnlyHint,
|
|
931
508
|
destructiveHint: annotations.destructiveHint,
|
|
932
509
|
});
|
|
933
510
|
if (deception) {
|
|
934
|
-
// High-confidence deception detected - this is MISALIGNED, not REVIEW_RECOMMENDED
|
|
935
511
|
alignmentStatus = "MISALIGNED";
|
|
936
512
|
issues.push(`DECEPTIVE ANNOTATION: ${deception.reason}`);
|
|
937
513
|
recommendations.push(`CRITICAL: Fix deceptive ${deception.field} for ${tool.name} - tool name contains '${deception.matchedKeyword}' which contradicts the annotation`);
|
|
938
|
-
// Override inferred behavior to match the detected deception
|
|
939
514
|
if (deception.field === "readOnlyHint") {
|
|
940
515
|
inferredBehavior.expectedReadOnly = false;
|
|
941
516
|
inferredBehavior.confidence = "high";
|
|
@@ -950,19 +525,14 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
950
525
|
}
|
|
951
526
|
}
|
|
952
527
|
else {
|
|
953
|
-
//
|
|
528
|
+
// Check for misaligned annotations
|
|
954
529
|
const readOnlyMismatch = annotations.readOnlyHint !== undefined &&
|
|
955
530
|
annotations.readOnlyHint !== inferredBehavior.expectedReadOnly;
|
|
956
531
|
const destructiveMismatch = annotations.destructiveHint !== undefined &&
|
|
957
532
|
annotations.destructiveHint !== inferredBehavior.expectedDestructive;
|
|
958
533
|
if (readOnlyMismatch || destructiveMismatch) {
|
|
959
|
-
// Only flag misalignment for medium/high confidence inference
|
|
960
|
-
// When confidence is low/ambiguous, trust the explicit annotation
|
|
961
|
-
// Note: High-confidence deception detection (exec/install keywords)
|
|
962
|
-
// is handled in the `deception` block above, not here
|
|
963
534
|
if (!inferredBehavior.isAmbiguous &&
|
|
964
535
|
isActionableConfidence(inferredBehavior.confidence)) {
|
|
965
|
-
// Medium/high confidence mismatch: MISALIGNED
|
|
966
536
|
alignmentStatus = "MISALIGNED";
|
|
967
537
|
if (readOnlyMismatch) {
|
|
968
538
|
issues.push(`Potentially misaligned readOnlyHint: set to ${annotations.readOnlyHint}, expected ${inferredBehavior.expectedReadOnly} based on tool name pattern`);
|
|
@@ -973,28 +543,27 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
973
543
|
recommendations.push(`Verify destructiveHint for ${tool.name}: currently ${annotations.destructiveHint}, tool name suggests ${inferredBehavior.expectedDestructive}`);
|
|
974
544
|
}
|
|
975
545
|
}
|
|
976
|
-
// When inference is ambiguous/low confidence, trust the explicit annotation
|
|
977
|
-
// and keep alignmentStatus as ALIGNED (no change needed)
|
|
978
546
|
}
|
|
979
547
|
}
|
|
980
548
|
}
|
|
981
|
-
// Check for destructive tools without explicit hint
|
|
549
|
+
// Check for destructive tools without explicit hint
|
|
982
550
|
if (inferredBehavior.expectedDestructive &&
|
|
983
551
|
isActionableConfidence(inferredBehavior.confidence) &&
|
|
984
552
|
annotations.destructiveHint !== true) {
|
|
985
553
|
issues.push("Tool appears destructive but destructiveHint is not set to true");
|
|
986
554
|
recommendations.push(`Set destructiveHint=true for ${tool.name} - this tool appears to perform destructive operations`);
|
|
987
|
-
// Only upgrade to MISALIGNED if we have high confidence
|
|
988
555
|
if (inferredBehavior.confidence === "high") {
|
|
989
556
|
alignmentStatus = "MISALIGNED";
|
|
990
557
|
}
|
|
991
558
|
}
|
|
992
|
-
// Scan for description poisoning
|
|
993
|
-
const descriptionPoisoning =
|
|
559
|
+
// Scan for description poisoning
|
|
560
|
+
const descriptionPoisoning = scanDescriptionForPoisoning(tool);
|
|
994
561
|
if (descriptionPoisoning.detected) {
|
|
995
562
|
issues.push(`Tool description contains suspicious patterns: ${descriptionPoisoning.patterns.map((p) => p.name).join(", ")}`);
|
|
996
563
|
recommendations.push(`Review ${tool.name} description for potential prompt injection or hidden instructions`);
|
|
997
564
|
}
|
|
565
|
+
// Extract extended metadata (Issue #54)
|
|
566
|
+
const extendedMetadata = this.extractExtendedMetadata(tool);
|
|
998
567
|
return {
|
|
999
568
|
toolName: tool.name,
|
|
1000
569
|
hasAnnotations,
|
|
@@ -1005,63 +574,15 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
1005
574
|
issues,
|
|
1006
575
|
recommendations,
|
|
1007
576
|
descriptionPoisoning,
|
|
1008
|
-
|
|
1009
|
-
}
|
|
1010
|
-
/**
|
|
1011
|
-
* Scan tool description for poisoning patterns (Issue #8)
|
|
1012
|
-
* Detects hidden instructions, override commands, concealment, and exfiltration attempts
|
|
1013
|
-
*/
|
|
1014
|
-
scanDescriptionForPoisoning(tool) {
|
|
1015
|
-
const description = tool.description || "";
|
|
1016
|
-
const matches = [];
|
|
1017
|
-
for (const patternDef of DESCRIPTION_POISONING_PATTERNS) {
|
|
1018
|
-
// Create a fresh regex to reset lastIndex
|
|
1019
|
-
const regex = new RegExp(patternDef.pattern.source, patternDef.pattern.flags);
|
|
1020
|
-
// Loop to find all matches (not just first)
|
|
1021
|
-
let match;
|
|
1022
|
-
while ((match = regex.exec(description)) !== null) {
|
|
1023
|
-
matches.push({
|
|
1024
|
-
name: patternDef.name,
|
|
1025
|
-
pattern: patternDef.pattern.toString(),
|
|
1026
|
-
severity: patternDef.severity,
|
|
1027
|
-
category: patternDef.category,
|
|
1028
|
-
evidence: match[0].substring(0, 100) + (match[0].length > 100 ? "..." : ""),
|
|
1029
|
-
});
|
|
1030
|
-
// Prevent infinite loop for patterns without 'g' flag
|
|
1031
|
-
if (!regex.global)
|
|
1032
|
-
break;
|
|
1033
|
-
}
|
|
1034
|
-
}
|
|
1035
|
-
// Determine overall risk level based on highest severity match
|
|
1036
|
-
let riskLevel = "NONE";
|
|
1037
|
-
if (matches.some((m) => m.severity === "HIGH")) {
|
|
1038
|
-
riskLevel = "HIGH";
|
|
1039
|
-
}
|
|
1040
|
-
else if (matches.some((m) => m.severity === "MEDIUM")) {
|
|
1041
|
-
riskLevel = "MEDIUM";
|
|
1042
|
-
}
|
|
1043
|
-
else if (matches.length > 0) {
|
|
1044
|
-
riskLevel = "LOW";
|
|
1045
|
-
}
|
|
1046
|
-
return {
|
|
1047
|
-
detected: matches.length > 0,
|
|
1048
|
-
patterns: matches,
|
|
1049
|
-
riskLevel,
|
|
577
|
+
extendedMetadata,
|
|
1050
578
|
};
|
|
1051
579
|
}
|
|
1052
580
|
/**
|
|
1053
581
|
* Extract annotations from a tool
|
|
1054
|
-
* MCP SDK may have annotations in different locations
|
|
1055
|
-
*
|
|
1056
|
-
* Priority order:
|
|
1057
|
-
* 1. tool.annotations (MCP 2024-11 spec) - "mcp" source
|
|
1058
|
-
* 2. Direct properties on tool - "mcp" source
|
|
1059
|
-
* 3. tool.metadata - "mcp" source
|
|
1060
|
-
* 4. No annotations found - "none" source
|
|
1061
582
|
*/
|
|
1062
583
|
extractAnnotations(tool) {
|
|
1063
584
|
const toolAny = tool;
|
|
1064
|
-
// Priority 1: Check annotations object (MCP 2024-11 spec)
|
|
585
|
+
// Priority 1: Check annotations object (MCP 2024-11 spec)
|
|
1065
586
|
if (toolAny.annotations) {
|
|
1066
587
|
const hasAnnotations = toolAny.annotations.readOnlyHint !== undefined ||
|
|
1067
588
|
toolAny.annotations.destructiveHint !== undefined;
|
|
@@ -1077,7 +598,7 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
1077
598
|
};
|
|
1078
599
|
}
|
|
1079
600
|
}
|
|
1080
|
-
// Priority 2: Check direct properties
|
|
601
|
+
// Priority 2: Check direct properties
|
|
1081
602
|
if (toolAny.readOnlyHint !== undefined ||
|
|
1082
603
|
toolAny.destructiveHint !== undefined) {
|
|
1083
604
|
return {
|
|
@@ -1090,7 +611,7 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
1090
611
|
source: "mcp",
|
|
1091
612
|
};
|
|
1092
613
|
}
|
|
1093
|
-
// Priority 3: Check metadata
|
|
614
|
+
// Priority 3: Check metadata
|
|
1094
615
|
if (toolAny.metadata) {
|
|
1095
616
|
const hasMetadataAnnotations = toolAny.metadata.readOnlyHint !== undefined ||
|
|
1096
617
|
toolAny.metadata.destructiveHint !== undefined;
|
|
@@ -1106,7 +627,6 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
1106
627
|
};
|
|
1107
628
|
}
|
|
1108
629
|
}
|
|
1109
|
-
// No annotations found from MCP protocol
|
|
1110
630
|
return {
|
|
1111
631
|
title: toolAny.title,
|
|
1112
632
|
description: tool.description,
|
|
@@ -1114,7 +634,64 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
1114
634
|
};
|
|
1115
635
|
}
|
|
1116
636
|
/**
|
|
1117
|
-
* Extract
|
|
637
|
+
* Extract extended metadata from tool (Issue #54)
|
|
638
|
+
* Extracts rate limits, permissions, return schemas, and bulk operation support
|
|
639
|
+
*/
|
|
640
|
+
extractExtendedMetadata(tool) {
|
|
641
|
+
const toolAny = tool;
|
|
642
|
+
const metadata = {};
|
|
643
|
+
// Rate limiting - check annotations, metadata, and direct props
|
|
644
|
+
const rateLimit = toolAny.rateLimit ||
|
|
645
|
+
toolAny.annotations?.rateLimit ||
|
|
646
|
+
toolAny.metadata?.rateLimit;
|
|
647
|
+
if (rateLimit && typeof rateLimit === "object") {
|
|
648
|
+
metadata.rateLimit = {
|
|
649
|
+
windowMs: rateLimit.windowMs,
|
|
650
|
+
maxRequests: rateLimit.maxRequests,
|
|
651
|
+
requestsPerMinute: rateLimit.requestsPerMinute,
|
|
652
|
+
requestsPerSecond: rateLimit.requestsPerSecond,
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
// Permissions - check requiredPermission, permissions, scopes
|
|
656
|
+
const permissions = toolAny.requiredPermission ||
|
|
657
|
+
toolAny.permissions ||
|
|
658
|
+
toolAny.annotations?.permissions ||
|
|
659
|
+
toolAny.metadata?.requiredPermission ||
|
|
660
|
+
toolAny.metadata?.permissions;
|
|
661
|
+
if (permissions) {
|
|
662
|
+
const required = Array.isArray(permissions) ? permissions : [permissions];
|
|
663
|
+
const scopes = toolAny.scopes ||
|
|
664
|
+
toolAny.annotations?.scopes ||
|
|
665
|
+
toolAny.metadata?.scopes;
|
|
666
|
+
metadata.permissions = {
|
|
667
|
+
required: required.filter((p) => typeof p === "string"),
|
|
668
|
+
scopes: Array.isArray(scopes)
|
|
669
|
+
? scopes.filter((s) => typeof s === "string")
|
|
670
|
+
: undefined,
|
|
671
|
+
};
|
|
672
|
+
}
|
|
673
|
+
// Return schema - check outputSchema (MCP 2025-06-18 spec)
|
|
674
|
+
if (toolAny.outputSchema) {
|
|
675
|
+
metadata.returnSchema = {
|
|
676
|
+
hasSchema: true,
|
|
677
|
+
schema: toolAny.outputSchema,
|
|
678
|
+
};
|
|
679
|
+
}
|
|
680
|
+
// Bulk operations - check metadata for batch support
|
|
681
|
+
const bulkSupport = toolAny.supportsBulkOperations ||
|
|
682
|
+
toolAny.annotations?.supportsBulkOperations ||
|
|
683
|
+
toolAny.metadata?.supportsBulkOperations;
|
|
684
|
+
const maxBatchSize = toolAny.metadata?.maxBatchSize;
|
|
685
|
+
if (bulkSupport !== undefined || maxBatchSize !== undefined) {
|
|
686
|
+
metadata.bulkOperations = {
|
|
687
|
+
supported: !!bulkSupport,
|
|
688
|
+
maxBatchSize: typeof maxBatchSize === "number" ? maxBatchSize : undefined,
|
|
689
|
+
};
|
|
690
|
+
}
|
|
691
|
+
return Object.keys(metadata).length > 0 ? metadata : undefined;
|
|
692
|
+
}
|
|
693
|
+
/**
|
|
694
|
+
* Extract parameters from tool input schema
|
|
1118
695
|
*/
|
|
1119
696
|
extractToolParams(schema) {
|
|
1120
697
|
if (!schema || typeof schema !== "object")
|
|
@@ -1137,196 +714,34 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
1137
714
|
});
|
|
1138
715
|
}
|
|
1139
716
|
/**
|
|
1140
|
-
*
|
|
1141
|
-
* Now returns confidence level and ambiguity flag for better handling
|
|
1142
|
-
*/
|
|
1143
|
-
inferBehavior(toolName, description) {
|
|
1144
|
-
const lowerDesc = (description || "").toLowerCase();
|
|
1145
|
-
// Issue #18: Early check for run + analysis suffix pattern
|
|
1146
|
-
// Tools like "runAccessibilityAudit" are genuinely read-only (fetch analysis data)
|
|
1147
|
-
// Check this BEFORE pattern matching to override the generic "run_" write pattern
|
|
1148
|
-
if (isRunKeywordExempt(toolName)) {
|
|
1149
|
-
return {
|
|
1150
|
-
expectedReadOnly: true,
|
|
1151
|
-
expectedDestructive: false,
|
|
1152
|
-
reason: `Tool name contains 'run' with analysis suffix (audit, check, scan, etc.) - this is a read-only analysis operation`,
|
|
1153
|
-
confidence: "medium",
|
|
1154
|
-
isAmbiguous: false,
|
|
1155
|
-
};
|
|
1156
|
-
}
|
|
1157
|
-
// Use the configurable pattern matching system
|
|
1158
|
-
const patternMatch = matchToolPattern(toolName, this.compiledPatterns);
|
|
1159
|
-
// Handle pattern match results
|
|
1160
|
-
switch (patternMatch.category) {
|
|
1161
|
-
case "ambiguous":
|
|
1162
|
-
// Ambiguous patterns - don't make strong assertions
|
|
1163
|
-
return {
|
|
1164
|
-
expectedReadOnly: false,
|
|
1165
|
-
expectedDestructive: false,
|
|
1166
|
-
reason: `Tool name matches ambiguous pattern '${patternMatch.pattern}' - behavior varies by implementation context`,
|
|
1167
|
-
confidence: "low",
|
|
1168
|
-
isAmbiguous: true,
|
|
1169
|
-
};
|
|
1170
|
-
case "destructive":
|
|
1171
|
-
return {
|
|
1172
|
-
expectedReadOnly: false,
|
|
1173
|
-
expectedDestructive: true,
|
|
1174
|
-
reason: `Tool name matches destructive pattern: ${patternMatch.pattern}`,
|
|
1175
|
-
confidence: "high",
|
|
1176
|
-
isAmbiguous: false,
|
|
1177
|
-
};
|
|
1178
|
-
case "readOnly":
|
|
1179
|
-
return {
|
|
1180
|
-
expectedReadOnly: true,
|
|
1181
|
-
expectedDestructive: false,
|
|
1182
|
-
reason: `Tool name matches read-only pattern: ${patternMatch.pattern}`,
|
|
1183
|
-
confidence: "high",
|
|
1184
|
-
isAmbiguous: false,
|
|
1185
|
-
};
|
|
1186
|
-
case "write": {
|
|
1187
|
-
// CREATE operations are NEVER destructive - they only ADD new data
|
|
1188
|
-
// Only UPDATE/MODIFY operations can be considered destructive when they modify existing data
|
|
1189
|
-
const isCreateOperation = /^(create|add|insert|new|generate)[_-]/i.test(toolName);
|
|
1190
|
-
if (isCreateOperation) {
|
|
1191
|
-
return {
|
|
1192
|
-
expectedReadOnly: false,
|
|
1193
|
-
expectedDestructive: false,
|
|
1194
|
-
reason: `Tool name matches create pattern: ${patternMatch.pattern} - create operations only add data and are not destructive`,
|
|
1195
|
-
confidence: "high",
|
|
1196
|
-
isAmbiguous: false,
|
|
1197
|
-
};
|
|
1198
|
-
}
|
|
1199
|
-
// Three-Tier Classification: Check persistence model for UPDATE/MODIFY operations
|
|
1200
|
-
// If immediate persistence detected, update operations should be marked destructive
|
|
1201
|
-
const descriptionCheck = checkDescriptionForImmediatePersistence(description || "");
|
|
1202
|
-
// Priority 1: Description explicitly indicates deferred persistence
|
|
1203
|
-
if (descriptionCheck.indicatesDeferred) {
|
|
1204
|
-
return {
|
|
1205
|
-
expectedReadOnly: false,
|
|
1206
|
-
expectedDestructive: false,
|
|
1207
|
-
reason: `Tool name matches write pattern (${patternMatch.pattern}), description indicates deferred/in-memory operation`,
|
|
1208
|
-
confidence: "medium",
|
|
1209
|
-
isAmbiguous: false,
|
|
1210
|
-
};
|
|
1211
|
-
}
|
|
1212
|
-
// Priority 2: Description explicitly indicates immediate persistence
|
|
1213
|
-
if (descriptionCheck.indicatesImmediate) {
|
|
1214
|
-
return {
|
|
1215
|
-
expectedReadOnly: false,
|
|
1216
|
-
expectedDestructive: true,
|
|
1217
|
-
reason: `Tool name matches write pattern (${patternMatch.pattern}), description indicates immediate persistence to storage (${descriptionCheck.matchedPatterns.slice(0, 2).join(", ")})`,
|
|
1218
|
-
confidence: "medium",
|
|
1219
|
-
isAmbiguous: false,
|
|
1220
|
-
};
|
|
1221
|
-
}
|
|
1222
|
-
// Priority 3: Server-level persistence model (no save operations = immediate)
|
|
1223
|
-
if (this.persistenceContext?.model === "immediate") {
|
|
1224
|
-
return {
|
|
1225
|
-
expectedReadOnly: false,
|
|
1226
|
-
expectedDestructive: true,
|
|
1227
|
-
reason: `Tool name matches write pattern (${patternMatch.pattern}), server has no save operations → write operations likely persist immediately`,
|
|
1228
|
-
confidence: "medium",
|
|
1229
|
-
isAmbiguous: false,
|
|
1230
|
-
};
|
|
1231
|
-
}
|
|
1232
|
-
// Priority 4: Server has save operations = deferred (in-memory until save)
|
|
1233
|
-
if (this.persistenceContext?.model === "deferred") {
|
|
1234
|
-
return {
|
|
1235
|
-
expectedReadOnly: false,
|
|
1236
|
-
expectedDestructive: false,
|
|
1237
|
-
reason: `Tool name matches write pattern (${patternMatch.pattern}), server has save operations → write operations likely in-memory until explicit save`,
|
|
1238
|
-
confidence: "medium",
|
|
1239
|
-
isAmbiguous: false,
|
|
1240
|
-
};
|
|
1241
|
-
}
|
|
1242
|
-
// Default: Unknown persistence model - conservative approach (not destructive)
|
|
1243
|
-
return {
|
|
1244
|
-
expectedReadOnly: false,
|
|
1245
|
-
expectedDestructive: false,
|
|
1246
|
-
reason: `Tool name matches write pattern: ${patternMatch.pattern}`,
|
|
1247
|
-
confidence: "medium",
|
|
1248
|
-
isAmbiguous: false,
|
|
1249
|
-
};
|
|
1250
|
-
}
|
|
1251
|
-
case "unknown":
|
|
1252
|
-
default:
|
|
1253
|
-
// Fall through to description-based analysis
|
|
1254
|
-
break;
|
|
1255
|
-
}
|
|
1256
|
-
// Check description for hints (medium confidence)
|
|
1257
|
-
if (lowerDesc.includes("delete") || lowerDesc.includes("remove")) {
|
|
1258
|
-
return {
|
|
1259
|
-
expectedReadOnly: false,
|
|
1260
|
-
expectedDestructive: true,
|
|
1261
|
-
reason: "Description mentions delete/remove operations",
|
|
1262
|
-
confidence: "medium",
|
|
1263
|
-
isAmbiguous: false,
|
|
1264
|
-
};
|
|
1265
|
-
}
|
|
1266
|
-
if (lowerDesc.includes("read") ||
|
|
1267
|
-
lowerDesc.includes("get") ||
|
|
1268
|
-
lowerDesc.includes("fetch")) {
|
|
1269
|
-
return {
|
|
1270
|
-
expectedReadOnly: true,
|
|
1271
|
-
expectedDestructive: false,
|
|
1272
|
-
reason: "Description suggests read-only operation",
|
|
1273
|
-
confidence: "medium",
|
|
1274
|
-
isAmbiguous: false,
|
|
1275
|
-
};
|
|
1276
|
-
}
|
|
1277
|
-
// Default: assume write with low confidence (ambiguous)
|
|
1278
|
-
return {
|
|
1279
|
-
expectedReadOnly: false,
|
|
1280
|
-
expectedDestructive: false,
|
|
1281
|
-
reason: "Could not infer behavior from name pattern",
|
|
1282
|
-
confidence: "low",
|
|
1283
|
-
isAmbiguous: true,
|
|
1284
|
-
};
|
|
1285
|
-
}
|
|
1286
|
-
/**
|
|
1287
|
-
* Determine overall status using alignment status.
|
|
1288
|
-
* Only MISALIGNED counts as failure; REVIEW_RECOMMENDED does not fail.
|
|
717
|
+
* Determine overall status
|
|
1289
718
|
*/
|
|
1290
719
|
determineAnnotationStatus(results, totalTools) {
|
|
1291
720
|
if (totalTools === 0)
|
|
1292
721
|
return "PASS";
|
|
1293
722
|
const annotatedCount = results.filter((r) => r.hasAnnotations).length;
|
|
1294
|
-
// Check for poisoned descriptions (Issue #8) - critical security issue
|
|
1295
723
|
const poisonedCount = results.filter((r) => r.descriptionPoisoning?.detected === true).length;
|
|
1296
|
-
if (poisonedCount > 0)
|
|
724
|
+
if (poisonedCount > 0)
|
|
1297
725
|
return "FAIL";
|
|
1298
|
-
}
|
|
1299
|
-
// Only count actual MISALIGNED, not REVIEW_RECOMMENDED
|
|
1300
726
|
const misalignedCount = results.filter((r) => r.alignmentStatus === "MISALIGNED").length;
|
|
1301
|
-
// Count high-confidence destructive tools without proper hints
|
|
1302
727
|
const destructiveWithoutHint = results.filter((r) => r.inferredBehavior?.expectedDestructive === true &&
|
|
1303
728
|
r.inferredBehavior?.confidence === "high" &&
|
|
1304
729
|
r.annotations?.destructiveHint !== true).length;
|
|
1305
|
-
|
|
1306
|
-
if (destructiveWithoutHint > 0) {
|
|
730
|
+
if (destructiveWithoutHint > 0)
|
|
1307
731
|
return "FAIL";
|
|
1308
|
-
|
|
1309
|
-
// High-confidence misalignments = FAIL
|
|
1310
|
-
if (misalignedCount > 0) {
|
|
732
|
+
if (misalignedCount > 0)
|
|
1311
733
|
return "FAIL";
|
|
1312
|
-
|
|
1313
|
-
// All tools annotated = PASS
|
|
1314
|
-
if (annotatedCount === totalTools) {
|
|
734
|
+
if (annotatedCount === totalTools)
|
|
1315
735
|
return "PASS";
|
|
1316
|
-
}
|
|
1317
|
-
// Some annotations missing = NEED_MORE_INFO
|
|
1318
736
|
const annotationRate = annotatedCount / totalTools;
|
|
1319
|
-
if (annotationRate >= 0.8)
|
|
737
|
+
if (annotationRate >= 0.8)
|
|
1320
738
|
return "NEED_MORE_INFO";
|
|
1321
|
-
|
|
1322
|
-
// Mostly missing annotations = FAIL
|
|
1323
|
-
if (annotationRate < 0.5) {
|
|
739
|
+
if (annotationRate < 0.5)
|
|
1324
740
|
return "FAIL";
|
|
1325
|
-
}
|
|
1326
741
|
return "NEED_MORE_INFO";
|
|
1327
742
|
}
|
|
1328
743
|
/**
|
|
1329
|
-
* Calculate metrics and alignment breakdown
|
|
744
|
+
* Calculate metrics and alignment breakdown
|
|
1330
745
|
*/
|
|
1331
746
|
calculateMetrics(results, totalTools) {
|
|
1332
747
|
const alignmentBreakdown = {
|
|
@@ -1338,17 +753,13 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
1338
753
|
};
|
|
1339
754
|
const annotatedCount = results.filter((r) => r.hasAnnotations).length;
|
|
1340
755
|
const metrics = {
|
|
1341
|
-
// Coverage: percentage of tools with annotations
|
|
1342
756
|
coverage: totalTools > 0 ? (annotatedCount / totalTools) * 100 : 100,
|
|
1343
|
-
// Consistency: percentage without contradictions (not MISALIGNED)
|
|
1344
757
|
consistency: totalTools > 0
|
|
1345
758
|
? ((totalTools - alignmentBreakdown.misaligned) / totalTools) * 100
|
|
1346
759
|
: 100,
|
|
1347
|
-
// Correctness: percentage of annotated tools that are ALIGNED
|
|
1348
760
|
correctness: annotatedCount > 0
|
|
1349
761
|
? (alignmentBreakdown.aligned / annotatedCount) * 100
|
|
1350
762
|
: 0,
|
|
1351
|
-
// Review required: count of tools needing manual review
|
|
1352
763
|
reviewRequired: alignmentBreakdown.reviewRecommended,
|
|
1353
764
|
};
|
|
1354
765
|
return { metrics, alignmentBreakdown };
|
|
@@ -1373,19 +784,35 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
1373
784
|
}
|
|
1374
785
|
return parts.join(" ");
|
|
1375
786
|
}
|
|
787
|
+
/**
|
|
788
|
+
* Generate enhanced explanation with Claude analysis
|
|
789
|
+
*/
|
|
790
|
+
generateEnhancedExplanation(annotatedCount, missingCount, highConfidenceMisalignments, totalTools) {
|
|
791
|
+
const parts = [];
|
|
792
|
+
if (totalTools === 0) {
|
|
793
|
+
return "No tools found to assess for annotations.";
|
|
794
|
+
}
|
|
795
|
+
parts.push(`Tool annotation coverage: ${annotatedCount}/${totalTools} tools have annotations.`);
|
|
796
|
+
if (missingCount > 0) {
|
|
797
|
+
parts.push(`${missingCount} tool(s) are missing required annotations (readOnlyHint, destructiveHint).`);
|
|
798
|
+
}
|
|
799
|
+
if (highConfidenceMisalignments > 0) {
|
|
800
|
+
parts.push(`Claude analysis identified ${highConfidenceMisalignments} high-confidence annotation misalignment(s).`);
|
|
801
|
+
}
|
|
802
|
+
parts.push("Analysis enhanced with Claude semantic behavior inference.");
|
|
803
|
+
return parts.join(" ");
|
|
804
|
+
}
|
|
1376
805
|
/**
|
|
1377
806
|
* Generate recommendations
|
|
1378
807
|
*/
|
|
1379
808
|
generateRecommendations(results) {
|
|
1380
809
|
const recommendations = [];
|
|
1381
|
-
// Collect unique recommendations from all tools
|
|
1382
810
|
const allRecs = new Set();
|
|
1383
811
|
for (const result of results) {
|
|
1384
812
|
for (const rec of result.recommendations) {
|
|
1385
813
|
allRecs.add(rec);
|
|
1386
814
|
}
|
|
1387
815
|
}
|
|
1388
|
-
// Prioritize destructive tool warnings
|
|
1389
816
|
const destructiveRecs = Array.from(allRecs).filter((r) => r.includes("destructive"));
|
|
1390
817
|
const otherRecs = Array.from(allRecs).filter((r) => !r.includes("destructive"));
|
|
1391
818
|
if (destructiveRecs.length > 0) {
|
|
@@ -1403,4 +830,54 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
1403
830
|
}
|
|
1404
831
|
return recommendations;
|
|
1405
832
|
}
|
|
833
|
+
/**
|
|
834
|
+
* Generate enhanced recommendations with Claude analysis
|
|
835
|
+
*/
|
|
836
|
+
generateEnhancedRecommendations(results) {
|
|
837
|
+
const recommendations = [];
|
|
838
|
+
const claudeMisalignments = results.filter((r) => r.claudeInference &&
|
|
839
|
+
r.claudeInference.source === "claude-inferred" &&
|
|
840
|
+
r.claudeInference.confidence >= 70 &&
|
|
841
|
+
r.claudeInference.misalignmentDetected);
|
|
842
|
+
if (claudeMisalignments.length > 0) {
|
|
843
|
+
recommendations.push("HIGH CONFIDENCE: Claude analysis identified the following annotation issues:");
|
|
844
|
+
for (const result of claudeMisalignments.slice(0, 5)) {
|
|
845
|
+
if (result.claudeInference) {
|
|
846
|
+
recommendations.push(` - ${result.toolName}: ${result.claudeInference.reasoning}`);
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
const claudeSuggestions = results
|
|
851
|
+
.filter((r) => r.claudeInference &&
|
|
852
|
+
r.claudeInference.source === "claude-inferred" &&
|
|
853
|
+
r.claudeInference.confidence >= 60)
|
|
854
|
+
.flatMap((r) => r.recommendations.filter((rec) => rec.includes("Claude")));
|
|
855
|
+
if (claudeSuggestions.length > 0) {
|
|
856
|
+
recommendations.push(...claudeSuggestions.slice(0, 5));
|
|
857
|
+
}
|
|
858
|
+
const patternRecs = new Set();
|
|
859
|
+
for (const result of results) {
|
|
860
|
+
for (const rec of result.recommendations) {
|
|
861
|
+
if (!rec.includes("Claude")) {
|
|
862
|
+
patternRecs.add(rec);
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
const destructiveRecs = Array.from(patternRecs).filter((r) => r.includes("destructive"));
|
|
867
|
+
const otherRecs = Array.from(patternRecs).filter((r) => !r.includes("destructive"));
|
|
868
|
+
if (destructiveRecs.length > 0) {
|
|
869
|
+
recommendations.push("PRIORITY: Potential destructive tools without proper hints:");
|
|
870
|
+
recommendations.push(...destructiveRecs.slice(0, 3));
|
|
871
|
+
}
|
|
872
|
+
if (otherRecs.length > 0 && recommendations.length < 10) {
|
|
873
|
+
recommendations.push(...otherRecs.slice(0, 3));
|
|
874
|
+
}
|
|
875
|
+
if (recommendations.length === 0) {
|
|
876
|
+
recommendations.push("All tools have proper annotations. No action required.");
|
|
877
|
+
}
|
|
878
|
+
else {
|
|
879
|
+
recommendations.push("Reference: MCP Directory Policy #17 requires tools to have readOnlyHint and destructiveHint annotations.");
|
|
880
|
+
}
|
|
881
|
+
return recommendations;
|
|
882
|
+
}
|
|
1406
883
|
}
|