@bryan-thompson/inspector-assessment-client 1.12.0 → 1.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-DD8JgGmx.js → OAuthCallback-Dg3beipA.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-CGeg00AP.js → OAuthDebugCallback-zRUPyR0T.js} +1 -1
- package/dist/assets/{index-sUICDw7A.js → index-DtKbQaUh.js} +136 -8
- package/dist/index.html +1 -1
- package/lib/lib/assessmentTypes.d.ts +33 -0
- package/lib/lib/assessmentTypes.d.ts.map +1 -1
- package/lib/lib/assessmentTypes.js +5 -0
- package/lib/lib/policyMapping.d.ts +183 -0
- package/lib/lib/policyMapping.d.ts.map +1 -0
- package/lib/lib/policyMapping.js +442 -0
- package/lib/lib/reportFormatters/MarkdownReportFormatter.d.ts +91 -0
- package/lib/lib/reportFormatters/MarkdownReportFormatter.d.ts.map +1 -0
- package/lib/lib/reportFormatters/MarkdownReportFormatter.js +498 -0
- package/lib/lib/reportFormatters/index.d.ts +50 -0
- package/lib/lib/reportFormatters/index.d.ts.map +1 -0
- package/lib/lib/reportFormatters/index.js +81 -0
- package/lib/lib/securityPatterns.d.ts +3 -3
- package/lib/lib/securityPatterns.d.ts.map +1 -1
- package/lib/lib/securityPatterns.js +129 -4
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +1 -0
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +22 -1
- package/lib/services/assessment/PolicyComplianceGenerator.d.ts +119 -0
- package/lib/services/assessment/PolicyComplianceGenerator.d.ts.map +1 -0
- package/lib/services/assessment/PolicyComplianceGenerator.js +632 -0
- package/lib/services/assessment/modules/ExternalAPIScannerAssessor.d.ts +58 -0
- package/lib/services/assessment/modules/ExternalAPIScannerAssessor.d.ts.map +1 -0
- package/lib/services/assessment/modules/ExternalAPIScannerAssessor.js +248 -0
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts +6 -0
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ToolAnnotationAssessor.js +77 -20
- package/lib/services/assessment/modules/index.d.ts +1 -0
- package/lib/services/assessment/modules/index.d.ts.map +1 -1
- package/lib/services/assessment/modules/index.js +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* External API Scanner Assessor
|
|
3
|
+
*
|
|
4
|
+
* Scans source code for external API dependencies and checks for affiliation.
|
|
5
|
+
* Helps identify:
|
|
6
|
+
* - External services used (GitHub, Slack, AWS, etc.)
|
|
7
|
+
* - Hardcoded URLs that may need privacy policy disclosure
|
|
8
|
+
* - Server names that suggest affiliation without verification
|
|
9
|
+
*
|
|
10
|
+
* Part of Priority 2 gap-closing features.
|
|
11
|
+
*/
|
|
12
|
+
import { BaseAssessor } from "./BaseAssessor.js";
|
|
13
|
+
/**
|
|
14
|
+
* Known external services and their URL patterns
|
|
15
|
+
*/
|
|
16
|
+
const KNOWN_SERVICES = {
|
|
17
|
+
github: [/api\.github\.com/i, /github\.com\/[^/]+\/[^/]+/i],
|
|
18
|
+
gitlab: [/gitlab\.com/i, /api\.gitlab\.com/i],
|
|
19
|
+
slack: [/slack\.com\/api/i, /api\.slack\.com/i, /hooks\.slack\.com/i],
|
|
20
|
+
discord: [/discord\.com\/api/i, /discordapp\.com/i],
|
|
21
|
+
aws: [/\.amazonaws\.com/i, /\.aws\.amazon\.com/i],
|
|
22
|
+
azure: [/\.azure\.com/i, /\.microsoft\.com/i],
|
|
23
|
+
gcp: [/\.googleapis\.com/i, /\.google\.com\/.*api/i],
|
|
24
|
+
openai: [/api\.openai\.com/i],
|
|
25
|
+
anthropic: [/api\.anthropic\.com/i],
|
|
26
|
+
huggingface: [/huggingface\.co/i, /api\.huggingface\.co/i],
|
|
27
|
+
stripe: [/api\.stripe\.com/i],
|
|
28
|
+
twilio: [/api\.twilio\.com/i],
|
|
29
|
+
sendgrid: [/api\.sendgrid\.com/i],
|
|
30
|
+
firebase: [/firebaseio\.com/i, /firebase\.google\.com/i],
|
|
31
|
+
supabase: [/supabase\.co/i],
|
|
32
|
+
notion: [/api\.notion\.com/i],
|
|
33
|
+
airtable: [/api\.airtable\.com/i],
|
|
34
|
+
dropbox: [/api\.dropbox\.com/i, /dropboxapi\.com/i],
|
|
35
|
+
jira: [/atlassian\.net/i, /jira\.com/i],
|
|
36
|
+
linear: [/api\.linear\.app/i],
|
|
37
|
+
asana: [/api\.asana\.com/i],
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* URL patterns to skip (not external APIs)
|
|
41
|
+
*/
|
|
42
|
+
const SKIP_URL_PATTERNS = [
|
|
43
|
+
/localhost/i,
|
|
44
|
+
/127\.0\.0\.1/i,
|
|
45
|
+
/0\.0\.0\.0/i,
|
|
46
|
+
/example\.com/i,
|
|
47
|
+
/test\.com/i,
|
|
48
|
+
/\.local\//i,
|
|
49
|
+
/schema\.org/i,
|
|
50
|
+
/w3\.org/i,
|
|
51
|
+
/json-schema\.org/i,
|
|
52
|
+
/npmjs\.com/i,
|
|
53
|
+
/unpkg\.com/i,
|
|
54
|
+
/cdn\./i,
|
|
55
|
+
/fonts\.googleapis\.com/i,
|
|
56
|
+
];
|
|
57
|
+
/**
|
|
58
|
+
* File patterns to skip during scanning
|
|
59
|
+
*/
|
|
60
|
+
const SKIP_FILE_PATTERNS = [
|
|
61
|
+
/node_modules/i,
|
|
62
|
+
/\.test\.(ts|js|tsx|jsx)$/i,
|
|
63
|
+
/\.spec\.(ts|js|tsx|jsx)$/i,
|
|
64
|
+
/\.d\.ts$/i,
|
|
65
|
+
/package-lock\.json$/i,
|
|
66
|
+
/yarn\.lock$/i,
|
|
67
|
+
/\.map$/i,
|
|
68
|
+
/README\.md$/i,
|
|
69
|
+
/CHANGELOG\.md$/i,
|
|
70
|
+
/LICENSE/i,
|
|
71
|
+
/\.git\//i,
|
|
72
|
+
/dist\//i,
|
|
73
|
+
/build\//i,
|
|
74
|
+
];
|
|
75
|
+
export class ExternalAPIScannerAssessor extends BaseAssessor {
|
|
76
|
+
async assess(context) {
|
|
77
|
+
this.log("Starting external API scanner assessment");
|
|
78
|
+
this.resetTestCount();
|
|
79
|
+
const detectedAPIs = [];
|
|
80
|
+
let scannedFiles = 0;
|
|
81
|
+
// Check if source code analysis is enabled
|
|
82
|
+
if (!context.sourceCodeFiles || !context.config.enableSourceCodeAnalysis) {
|
|
83
|
+
this.log("Source code analysis not enabled, skipping external API scan");
|
|
84
|
+
return this.createNoSourceResult();
|
|
85
|
+
}
|
|
86
|
+
// Scan each source file
|
|
87
|
+
for (const [filePath, content] of context.sourceCodeFiles) {
|
|
88
|
+
if (this.shouldSkipFile(filePath))
|
|
89
|
+
continue;
|
|
90
|
+
this.testCount++;
|
|
91
|
+
scannedFiles++;
|
|
92
|
+
const fileAPIs = this.scanFileForAPIs(filePath, content);
|
|
93
|
+
detectedAPIs.push(...fileAPIs);
|
|
94
|
+
}
|
|
95
|
+
// Extract unique services
|
|
96
|
+
const uniqueServices = [...new Set(detectedAPIs.map((api) => api.service))];
|
|
97
|
+
// Check for affiliation concerns
|
|
98
|
+
const affiliationWarning = this.checkAffiliation(context.serverName, uniqueServices);
|
|
99
|
+
// Determine status
|
|
100
|
+
const status = this.computeStatus(detectedAPIs, affiliationWarning);
|
|
101
|
+
const explanation = this.generateExplanation(detectedAPIs, uniqueServices, affiliationWarning, scannedFiles);
|
|
102
|
+
const recommendations = this.generateRecommendations(uniqueServices, affiliationWarning);
|
|
103
|
+
this.log(`External API scan complete: ${detectedAPIs.length} APIs found in ${scannedFiles} files`);
|
|
104
|
+
return {
|
|
105
|
+
detectedAPIs,
|
|
106
|
+
uniqueServices,
|
|
107
|
+
affiliationWarning,
|
|
108
|
+
scannedFiles,
|
|
109
|
+
status,
|
|
110
|
+
explanation,
|
|
111
|
+
recommendations,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Check if file should be skipped
|
|
116
|
+
*/
|
|
117
|
+
shouldSkipFile(filePath) {
|
|
118
|
+
return SKIP_FILE_PATTERNS.some((pattern) => pattern.test(filePath));
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Scan a file for external API URLs
|
|
122
|
+
*/
|
|
123
|
+
scanFileForAPIs(filePath, content) {
|
|
124
|
+
const apis = [];
|
|
125
|
+
const urlPattern = /https?:\/\/[^\s'"`)}\]><]+/g;
|
|
126
|
+
const matches = content.match(urlPattern) || [];
|
|
127
|
+
for (const url of matches) {
|
|
128
|
+
// Skip non-API URLs
|
|
129
|
+
if (this.shouldSkipUrl(url))
|
|
130
|
+
continue;
|
|
131
|
+
// Identify the service
|
|
132
|
+
const service = this.identifyService(url);
|
|
133
|
+
// Avoid duplicates in the same file
|
|
134
|
+
if (!apis.some((api) => api.url === url && api.filePath === filePath)) {
|
|
135
|
+
apis.push({
|
|
136
|
+
url: this.cleanUrl(url),
|
|
137
|
+
service,
|
|
138
|
+
filePath,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return apis;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Check if URL should be skipped
|
|
146
|
+
*/
|
|
147
|
+
shouldSkipUrl(url) {
|
|
148
|
+
return SKIP_URL_PATTERNS.some((pattern) => pattern.test(url));
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Clean URL by removing trailing punctuation
|
|
152
|
+
*/
|
|
153
|
+
cleanUrl(url) {
|
|
154
|
+
return url.replace(/[.,;:!?'")\]}>]+$/, "");
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Identify which service a URL belongs to
|
|
158
|
+
*/
|
|
159
|
+
identifyService(url) {
|
|
160
|
+
for (const [service, patterns] of Object.entries(KNOWN_SERVICES)) {
|
|
161
|
+
if (patterns.some((pattern) => pattern.test(url))) {
|
|
162
|
+
return service;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return "unknown";
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Check if server name suggests affiliation that needs verification
|
|
169
|
+
*/
|
|
170
|
+
checkAffiliation(serverName, detectedServices) {
|
|
171
|
+
const nameLower = serverName.toLowerCase();
|
|
172
|
+
const nameParts = nameLower.split(/[-_\s]/);
|
|
173
|
+
// Check if server name contains a known service name
|
|
174
|
+
for (const service of Object.keys(KNOWN_SERVICES)) {
|
|
175
|
+
if (nameParts.includes(service) || nameLower.includes(service)) {
|
|
176
|
+
// Server claims this service - check if actually uses it
|
|
177
|
+
if (detectedServices.includes(service)) {
|
|
178
|
+
return `Server name "${serverName}" suggests affiliation with ${service}. Verify the author is officially affiliated before approving.`;
|
|
179
|
+
}
|
|
180
|
+
else {
|
|
181
|
+
return `Server name "${serverName}" suggests affiliation with ${service}, but no ${service} API calls detected. This may be misleading.`;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return undefined;
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Compute assessment status based on scan results
|
|
189
|
+
*/
|
|
190
|
+
computeStatus(apis, affiliationWarning) {
|
|
191
|
+
if (affiliationWarning) {
|
|
192
|
+
return "NEED_MORE_INFO";
|
|
193
|
+
}
|
|
194
|
+
if (apis.length === 0) {
|
|
195
|
+
return "PASS";
|
|
196
|
+
}
|
|
197
|
+
// Having external APIs isn't a failure, just informational
|
|
198
|
+
return "PASS";
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Generate explanation text
|
|
202
|
+
*/
|
|
203
|
+
generateExplanation(apis, services, affiliationWarning, scannedFiles) {
|
|
204
|
+
const parts = [];
|
|
205
|
+
parts.push(`Scanned ${scannedFiles} source files.`);
|
|
206
|
+
if (apis.length === 0) {
|
|
207
|
+
parts.push("No external API dependencies detected.");
|
|
208
|
+
}
|
|
209
|
+
else {
|
|
210
|
+
parts.push(`Found ${apis.length} external API URL(s) across ${services.length} service(s): ${services.join(", ")}.`);
|
|
211
|
+
}
|
|
212
|
+
if (affiliationWarning) {
|
|
213
|
+
parts.push(`ATTENTION: ${affiliationWarning}`);
|
|
214
|
+
}
|
|
215
|
+
return parts.join(" ");
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Generate recommendations
|
|
219
|
+
*/
|
|
220
|
+
generateRecommendations(services, affiliationWarning) {
|
|
221
|
+
const recommendations = [];
|
|
222
|
+
if (affiliationWarning) {
|
|
223
|
+
recommendations.push("Verify author affiliation with claimed service before directory approval");
|
|
224
|
+
}
|
|
225
|
+
if (services.length > 0 && !services.every((s) => s === "unknown")) {
|
|
226
|
+
recommendations.push(`Ensure privacy policies are declared for external services: ${services.filter((s) => s !== "unknown").join(", ")}`);
|
|
227
|
+
}
|
|
228
|
+
if (services.includes("unknown")) {
|
|
229
|
+
recommendations.push("Review unrecognized external URLs for security and privacy implications");
|
|
230
|
+
}
|
|
231
|
+
return recommendations;
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Create result when source code analysis is not available
|
|
235
|
+
*/
|
|
236
|
+
createNoSourceResult() {
|
|
237
|
+
return {
|
|
238
|
+
detectedAPIs: [],
|
|
239
|
+
uniqueServices: [],
|
|
240
|
+
scannedFiles: 0,
|
|
241
|
+
status: "NEED_MORE_INFO",
|
|
242
|
+
explanation: "External API scanning requires source code analysis. Enable with --source flag.",
|
|
243
|
+
recommendations: [
|
|
244
|
+
"Re-run assessment with --source <path> to enable external API scanning",
|
|
245
|
+
],
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
}
|
|
@@ -82,6 +82,12 @@ export declare class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
82
82
|
/**
|
|
83
83
|
* Extract annotations from a tool
|
|
84
84
|
* MCP SDK may have annotations in different locations
|
|
85
|
+
*
|
|
86
|
+
* Priority order:
|
|
87
|
+
* 1. tool.annotations (MCP 2024-11 spec) - "mcp" source
|
|
88
|
+
* 2. Direct properties on tool - "mcp" source
|
|
89
|
+
* 3. tool.metadata - "mcp" source
|
|
90
|
+
* 4. No annotations found - "none" source
|
|
85
91
|
*/
|
|
86
92
|
private extractAnnotations;
|
|
87
93
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ToolAnnotationAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ToolAnnotationAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EACV,wBAAwB,EACxB,oBAAoB,EAKpB,uBAAuB,
|
|
1
|
+
{"version":3,"file":"ToolAnnotationAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ToolAnnotationAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EACV,wBAAwB,EACxB,oBAAoB,EAKpB,uBAAuB,EAExB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EACL,KAAK,gBAAgB,EAGtB,MAAM,8BAA8B,CAAC;AAEtC;;GAEG;AACH,MAAM,WAAW,4BAA6B,SAAQ,oBAAoB;IACxE,eAAe,CAAC,EAAE;QAChB,gBAAgB,EAAE,OAAO,CAAC;QAC1B,mBAAmB,EAAE,OAAO,CAAC;QAC7B,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,oBAAoB,EAAE;YACpB,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,eAAe,CAAC,EAAE,OAAO,CAAC;YAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;SAC1B,CAAC;QACF,oBAAoB,EAAE,OAAO,CAAC;QAC9B,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,MAAM,EAAE,iBAAiB,GAAG,eAAe,CAAC;KAC7C,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gCAAiC,SAAQ,wBAAwB;IAChF,WAAW,EAAE,4BAA4B,EAAE,CAAC;IAC5C,cAAc,EAAE,OAAO,CAAC;IACxB,2BAA2B,EAAE,4BAA4B,EAAE,CAAC;CAC7D;AAKD,qBAAa,sBAAuB,SAAQ,YAAY;IACtD,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,gBAAgB,CAAmB;gBAE/B,MAAM,EAAE,uBAAuB;IAM3C;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,gBAAgB,GAAG,IAAI;IAK7C;;OAEG;IACH,eAAe,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAK/C;;OAEG;IACH,eAAe,IAAI,OAAO;IAO1B;;OAEG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,wBAAwB,GAAG,gCAAgC,CAAC;IA8QvE;;OAEG;YACW,0BAA0B;IA+IxC;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAiCnC;;OAEG;IACH,OAAO,CAAC,+BAA+B;IAoFvC;;;OAGG;IACH,OAAO,CAAC,UAAU;IA+GlB;;;;;;;;;OASG;IACH,OAAO,CAAC,kBAAkB;IAyE1B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAuBzB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAgGrB;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAkDjC;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAiDxB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAmC3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CA2ChC"}
|
|
@@ -53,6 +53,13 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
53
53
|
let annotatedCount = 0;
|
|
54
54
|
let missingAnnotationsCount = 0;
|
|
55
55
|
let misalignedAnnotationsCount = 0;
|
|
56
|
+
// Track annotation sources
|
|
57
|
+
const annotationSourceCounts = {
|
|
58
|
+
mcp: 0,
|
|
59
|
+
sourceCode: 0,
|
|
60
|
+
inferred: 0,
|
|
61
|
+
none: 0,
|
|
62
|
+
};
|
|
56
63
|
const useClaudeInference = this.isClaudeEnabled();
|
|
57
64
|
if (useClaudeInference) {
|
|
58
65
|
this.log("Claude Code integration enabled - using semantic behavior inference");
|
|
@@ -106,7 +113,23 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
106
113
|
}
|
|
107
114
|
else {
|
|
108
115
|
missingAnnotationsCount++;
|
|
109
|
-
|
|
116
|
+
}
|
|
117
|
+
// Track annotation source
|
|
118
|
+
const source = latestResult.annotationSource;
|
|
119
|
+
if (source === "mcp") {
|
|
120
|
+
annotationSourceCounts.mcp++;
|
|
121
|
+
}
|
|
122
|
+
else if (source === "source-code") {
|
|
123
|
+
annotationSourceCounts.sourceCode++;
|
|
124
|
+
}
|
|
125
|
+
else if (source === "inferred") {
|
|
126
|
+
annotationSourceCounts.inferred++;
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
annotationSourceCounts.none++;
|
|
130
|
+
}
|
|
131
|
+
// Emit annotation_missing event with tool details
|
|
132
|
+
if (!latestResult.hasAnnotations) {
|
|
110
133
|
if (context.onProgress && latestResult.inferredBehavior) {
|
|
111
134
|
const annotations = this.extractAnnotations(tool);
|
|
112
135
|
context.onProgress({
|
|
@@ -225,6 +248,7 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
225
248
|
recommendations: this.generateEnhancedRecommendations(toolResults),
|
|
226
249
|
metrics,
|
|
227
250
|
alignmentBreakdown,
|
|
251
|
+
annotationSources: annotationSourceCounts,
|
|
228
252
|
claudeEnhanced: true,
|
|
229
253
|
highConfidenceMisalignments,
|
|
230
254
|
};
|
|
@@ -239,6 +263,7 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
239
263
|
recommendations,
|
|
240
264
|
metrics,
|
|
241
265
|
alignmentBreakdown,
|
|
266
|
+
annotationSources: annotationSourceCounts,
|
|
242
267
|
};
|
|
243
268
|
}
|
|
244
269
|
/**
|
|
@@ -495,6 +520,7 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
495
520
|
toolName: tool.name,
|
|
496
521
|
hasAnnotations,
|
|
497
522
|
annotations: hasAnnotations ? annotations : undefined,
|
|
523
|
+
annotationSource: annotations.source,
|
|
498
524
|
inferredBehavior,
|
|
499
525
|
alignmentStatus,
|
|
500
526
|
issues,
|
|
@@ -504,34 +530,65 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
504
530
|
/**
|
|
505
531
|
* Extract annotations from a tool
|
|
506
532
|
* MCP SDK may have annotations in different locations
|
|
533
|
+
*
|
|
534
|
+
* Priority order:
|
|
535
|
+
* 1. tool.annotations (MCP 2024-11 spec) - "mcp" source
|
|
536
|
+
* 2. Direct properties on tool - "mcp" source
|
|
537
|
+
* 3. tool.metadata - "mcp" source
|
|
538
|
+
* 4. No annotations found - "none" source
|
|
507
539
|
*/
|
|
508
540
|
extractAnnotations(tool) {
|
|
509
|
-
// Try to find annotations in various locations
|
|
510
541
|
const toolAny = tool;
|
|
511
|
-
// Check
|
|
512
|
-
let readOnlyHint = toolAny.readOnlyHint;
|
|
513
|
-
let destructiveHint = toolAny.destructiveHint;
|
|
514
|
-
let idempotentHint = toolAny.idempotentHint;
|
|
515
|
-
let openWorldHint = toolAny.openWorldHint;
|
|
516
|
-
// Check annotations object (MCP 2024-11 spec)
|
|
542
|
+
// Priority 1: Check annotations object (MCP 2024-11 spec) - primary source
|
|
517
543
|
if (toolAny.annotations) {
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
544
|
+
const hasAnnotations = toolAny.annotations.readOnlyHint !== undefined ||
|
|
545
|
+
toolAny.annotations.destructiveHint !== undefined;
|
|
546
|
+
if (hasAnnotations) {
|
|
547
|
+
return {
|
|
548
|
+
readOnlyHint: toolAny.annotations.readOnlyHint,
|
|
549
|
+
destructiveHint: toolAny.annotations.destructiveHint,
|
|
550
|
+
title: toolAny.annotations.title || toolAny.title,
|
|
551
|
+
description: tool.description,
|
|
552
|
+
idempotentHint: toolAny.annotations.idempotentHint,
|
|
553
|
+
openWorldHint: toolAny.annotations.openWorldHint,
|
|
554
|
+
source: "mcp",
|
|
555
|
+
};
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
// Priority 2: Check direct properties on tool object
|
|
559
|
+
if (toolAny.readOnlyHint !== undefined ||
|
|
560
|
+
toolAny.destructiveHint !== undefined) {
|
|
561
|
+
return {
|
|
562
|
+
readOnlyHint: toolAny.readOnlyHint,
|
|
563
|
+
destructiveHint: toolAny.destructiveHint,
|
|
564
|
+
title: toolAny.title,
|
|
565
|
+
description: tool.description,
|
|
566
|
+
idempotentHint: toolAny.idempotentHint,
|
|
567
|
+
openWorldHint: toolAny.openWorldHint,
|
|
568
|
+
source: "mcp",
|
|
569
|
+
};
|
|
522
570
|
}
|
|
523
|
-
// Check metadata (some servers use this)
|
|
571
|
+
// Priority 3: Check metadata (some servers use this)
|
|
524
572
|
if (toolAny.metadata) {
|
|
525
|
-
|
|
526
|
-
|
|
573
|
+
const hasMetadataAnnotations = toolAny.metadata.readOnlyHint !== undefined ||
|
|
574
|
+
toolAny.metadata.destructiveHint !== undefined;
|
|
575
|
+
if (hasMetadataAnnotations) {
|
|
576
|
+
return {
|
|
577
|
+
readOnlyHint: toolAny.metadata.readOnlyHint,
|
|
578
|
+
destructiveHint: toolAny.metadata.destructiveHint,
|
|
579
|
+
title: toolAny.metadata.title || toolAny.title,
|
|
580
|
+
description: tool.description,
|
|
581
|
+
idempotentHint: toolAny.metadata.idempotentHint,
|
|
582
|
+
openWorldHint: toolAny.metadata.openWorldHint,
|
|
583
|
+
source: "mcp",
|
|
584
|
+
};
|
|
585
|
+
}
|
|
527
586
|
}
|
|
587
|
+
// No annotations found from MCP protocol
|
|
528
588
|
return {
|
|
529
|
-
|
|
530
|
-
destructiveHint,
|
|
531
|
-
title: toolAny.title || toolAny.annotations?.title,
|
|
589
|
+
title: toolAny.title,
|
|
532
590
|
description: tool.description,
|
|
533
|
-
|
|
534
|
-
openWorldHint,
|
|
591
|
+
source: "none",
|
|
535
592
|
};
|
|
536
593
|
}
|
|
537
594
|
/**
|
|
@@ -30,4 +30,5 @@ export { ToolAnnotationAssessor } from "./ToolAnnotationAssessor.js";
|
|
|
30
30
|
export { ProhibitedLibrariesAssessor } from "./ProhibitedLibrariesAssessor.js";
|
|
31
31
|
export { ManifestValidationAssessor } from "./ManifestValidationAssessor.js";
|
|
32
32
|
export { PortabilityAssessor } from "./PortabilityAssessor.js";
|
|
33
|
+
export { ExternalAPIScannerAssessor } from "./ExternalAPIScannerAssessor.js";
|
|
33
34
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AAGxE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,2BAA2B,EAAE,MAAM,+BAA+B,CAAC;AAC5E,OAAO,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAC;AAC1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AAGxE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,2BAA2B,EAAE,MAAM,+BAA+B,CAAC;AAC5E,OAAO,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAC;AAC1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAC"}
|
|
@@ -33,3 +33,4 @@ export { ToolAnnotationAssessor } from "./ToolAnnotationAssessor.js";
|
|
|
33
33
|
export { ProhibitedLibrariesAssessor } from "./ProhibitedLibrariesAssessor.js";
|
|
34
34
|
export { ManifestValidationAssessor } from "./ManifestValidationAssessor.js";
|
|
35
35
|
export { PortabilityAssessor } from "./PortabilityAssessor.js";
|
|
36
|
+
export { ExternalAPIScannerAssessor } from "./ExternalAPIScannerAssessor.js";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bryan-thompson/inspector-assessment-client",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.13.1",
|
|
4
4
|
"description": "Client-side application for the Enhanced MCP Inspector with assessment capabilities",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Bryan Thompson <bryan@triepod.ai>",
|