@bryan-thompson/inspector-assessment-client 1.10.1 → 1.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-Bi_akC9k.js → OAuthCallback-DA2koy6X.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-DupZK8qN.js → OAuthDebugCallback-Bx60PQTT.js} +1 -1
- package/dist/assets/{index-BQQRMUvH.js → index-kJ0jPd4m.js} +1218 -384
- package/dist/index.html +1 -1
- package/lib/lib/assessmentTypes.d.ts +68 -1
- package/lib/lib/assessmentTypes.d.ts.map +1 -1
- package/lib/lib/moduleScoring.d.ts +23 -0
- package/lib/lib/moduleScoring.d.ts.map +1 -0
- package/lib/lib/moduleScoring.js +53 -0
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +14 -37
- package/lib/services/assessment/TestDataGenerator.d.ts +22 -0
- package/lib/services/assessment/TestDataGenerator.d.ts.map +1 -1
- package/lib/services/assessment/TestDataGenerator.js +78 -0
- package/lib/services/assessment/modules/FunctionalityAssessor.d.ts +15 -0
- package/lib/services/assessment/modules/FunctionalityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/FunctionalityAssessor.js +137 -6
- package/lib/services/assessment/modules/SecurityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/SecurityAssessor.js +26 -0
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts +4 -0
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ToolAnnotationAssessor.js +81 -1
- package/package.json +1 -1
|
@@ -5,7 +5,10 @@
|
|
|
5
5
|
import { BaseAssessor } from "./BaseAssessor.js";
|
|
6
6
|
import { ResponseValidator } from "../ResponseValidator.js";
|
|
7
7
|
import { createConcurrencyLimit } from "../lib/concurrencyLimit.js";
|
|
8
|
+
import { ToolClassifier, ToolCategory } from "../ToolClassifier.js";
|
|
9
|
+
import { TestDataGenerator } from "../TestDataGenerator.js";
|
|
8
10
|
export class FunctionalityAssessor extends BaseAssessor {
|
|
11
|
+
toolClassifier = new ToolClassifier();
|
|
9
12
|
/**
|
|
10
13
|
* Select tools for testing based on configuration
|
|
11
14
|
*/
|
|
@@ -118,9 +121,9 @@ export class FunctionalityAssessor extends BaseAssessor {
|
|
|
118
121
|
}
|
|
119
122
|
async testTool(tool, callTool) {
|
|
120
123
|
const startTime = Date.now();
|
|
124
|
+
// Generate minimal valid parameters with metadata
|
|
125
|
+
const { params: testParams, metadata } = this.generateMinimalParams(tool);
|
|
121
126
|
try {
|
|
122
|
-
// Generate minimal valid parameters
|
|
123
|
-
const testParams = this.generateMinimalParams(tool);
|
|
124
127
|
this.log(`Testing tool: ${tool.name} with params: ${JSON.stringify(testParams)}`);
|
|
125
128
|
// Execute tool with timeout
|
|
126
129
|
const response = await this.executeWithTimeout(callTool(tool.name, testParams), this.config.testTimeout);
|
|
@@ -145,6 +148,7 @@ export class FunctionalityAssessor extends BaseAssessor {
|
|
|
145
148
|
executionTime,
|
|
146
149
|
testParameters: testParams,
|
|
147
150
|
response,
|
|
151
|
+
testInputMetadata: metadata,
|
|
148
152
|
};
|
|
149
153
|
}
|
|
150
154
|
// Real tool failure (not just validation)
|
|
@@ -156,6 +160,7 @@ export class FunctionalityAssessor extends BaseAssessor {
|
|
|
156
160
|
executionTime,
|
|
157
161
|
testParameters: testParams,
|
|
158
162
|
response,
|
|
163
|
+
testInputMetadata: metadata,
|
|
159
164
|
};
|
|
160
165
|
}
|
|
161
166
|
return {
|
|
@@ -165,6 +170,7 @@ export class FunctionalityAssessor extends BaseAssessor {
|
|
|
165
170
|
executionTime,
|
|
166
171
|
testParameters: testParams,
|
|
167
172
|
response,
|
|
173
|
+
testInputMetadata: metadata,
|
|
168
174
|
};
|
|
169
175
|
}
|
|
170
176
|
catch (error) {
|
|
@@ -174,28 +180,50 @@ export class FunctionalityAssessor extends BaseAssessor {
|
|
|
174
180
|
status: "broken",
|
|
175
181
|
error: this.extractErrorMessage(error),
|
|
176
182
|
executionTime: Date.now() - startTime,
|
|
183
|
+
testInputMetadata: metadata,
|
|
177
184
|
};
|
|
178
185
|
}
|
|
179
186
|
}
|
|
180
187
|
generateMinimalParams(tool) {
|
|
188
|
+
// Classify tool to get category for smart parameter generation
|
|
189
|
+
const classification = this.toolClassifier.classify(tool.name, tool.description || "");
|
|
190
|
+
const primaryCategory = classification.categories[0] || ToolCategory.GENERIC;
|
|
191
|
+
const emptyResult = {
|
|
192
|
+
params: {},
|
|
193
|
+
metadata: {
|
|
194
|
+
toolCategory: primaryCategory,
|
|
195
|
+
generationStrategy: "default",
|
|
196
|
+
fieldSources: {},
|
|
197
|
+
},
|
|
198
|
+
};
|
|
181
199
|
if (!tool.inputSchema)
|
|
182
|
-
return
|
|
200
|
+
return emptyResult;
|
|
183
201
|
const schema = typeof tool.inputSchema === "string"
|
|
184
202
|
? this.safeJsonParse(tool.inputSchema)
|
|
185
203
|
: tool.inputSchema;
|
|
186
204
|
if (!schema?.properties)
|
|
187
|
-
return
|
|
205
|
+
return emptyResult;
|
|
188
206
|
const params = {};
|
|
207
|
+
const fieldSources = {};
|
|
189
208
|
const required = schema.required || [];
|
|
190
209
|
// For functionality testing, only generate REQUIRED parameters
|
|
191
210
|
// This avoids triggering validation errors on optional parameters with complex rules
|
|
192
211
|
for (const [key, prop] of Object.entries(schema.properties)) {
|
|
193
212
|
// Only include required parameters for basic functionality testing
|
|
194
213
|
if (required.includes(key)) {
|
|
195
|
-
|
|
214
|
+
const { value, source, reason } = this.generateSmartParamValueWithMetadata(prop, key, primaryCategory);
|
|
215
|
+
params[key] = value;
|
|
216
|
+
fieldSources[key] = { field: key, value, source, reason };
|
|
196
217
|
}
|
|
197
218
|
}
|
|
198
|
-
return
|
|
219
|
+
return {
|
|
220
|
+
params,
|
|
221
|
+
metadata: {
|
|
222
|
+
toolCategory: primaryCategory,
|
|
223
|
+
generationStrategy: this.determineStrategy(fieldSources),
|
|
224
|
+
fieldSources,
|
|
225
|
+
},
|
|
226
|
+
};
|
|
199
227
|
}
|
|
200
228
|
generateParamValue(prop, fieldName, includeOptional = false) {
|
|
201
229
|
const type = prop.type;
|
|
@@ -269,6 +297,109 @@ export class FunctionalityAssessor extends BaseAssessor {
|
|
|
269
297
|
return {};
|
|
270
298
|
}
|
|
271
299
|
}
|
|
300
|
+
/**
|
|
301
|
+
* Field names that indicate specific data types regardless of tool category.
|
|
302
|
+
* These take precedence over category-specific generation.
|
|
303
|
+
*/
|
|
304
|
+
static SPECIFIC_FIELD_PATTERNS = [
|
|
305
|
+
/url/i,
|
|
306
|
+
/endpoint/i,
|
|
307
|
+
/link/i,
|
|
308
|
+
/email/i,
|
|
309
|
+
/mail/i,
|
|
310
|
+
/path/i,
|
|
311
|
+
/file/i,
|
|
312
|
+
/directory/i,
|
|
313
|
+
/folder/i,
|
|
314
|
+
/uuid/i,
|
|
315
|
+
/page_id/i,
|
|
316
|
+
/database_id/i,
|
|
317
|
+
/user_id/i,
|
|
318
|
+
/block_id/i,
|
|
319
|
+
];
|
|
320
|
+
/**
|
|
321
|
+
* Generate smart parameter value with metadata about how it was generated.
|
|
322
|
+
* Returns value, source type, and reason for downstream consumers.
|
|
323
|
+
*/
|
|
324
|
+
generateSmartParamValueWithMetadata(prop, fieldName, category) {
|
|
325
|
+
// Handle enum first
|
|
326
|
+
if (prop.enum && prop.enum.length > 0) {
|
|
327
|
+
return {
|
|
328
|
+
value: prop.enum[0],
|
|
329
|
+
source: "enum",
|
|
330
|
+
reason: `First enum value: ${prop.enum[0]}`,
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
// Handle format (uri, email, etc.)
|
|
334
|
+
if (prop.format === "uri") {
|
|
335
|
+
return {
|
|
336
|
+
value: "https://example.com",
|
|
337
|
+
source: "format",
|
|
338
|
+
reason: "URI format detected",
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
if (prop.format === "email") {
|
|
342
|
+
return {
|
|
343
|
+
value: "test@example.com",
|
|
344
|
+
source: "format",
|
|
345
|
+
reason: "Email format detected",
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
// For non-string types, use standard generation
|
|
349
|
+
if (prop.type !== "string") {
|
|
350
|
+
const value = this.generateParamValue(prop, fieldName);
|
|
351
|
+
return {
|
|
352
|
+
value,
|
|
353
|
+
source: "default",
|
|
354
|
+
reason: `Default for type: ${prop.type}`,
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
// Specific field names (url, email, path, etc.) take precedence over category
|
|
358
|
+
// These indicate explicit data type requirements regardless of tool category
|
|
359
|
+
const isSpecificFieldName = FunctionalityAssessor.SPECIFIC_FIELD_PATTERNS.some((pattern) => pattern.test(fieldName));
|
|
360
|
+
if (isSpecificFieldName) {
|
|
361
|
+
const fieldValue = TestDataGenerator.generateSingleValue(fieldName, prop);
|
|
362
|
+
return {
|
|
363
|
+
value: fieldValue,
|
|
364
|
+
source: "field-name",
|
|
365
|
+
reason: `Field name pattern: ${fieldName}`,
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
// Check category-specific data
|
|
369
|
+
const categoryData = TestDataGenerator.TOOL_CATEGORY_DATA[category];
|
|
370
|
+
if (categoryData?.default) {
|
|
371
|
+
return {
|
|
372
|
+
value: categoryData.default[0],
|
|
373
|
+
source: "category",
|
|
374
|
+
reason: `Category ${category} default value`,
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
// Fall back to field-name detection for generic fields
|
|
378
|
+
const fieldValue = TestDataGenerator.generateSingleValue(fieldName, prop);
|
|
379
|
+
if (fieldValue !== "test") {
|
|
380
|
+
return {
|
|
381
|
+
value: fieldValue,
|
|
382
|
+
source: "field-name",
|
|
383
|
+
reason: `Field name pattern: ${fieldName}`,
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
return {
|
|
387
|
+
value: "test",
|
|
388
|
+
source: "default",
|
|
389
|
+
reason: "No specific pattern matched",
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
/**
|
|
393
|
+
* Determine overall generation strategy based on field sources
|
|
394
|
+
*/
|
|
395
|
+
determineStrategy(fieldSources) {
|
|
396
|
+
const sources = Object.values(fieldSources).map((f) => f.source);
|
|
397
|
+
if (sources.includes("category"))
|
|
398
|
+
return "category-specific";
|
|
399
|
+
if (sources.includes("field-name"))
|
|
400
|
+
return "field-name-aware";
|
|
401
|
+
return "default";
|
|
402
|
+
}
|
|
272
403
|
// Public method for testing purposes - allows tests to verify parameter generation logic
|
|
273
404
|
// Always includes optional properties to test full schema
|
|
274
405
|
generateTestInput(schema) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SecurityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/SecurityAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,kBAAkB,EAInB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAU9D,qBAAa,gBAAiB,SAAQ,YAAY;IAC1C,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAuFrE;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAkC7B;;;;OAIG;YACW,yBAAyB;
|
|
1
|
+
{"version":3,"file":"SecurityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/SecurityAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,kBAAkB,EAInB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAU9D,qBAAa,gBAAiB,SAAQ,YAAY;IAC1C,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAuFrE;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAkC7B;;;;OAIG;YACW,yBAAyB;IAuKvC;;;;OAIG;YACW,qBAAqB;IA2JnC;;OAEG;YACW,WAAW;IA2HzB;;;;;OAKG;IACH,OAAO,CAAC,iBAAiB;IAkDzB;;;OAGG;IACH,OAAO,CAAC,8BAA8B;IAmDtC;;OAEG;IACH,OAAO,CAAC,aAAa;IA+BrB;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAgClC;;;OAGG;IACH,OAAO,CAAC,eAAe;IA6HvB;;;;;;;OAOG;IACH,OAAO,CAAC,qBAAqB;IAiE7B;;;;;;;;;OASG;IACH,OAAO,CAAC,oBAAoB;IAqC5B;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB;IAsB3B;;;;;;;OAOG;IACH,OAAO,CAAC,oBAAoB;IAkC5B;;OAEG;YACW,+BAA+B;IAiC7C;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAYjC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA0B/B;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAuI3B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAsB5B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,oBAAoB;IAiM5B;;;;;;OAMG;IACH,OAAO,CAAC,wBAAwB;IAiDhC;;;OAGG;IACH,OAAO,CAAC,wBAAwB;IA8BhC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAW9B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAO1B,OAAO,CAAC,oBAAoB;IAoE5B;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;;OAGG;IACH,OAAO,CAAC,eAAe;IASvB;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAiB9B;;;OAGG;IACH,OAAO,CAAC,kBAAkB;CAmB3B"}
|
|
@@ -175,6 +175,19 @@ export class SecurityAssessor extends BaseAssessor {
|
|
|
175
175
|
toolResults.push(result);
|
|
176
176
|
if (result.vulnerable) {
|
|
177
177
|
this.log(`🚨 VULNERABILITY: ${tool.name} - ${attackPattern.attackName} (${payload.payloadType}: ${payload.description})`);
|
|
178
|
+
// Emit real-time vulnerability_found event
|
|
179
|
+
if (context.onProgress) {
|
|
180
|
+
context.onProgress({
|
|
181
|
+
type: "vulnerability_found",
|
|
182
|
+
tool: tool.name,
|
|
183
|
+
pattern: attackPattern.attackName,
|
|
184
|
+
confidence: result.confidence || "medium",
|
|
185
|
+
evidence: result.evidence || "Vulnerability detected",
|
|
186
|
+
riskLevel: payload.riskLevel,
|
|
187
|
+
requiresReview: result.requiresManualReview || false,
|
|
188
|
+
payload: payload.payload,
|
|
189
|
+
});
|
|
190
|
+
}
|
|
178
191
|
}
|
|
179
192
|
}
|
|
180
193
|
catch (error) {
|
|
@@ -285,6 +298,19 @@ export class SecurityAssessor extends BaseAssessor {
|
|
|
285
298
|
results.push(result);
|
|
286
299
|
if (result.vulnerable) {
|
|
287
300
|
this.log(`🚨 VULNERABILITY: ${tool.name} - ${attackPattern.attackName}`);
|
|
301
|
+
// Emit real-time vulnerability_found event
|
|
302
|
+
if (context.onProgress) {
|
|
303
|
+
context.onProgress({
|
|
304
|
+
type: "vulnerability_found",
|
|
305
|
+
tool: tool.name,
|
|
306
|
+
pattern: attackPattern.attackName,
|
|
307
|
+
confidence: result.confidence || "medium",
|
|
308
|
+
evidence: result.evidence || "Vulnerability detected",
|
|
309
|
+
riskLevel: payload.riskLevel,
|
|
310
|
+
requiresReview: result.requiresManualReview || false,
|
|
311
|
+
payload: payload.payload,
|
|
312
|
+
});
|
|
313
|
+
}
|
|
288
314
|
}
|
|
289
315
|
}
|
|
290
316
|
catch (error) {
|
|
@@ -76,6 +76,10 @@ export declare class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
76
76
|
* MCP SDK may have annotations in different locations
|
|
77
77
|
*/
|
|
78
78
|
private extractAnnotations;
|
|
79
|
+
/**
|
|
80
|
+
* Extract parameters from tool input schema for event emission
|
|
81
|
+
*/
|
|
82
|
+
private extractToolParams;
|
|
79
83
|
/**
|
|
80
84
|
* Infer expected behavior from tool name and description
|
|
81
85
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ToolAnnotationAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ToolAnnotationAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EACV,wBAAwB,EACxB,oBAAoB,
|
|
1
|
+
{"version":3,"file":"ToolAnnotationAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ToolAnnotationAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EACV,wBAAwB,EACxB,oBAAoB,EAGrB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAEhE;;GAEG;AACH,MAAM,WAAW,4BAA6B,SAAQ,oBAAoB;IACxE,eAAe,CAAC,EAAE;QAChB,gBAAgB,EAAE,OAAO,CAAC;QAC1B,mBAAmB,EAAE,OAAO,CAAC;QAC7B,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,oBAAoB,EAAE;YACpB,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,eAAe,CAAC,EAAE,OAAO,CAAC;YAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;SAC1B,CAAC;QACF,oBAAoB,EAAE,OAAO,CAAC;QAC9B,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,MAAM,EAAE,iBAAiB,GAAG,eAAe,CAAC;KAC7C,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gCAAiC,SAAQ,wBAAwB;IAChF,WAAW,EAAE,4BAA4B,EAAE,CAAC;IAC5C,cAAc,EAAE,OAAO,CAAC;IACxB,2BAA2B,EAAE,4BAA4B,EAAE,CAAC;CAC7D;AAwED,qBAAa,sBAAuB,SAAQ,YAAY;IACtD,OAAO,CAAC,YAAY,CAAC,CAAmB;IAExC;;OAEG;IACH,eAAe,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAK/C;;OAEG;IACH,eAAe,IAAI,OAAO;IAO1B;;OAEG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,wBAAwB,GAAG,gCAAgC,CAAC;IAsMvE;;OAEG;YACW,0BAA0B;IA+IxC;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAiCnC;;OAEG;IACH,OAAO,CAAC,+BAA+B;IAoFvC;;OAEG;IACH,OAAO,CAAC,UAAU;IAqElB;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAyC1B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAuBzB;;OAEG;IACH,OAAO,CAAC,aAAa;IA0ErB;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAsCjC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAmC3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CA2ChC"}
|
|
@@ -151,11 +151,68 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
151
151
|
misalignedAnnotationsCount++;
|
|
152
152
|
}
|
|
153
153
|
}
|
|
154
|
-
|
|
154
|
+
const latestResult = toolResults[toolResults.length - 1];
|
|
155
|
+
if (latestResult.hasAnnotations) {
|
|
155
156
|
annotatedCount++;
|
|
156
157
|
}
|
|
157
158
|
else {
|
|
158
159
|
missingAnnotationsCount++;
|
|
160
|
+
// Emit annotation_missing event with tool details
|
|
161
|
+
if (context.onProgress && latestResult.inferredBehavior) {
|
|
162
|
+
const annotations = this.extractAnnotations(tool);
|
|
163
|
+
context.onProgress({
|
|
164
|
+
type: "annotation_missing",
|
|
165
|
+
tool: tool.name,
|
|
166
|
+
title: annotations.title,
|
|
167
|
+
description: tool.description,
|
|
168
|
+
parameters: this.extractToolParams(tool.inputSchema),
|
|
169
|
+
inferredBehavior: {
|
|
170
|
+
expectedReadOnly: latestResult.inferredBehavior.expectedReadOnly,
|
|
171
|
+
expectedDestructive: latestResult.inferredBehavior.expectedDestructive,
|
|
172
|
+
reason: latestResult.inferredBehavior.reason,
|
|
173
|
+
},
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
// Emit annotation_misaligned events for each misalignment
|
|
178
|
+
if (context.onProgress && latestResult.inferredBehavior) {
|
|
179
|
+
const annotations = latestResult.annotations;
|
|
180
|
+
const inferred = latestResult.inferredBehavior;
|
|
181
|
+
const confidence = latestResult.claudeInference?.confidence ?? 50;
|
|
182
|
+
const toolParams = this.extractToolParams(tool.inputSchema);
|
|
183
|
+
const toolAnnotations = this.extractAnnotations(tool);
|
|
184
|
+
// Check readOnlyHint misalignment
|
|
185
|
+
if (annotations?.readOnlyHint !== undefined &&
|
|
186
|
+
annotations.readOnlyHint !== inferred.expectedReadOnly) {
|
|
187
|
+
context.onProgress({
|
|
188
|
+
type: "annotation_misaligned",
|
|
189
|
+
tool: tool.name,
|
|
190
|
+
title: toolAnnotations.title,
|
|
191
|
+
description: tool.description,
|
|
192
|
+
parameters: toolParams,
|
|
193
|
+
field: "readOnlyHint",
|
|
194
|
+
actual: annotations.readOnlyHint,
|
|
195
|
+
expected: inferred.expectedReadOnly,
|
|
196
|
+
confidence,
|
|
197
|
+
reason: `Tool has readOnlyHint=${annotations.readOnlyHint}, but ${inferred.reason}`,
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
// Check destructiveHint misalignment
|
|
201
|
+
if (annotations?.destructiveHint !== undefined &&
|
|
202
|
+
annotations.destructiveHint !== inferred.expectedDestructive) {
|
|
203
|
+
context.onProgress({
|
|
204
|
+
type: "annotation_misaligned",
|
|
205
|
+
tool: tool.name,
|
|
206
|
+
title: toolAnnotations.title,
|
|
207
|
+
description: tool.description,
|
|
208
|
+
parameters: toolParams,
|
|
209
|
+
field: "destructiveHint",
|
|
210
|
+
actual: annotations.destructiveHint,
|
|
211
|
+
expected: inferred.expectedDestructive,
|
|
212
|
+
confidence,
|
|
213
|
+
reason: `Tool has destructiveHint=${annotations.destructiveHint}, but ${inferred.reason}`,
|
|
214
|
+
});
|
|
215
|
+
}
|
|
159
216
|
}
|
|
160
217
|
}
|
|
161
218
|
const status = this.determineAnnotationStatus(toolResults, context.tools.length);
|
|
@@ -452,6 +509,29 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
452
509
|
openWorldHint,
|
|
453
510
|
};
|
|
454
511
|
}
|
|
512
|
+
/**
|
|
513
|
+
* Extract parameters from tool input schema for event emission
|
|
514
|
+
*/
|
|
515
|
+
extractToolParams(schema) {
|
|
516
|
+
if (!schema || typeof schema !== "object")
|
|
517
|
+
return [];
|
|
518
|
+
const s = schema;
|
|
519
|
+
if (!s.properties || typeof s.properties !== "object")
|
|
520
|
+
return [];
|
|
521
|
+
const required = new Set(Array.isArray(s.required) ? s.required : []);
|
|
522
|
+
const properties = s.properties;
|
|
523
|
+
return Object.entries(properties).map(([name, prop]) => {
|
|
524
|
+
const param = {
|
|
525
|
+
name,
|
|
526
|
+
type: prop.type || "any",
|
|
527
|
+
required: required.has(name),
|
|
528
|
+
};
|
|
529
|
+
if (prop.description) {
|
|
530
|
+
param.description = prop.description;
|
|
531
|
+
}
|
|
532
|
+
return param;
|
|
533
|
+
});
|
|
534
|
+
}
|
|
455
535
|
/**
|
|
456
536
|
* Infer expected behavior from tool name and description
|
|
457
537
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bryan-thompson/inspector-assessment-client",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.11.1",
|
|
4
4
|
"description": "Client-side application for the Enhanced MCP Inspector with assessment capabilities",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Bryan Thompson <bryan@triepod.ai>",
|