@bryan-thompson/inspector-assessment-client 1.22.8 → 1.22.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/lib/moduleScoring.d.ts +4 -1
- package/lib/lib/moduleScoring.d.ts.map +1 -1
- package/lib/lib/moduleScoring.js +4 -1
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +4 -1
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ToolAnnotationAssessor.js +13 -4
- package/package.json +1 -1
|
@@ -13,8 +13,11 @@ export declare function normalizeModuleKey(moduleName: string): string;
|
|
|
13
13
|
* Calculate module score from assessment result.
|
|
14
14
|
* Handles different result shapes from different assessors.
|
|
15
15
|
* This centralizes score calculation logic used by the orchestrator.
|
|
16
|
+
*
|
|
17
|
+
* Returns null for undefined/missing results (e.g., skipped modules via --skip-modules).
|
|
18
|
+
* Callers should check for null and skip emission/display for skipped modules.
|
|
16
19
|
*/
|
|
17
|
-
export declare function calculateModuleScore(result: unknown): number;
|
|
20
|
+
export declare function calculateModuleScore(result: unknown): number | null;
|
|
18
21
|
/**
|
|
19
22
|
* Current inspector-assessment version for event compatibility checking.
|
|
20
23
|
* This should match the version in package.json.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"moduleScoring.d.ts","sourceRoot":"","sources":["../../src/lib/moduleScoring.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAE7D;AAED
|
|
1
|
+
{"version":3,"file":"moduleScoring.d.ts","sourceRoot":"","sources":["../../src/lib/moduleScoring.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAE7D;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CA6BnE;AAED;;;GAGG;AACH,eAAO,MAAM,iBAAiB,WAAW,CAAC"}
|
package/lib/lib/moduleScoring.js
CHANGED
|
@@ -15,10 +15,13 @@ export function normalizeModuleKey(moduleName) {
|
|
|
15
15
|
* Calculate module score from assessment result.
|
|
16
16
|
* Handles different result shapes from different assessors.
|
|
17
17
|
* This centralizes score calculation logic used by the orchestrator.
|
|
18
|
+
*
|
|
19
|
+
* Returns null for undefined/missing results (e.g., skipped modules via --skip-modules).
|
|
20
|
+
* Callers should check for null and skip emission/display for skipped modules.
|
|
18
21
|
*/
|
|
19
22
|
export function calculateModuleScore(result) {
|
|
20
23
|
if (!result || typeof result !== "object")
|
|
21
|
-
return
|
|
24
|
+
return null;
|
|
22
25
|
const r = result;
|
|
23
26
|
// ErrorHandling module - uses metrics.mcpComplianceScore
|
|
24
27
|
const metrics = r.metrics;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AssessmentOrchestrator.d.ts","sourceRoot":"","sources":["../../../src/services/assessment/AssessmentOrchestrator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,sBAAsB,EACtB,uBAAuB,EAGvB,kBAAkB,EAClB,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,IAAI,EACJ,2BAA2B,EAC5B,MAAM,oCAAoC,CAAC;AAiC5C,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EAEvB,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"AssessmentOrchestrator.d.ts","sourceRoot":"","sources":["../../../src/services/assessment/AssessmentOrchestrator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,sBAAsB,EACtB,uBAAuB,EAGvB,kBAAkB,EAClB,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,IAAI,EACJ,2BAA2B,EAC5B,MAAM,oCAAoC,CAAC;AAiC5C,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EAEvB,MAAM,wBAAwB,CAAC;AAmKhC;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,KAAK,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,KAAK,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IAClC,SAAS,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IAC3D,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC;AAED,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC5B,OAAO,CAAC,2BAA2B,CAAC,CAAC;IAC1C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,uBAAuB,CAAC;IAChC,UAAU,CAAC,EAAE;QACX,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;IAIF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAGtC,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAClC,WAAW,CAAC,EAAE,MAAM,CAAC;IAIrB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAG9B,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;IAC1B,iBAAiB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC1C,OAAO,CAAC,EAAE,SAAS,EAAE,CAAC;IACtB,kBAAkB,CAAC,EAAE,qBAAqB,CAAC;IAG3C,YAAY,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAChD,SAAS,CAAC,EAAE,CACV,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KACzB,OAAO,CAAC;QAAE,QAAQ,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC,CAAC;IAGrE,eAAe,CAAC,EAAE;QAChB,IAAI,EAAE,OAAO,GAAG,KAAK,GAAG,iBAAiB,CAAC;QAC1C,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,CAAC;IAIF,SAAS,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;CACnC;AAED,qBAAa,sBAAsB;IACjC,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,aAAa,CAAa;IAGlC,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,aAAa,CAAkB;IAGvC,OAAO,CAAC,qBAAqB,CAAC,CAAwB;IACtD,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,qBAAqB,CAAC,CAAwB;IACtD,OAAO,CAAC,qBAAqB,CAAC,CAAwB;IACtD,OAAO,CAAC,iBAAiB,CAAC,CAAoB;IAG9C,OAAO,CAAC,eAAe,CAAC,CAA4B;IAGpD,OAAO,CAAC,qBAAqB,CAAC,CAAwB;IACtD,OAAO,CAAC,sBAAsB,CAAC,CAAyB;IACxD,OAAO,CAAC,2BAA2B,CAAC,CAA8B;IAClE,OAAO,CAAC,0BAA0B,CAAC,CAA6B;IAChE,OAAO,CAAC,mBAAmB,CAAC,CAAsB;IAClD,OAAO,CAAC,0BAA0B,CAAC,CAA6B;IAChE,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAG5C,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,uBAAuB,CAAC,CAAkC;gBAEtD,MAAM,GAAE,OAAO,CAAC,uBAAuB,CAAM;IAgGzD;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAiB9B;;;OAGG;IACH,gBAAgB,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,sBAAsB,CAAC,GAAG,IAAI;IAqBhE;;OAEG;IACH,eAAe,IAAI,OAAO;IAI1B;;OAEG;IACH,eAAe,IAAI,gBAAgB,GAAG,SAAS;IAI/C;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAqC1B;;OAEG;IACG,iBAAiB,CACrB,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,sBAAsB,CAAC;IA2gBlC;;OAEG;IACG,MAAM,CACV,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,IAAI,EAAE,EACb,QAAQ,EAAE,CACR,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC5B,OAAO,CAAC,2BAA2B,CAAC,EACzC,UAAU,CAAC,EAAE,GAAG,EAChB,aAAa,CAAC,EAAE,MAAM,EACtB,WAAW,CAAC,EAAE,GAAG,GAChB,OAAO,CAAC,sBAAsB,CAAC;IAclC,OAAO,CAAC,qBAAqB;IAsE7B,OAAO,CAAC,sBAAsB;IAoB9B,OAAO,CAAC,eAAe;IA8DvB,OAAO,CAAC,uBAAuB;IAc/B;;OAEG;IACH,SAAS,IAAI,uBAAuB;IAIpC;;OAEG;IACH,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,uBAAuB,CAAC,GAAG,IAAI;CAG7D"}
|
|
@@ -54,9 +54,12 @@ function emitModuleStartedEvent(moduleName, estimatedTests, toolCount) {
|
|
|
54
54
|
* For AUP module, includes enriched violation data for Claude analysis.
|
|
55
55
|
*/
|
|
56
56
|
function emitModuleProgress(moduleName, status, result, testsRun = 0) {
|
|
57
|
-
const moduleKey = normalizeModuleKey(moduleName);
|
|
58
57
|
// Calculate score using shared helper
|
|
59
58
|
const score = calculateModuleScore(result);
|
|
59
|
+
// Don't emit events for skipped modules (null score means module wasn't run)
|
|
60
|
+
if (score === null)
|
|
61
|
+
return;
|
|
62
|
+
const moduleKey = normalizeModuleKey(moduleName);
|
|
60
63
|
// Calculate duration from module start time
|
|
61
64
|
const startTime = moduleStartTimes.get(moduleKey);
|
|
62
65
|
const duration = startTime ? Date.now() - startTime : 0;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ToolAnnotationAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ToolAnnotationAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EACV,wBAAwB,EACxB,oBAAoB,EAKpB,uBAAuB,EAExB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,EAK9B,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"ToolAnnotationAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ToolAnnotationAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EACV,wBAAwB,EACxB,oBAAoB,EAKpB,uBAAuB,EAExB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,EAK9B,MAAM,8BAA8B,CAAC;AAiYtC;;GAEG;AACH,MAAM,WAAW,4BAA6B,SAAQ,oBAAoB;IACxE,eAAe,CAAC,EAAE;QAChB,gBAAgB,EAAE,OAAO,CAAC;QAC1B,mBAAmB,EAAE,OAAO,CAAC;QAC7B,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,oBAAoB,EAAE;YACpB,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,eAAe,CAAC,EAAE,OAAO,CAAC;YAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;SAC1B,CAAC;QACF,oBAAoB,EAAE,OAAO,CAAC;QAC9B,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,MAAM,EAAE,iBAAiB,GAAG,eAAe,CAAC;KAC7C,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gCAAiC,SAAQ,wBAAwB;IAChF,WAAW,EAAE,4BAA4B,EAAE,CAAC;IAC5C,cAAc,EAAE,OAAO,CAAC;IACxB,2BAA2B,EAAE,4BAA4B,EAAE,CAAC;CAC7D;AAKD,qBAAa,sBAAuB,SAAQ,YAAY;IACtD,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,kBAAkB,CAAC,CAA2B;gBAE1C,MAAM,EAAE,uBAAuB;IAM3C;;OAEG;IACH,qBAAqB,IAAI,wBAAwB,GAAG,SAAS;IAI7D;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,gBAAgB,GAAG,IAAI;IAK7C;;OAEG;IACH,eAAe,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAK/C;;OAEG;IACH,eAAe,IAAI,OAAO;IAO1B;;OAEG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,wBAAwB,GAAG,gCAAgC,CAAC;IA+UvE;;OAEG;YACW,0BAA0B;IA+IxC;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAiCnC;;OAEG;IACH,OAAO,CAAC,+BAA+B;IAoFvC;;;;OAIG;IACH,OAAO,CAAC,UAAU;IA0IlB;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IA2DnC;;;;;;;;;OASG;IACH,OAAO,CAAC,kBAAkB;IAyE1B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAuBzB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAkKrB;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IA0DjC;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAiDxB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAmC3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CA2ChC"}
|
|
@@ -92,6 +92,13 @@ function containsKeyword(toolName, keywords) {
|
|
|
92
92
|
}
|
|
93
93
|
return null;
|
|
94
94
|
}
|
|
95
|
+
/**
|
|
96
|
+
* Type guard for confidence levels that warrant event emission or status changes.
|
|
97
|
+
* Uses positive check for acceptable levels (safer than !== "low" if new levels added).
|
|
98
|
+
*/
|
|
99
|
+
function isActionableConfidence(confidence) {
|
|
100
|
+
return confidence === "high" || confidence === "medium";
|
|
101
|
+
}
|
|
95
102
|
/**
|
|
96
103
|
* Detect high-confidence annotation deception
|
|
97
104
|
* Returns misalignment info if obvious deception detected, null otherwise
|
|
@@ -554,7 +561,8 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
554
561
|
reason: inferred.reason,
|
|
555
562
|
});
|
|
556
563
|
}
|
|
557
|
-
else if (!inferred.isAmbiguous &&
|
|
564
|
+
else if (!inferred.isAmbiguous &&
|
|
565
|
+
isActionableConfidence(inferred.confidence)) {
|
|
558
566
|
// Emit misaligned only for medium/high-confidence mismatches
|
|
559
567
|
// When inference is low-confidence/ambiguous, trust explicit annotation
|
|
560
568
|
context.onProgress({
|
|
@@ -592,7 +600,8 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
592
600
|
reason: inferred.reason,
|
|
593
601
|
});
|
|
594
602
|
}
|
|
595
|
-
else if (!inferred.isAmbiguous &&
|
|
603
|
+
else if (!inferred.isAmbiguous &&
|
|
604
|
+
isActionableConfidence(inferred.confidence)) {
|
|
596
605
|
// Emit misaligned only for medium/high-confidence mismatches
|
|
597
606
|
// When inference is low-confidence/ambiguous, trust explicit annotation
|
|
598
607
|
context.onProgress({
|
|
@@ -898,7 +907,7 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
898
907
|
// Note: High-confidence deception detection (exec/install keywords)
|
|
899
908
|
// is handled in the `deception` block above, not here
|
|
900
909
|
if (!inferredBehavior.isAmbiguous &&
|
|
901
|
-
inferredBehavior.confidence
|
|
910
|
+
isActionableConfidence(inferredBehavior.confidence)) {
|
|
902
911
|
// Medium/high confidence mismatch: MISALIGNED
|
|
903
912
|
alignmentStatus = "MISALIGNED";
|
|
904
913
|
if (readOnlyMismatch) {
|
|
@@ -917,7 +926,7 @@ export class ToolAnnotationAssessor extends BaseAssessor {
|
|
|
917
926
|
}
|
|
918
927
|
// Check for destructive tools without explicit hint (only for high-confidence patterns)
|
|
919
928
|
if (inferredBehavior.expectedDestructive &&
|
|
920
|
-
inferredBehavior.confidence
|
|
929
|
+
isActionableConfidence(inferredBehavior.confidence) &&
|
|
921
930
|
annotations.destructiveHint !== true) {
|
|
922
931
|
issues.push("Tool appears destructive but destructiveHint is not set to true");
|
|
923
932
|
recommendations.push(`Set destructiveHint=true for ${tool.name} - this tool appears to perform destructive operations`);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bryan-thompson/inspector-assessment-client",
|
|
3
|
-
"version": "1.22.
|
|
3
|
+
"version": "1.22.9",
|
|
4
4
|
"description": "Client-side application for the Enhanced MCP Inspector with assessment capabilities",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Bryan Thompson <bryan@triepod.ai>",
|