@bryan-thompson/inspector-assessment-client 1.13.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-D8KW6pFf.js → OAuthCallback-DaKwjxdn.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-D15nNAOl.js → OAuthDebugCallback-HiI2IPgB.js} +1 -1
- package/dist/assets/{index-cVkEgqCc.js → index-BAJG90Yd.js} +20 -8
- package/dist/assets/{index-Cuc9GxjD.css → index-BdXNC65t.css} +3 -0
- package/dist/index.html +2 -2
- package/lib/lib/assessmentDiffer.d.ts +79 -0
- package/lib/lib/assessmentDiffer.d.ts.map +1 -0
- package/lib/lib/assessmentDiffer.js +289 -0
- package/lib/lib/assessmentTypes.d.ts +69 -0
- package/lib/lib/assessmentTypes.d.ts.map +1 -1
- package/lib/lib/assessmentTypes.js +10 -0
- package/lib/lib/reportFormatters/DiffReportFormatter.d.ts +10 -0
- package/lib/lib/reportFormatters/DiffReportFormatter.d.ts.map +1 -0
- package/lib/lib/reportFormatters/DiffReportFormatter.js +177 -0
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +1 -0
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +22 -1
- package/lib/services/assessment/modules/AuthenticationAssessor.d.ts +48 -0
- package/lib/services/assessment/modules/AuthenticationAssessor.d.ts.map +1 -0
- package/lib/services/assessment/modules/AuthenticationAssessor.js +270 -0
- package/lib/services/assessment/modules/ExternalAPIScannerAssessor.d.ts +58 -0
- package/lib/services/assessment/modules/ExternalAPIScannerAssessor.d.ts.map +1 -0
- package/lib/services/assessment/modules/ExternalAPIScannerAssessor.js +248 -0
- package/lib/services/assessment/modules/FunctionalityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/FunctionalityAssessor.js +7 -0
- package/lib/services/assessment/modules/ManifestValidationAssessor.d.ts +4 -0
- package/lib/services/assessment/modules/ManifestValidationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ManifestValidationAssessor.js +118 -4
- package/lib/services/assessment/modules/index.d.ts +1 -0
- package/lib/services/assessment/modules/index.d.ts.map +1 -1
- package/lib/services/assessment/modules/index.js +1 -0
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { u as useToast, r as reactExports, j as jsxRuntimeExports, p as parseOAuthCallbackParams, g as generateOAuthErrorDescription, S as SESSION_KEYS, I as InspectorOAuthClientProvider, a as auth } from "./index-
|
|
1
|
+
import { u as useToast, r as reactExports, j as jsxRuntimeExports, p as parseOAuthCallbackParams, g as generateOAuthErrorDescription, S as SESSION_KEYS, I as InspectorOAuthClientProvider, a as auth } from "./index-BAJG90Yd.js";
|
|
2
2
|
const OAuthCallback = ({ onConnect }) => {
|
|
3
3
|
const { toast } = useToast();
|
|
4
4
|
const hasProcessedRef = reactExports.useRef(false);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { r as reactExports, S as SESSION_KEYS, p as parseOAuthCallbackParams, j as jsxRuntimeExports, g as generateOAuthErrorDescription } from "./index-
|
|
1
|
+
import { r as reactExports, S as SESSION_KEYS, p as parseOAuthCallbackParams, j as jsxRuntimeExports, g as generateOAuthErrorDescription } from "./index-BAJG90Yd.js";
|
|
2
2
|
const OAuthDebugCallback = ({ onConnect }) => {
|
|
3
3
|
reactExports.useEffect(() => {
|
|
4
4
|
let isProcessed = false;
|
|
@@ -16320,7 +16320,7 @@ object({
|
|
|
16320
16320
|
token_type_hint: string().optional()
|
|
16321
16321
|
}).strip();
|
|
16322
16322
|
const name = "@bryan-thompson/inspector-assessment-client";
|
|
16323
|
-
const version$1 = "1.
|
|
16323
|
+
const version$1 = "1.14.0";
|
|
16324
16324
|
const packageJson = {
|
|
16325
16325
|
name,
|
|
16326
16326
|
version: version$1
|
|
@@ -48863,7 +48863,7 @@ const useTheme = () => {
|
|
|
48863
48863
|
[theme, setThemeWithSideEffect]
|
|
48864
48864
|
);
|
|
48865
48865
|
};
|
|
48866
|
-
const version = "1.
|
|
48866
|
+
const version = "1.14.0";
|
|
48867
48867
|
var [createTooltipContext] = createContextScope("Tooltip", [
|
|
48868
48868
|
createPopperScope
|
|
48869
48869
|
]);
|
|
@@ -51555,7 +51555,9 @@ const DEFAULT_ASSESSMENT_CONFIG = {
|
|
|
51555
51555
|
toolAnnotations: false,
|
|
51556
51556
|
prohibitedLibraries: false,
|
|
51557
51557
|
manifestValidation: false,
|
|
51558
|
-
portability: false
|
|
51558
|
+
portability: false,
|
|
51559
|
+
externalAPIScanner: false,
|
|
51560
|
+
authentication: false
|
|
51559
51561
|
}
|
|
51560
51562
|
};
|
|
51561
51563
|
const REVIEWER_MODE_CONFIG = {
|
|
@@ -51594,7 +51596,9 @@ const REVIEWER_MODE_CONFIG = {
|
|
|
51594
51596
|
toolAnnotations: false,
|
|
51595
51597
|
prohibitedLibraries: false,
|
|
51596
51598
|
manifestValidation: false,
|
|
51597
|
-
portability: false
|
|
51599
|
+
portability: false,
|
|
51600
|
+
externalAPIScanner: false,
|
|
51601
|
+
authentication: false
|
|
51598
51602
|
}
|
|
51599
51603
|
};
|
|
51600
51604
|
const DEVELOPER_MODE_CONFIG = {
|
|
@@ -51630,7 +51634,9 @@ const DEVELOPER_MODE_CONFIG = {
|
|
|
51630
51634
|
toolAnnotations: true,
|
|
51631
51635
|
prohibitedLibraries: true,
|
|
51632
51636
|
manifestValidation: true,
|
|
51633
|
-
portability: true
|
|
51637
|
+
portability: true,
|
|
51638
|
+
externalAPIScanner: true,
|
|
51639
|
+
authentication: true
|
|
51634
51640
|
}
|
|
51635
51641
|
};
|
|
51636
51642
|
class BaseAssessor {
|
|
@@ -54774,6 +54780,11 @@ class FunctionalityAssessor extends BaseAssessor {
|
|
|
54774
54780
|
workingTools,
|
|
54775
54781
|
brokenTools
|
|
54776
54782
|
);
|
|
54783
|
+
const tools = context.tools.map((t) => ({
|
|
54784
|
+
name: t.name,
|
|
54785
|
+
description: t.description,
|
|
54786
|
+
inputSchema: t.inputSchema
|
|
54787
|
+
}));
|
|
54777
54788
|
return {
|
|
54778
54789
|
totalTools,
|
|
54779
54790
|
testedTools,
|
|
@@ -54782,7 +54793,8 @@ class FunctionalityAssessor extends BaseAssessor {
|
|
|
54782
54793
|
coveragePercentage,
|
|
54783
54794
|
status,
|
|
54784
54795
|
explanation,
|
|
54785
|
-
toolResults
|
|
54796
|
+
toolResults,
|
|
54797
|
+
tools
|
|
54786
54798
|
};
|
|
54787
54799
|
}
|
|
54788
54800
|
async testTool(tool, callTool) {
|
|
@@ -61865,13 +61877,13 @@ const App = () => {
|
|
|
61865
61877
|
) });
|
|
61866
61878
|
if (window.location.pathname === "/oauth/callback") {
|
|
61867
61879
|
const OAuthCallback = React.lazy(
|
|
61868
|
-
() => __vitePreload(() => import("./OAuthCallback-
|
|
61880
|
+
() => __vitePreload(() => import("./OAuthCallback-DaKwjxdn.js"), true ? [] : void 0)
|
|
61869
61881
|
);
|
|
61870
61882
|
return /* @__PURE__ */ jsxRuntimeExports.jsx(reactExports.Suspense, { fallback: /* @__PURE__ */ jsxRuntimeExports.jsx("div", { children: "Loading..." }), children: /* @__PURE__ */ jsxRuntimeExports.jsx(OAuthCallback, { onConnect: onOAuthConnect }) });
|
|
61871
61883
|
}
|
|
61872
61884
|
if (window.location.pathname === "/oauth/callback/debug") {
|
|
61873
61885
|
const OAuthDebugCallback = React.lazy(
|
|
61874
|
-
() => __vitePreload(() => import("./OAuthDebugCallback-
|
|
61886
|
+
() => __vitePreload(() => import("./OAuthDebugCallback-HiI2IPgB.js"), true ? [] : void 0)
|
|
61875
61887
|
);
|
|
61876
61888
|
return /* @__PURE__ */ jsxRuntimeExports.jsx(reactExports.Suspense, { fallback: /* @__PURE__ */ jsxRuntimeExports.jsx("div", { children: "Loading..." }), children: /* @__PURE__ */ jsxRuntimeExports.jsx(OAuthDebugCallback, { onConnect: onOAuthDebugConnect }) });
|
|
61877
61889
|
}
|
package/dist/index.html
CHANGED
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
<link rel="icon" type="image/svg+xml" href="/mcp.svg" />
|
|
6
6
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
7
7
|
<title>MCP Inspector</title>
|
|
8
|
-
<script type="module" crossorigin src="/assets/index-
|
|
9
|
-
<link rel="stylesheet" crossorigin href="/assets/index-
|
|
8
|
+
<script type="module" crossorigin src="/assets/index-BAJG90Yd.js"></script>
|
|
9
|
+
<link rel="stylesheet" crossorigin href="/assets/index-BdXNC65t.css">
|
|
10
10
|
</head>
|
|
11
11
|
<body>
|
|
12
12
|
<div id="root" class="w-full"></div>
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assessment Differ
|
|
3
|
+
* Compares two assessment runs to detect regressions or improvements.
|
|
4
|
+
*
|
|
5
|
+
* Use Cases:
|
|
6
|
+
* - Compare bundle v1.0 vs v1.1
|
|
7
|
+
* - Track assessment changes over time
|
|
8
|
+
* - CI/CD regression detection
|
|
9
|
+
*/
|
|
10
|
+
import type { MCPDirectoryAssessment, AssessmentStatus } from "./assessmentTypes.js";
|
|
11
|
+
/**
|
|
12
|
+
* Change direction for a metric
|
|
13
|
+
*/
|
|
14
|
+
export type ChangeDirection = "improved" | "regressed" | "unchanged";
|
|
15
|
+
/**
|
|
16
|
+
* Detailed change information for a single module
|
|
17
|
+
*/
|
|
18
|
+
export interface ModuleChange {
|
|
19
|
+
module: string;
|
|
20
|
+
baselineStatus: AssessmentStatus;
|
|
21
|
+
currentStatus: AssessmentStatus;
|
|
22
|
+
baselineScore?: number;
|
|
23
|
+
currentScore?: number;
|
|
24
|
+
change: ChangeDirection;
|
|
25
|
+
details: string;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Security-specific delta tracking
|
|
29
|
+
*/
|
|
30
|
+
export interface SecurityDelta {
|
|
31
|
+
newVulnerabilities: string[];
|
|
32
|
+
fixedVulnerabilities: string[];
|
|
33
|
+
netChange: number;
|
|
34
|
+
baselineCount: number;
|
|
35
|
+
currentCount: number;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Functionality-specific delta tracking
|
|
39
|
+
*/
|
|
40
|
+
export interface FunctionalityDelta {
|
|
41
|
+
newBrokenTools: string[];
|
|
42
|
+
fixedTools: string[];
|
|
43
|
+
netChange: number;
|
|
44
|
+
baselineWorking: number;
|
|
45
|
+
currentWorking: number;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Full assessment comparison result
|
|
49
|
+
*/
|
|
50
|
+
export interface AssessmentDiff {
|
|
51
|
+
serverName: string;
|
|
52
|
+
baseline: {
|
|
53
|
+
version?: string;
|
|
54
|
+
date: string;
|
|
55
|
+
assessorVersion: string;
|
|
56
|
+
};
|
|
57
|
+
current: {
|
|
58
|
+
version?: string;
|
|
59
|
+
date: string;
|
|
60
|
+
assessorVersion: string;
|
|
61
|
+
};
|
|
62
|
+
summary: {
|
|
63
|
+
overallChange: ChangeDirection;
|
|
64
|
+
baselineStatus: AssessmentStatus;
|
|
65
|
+
currentStatus: AssessmentStatus;
|
|
66
|
+
modulesImproved: number;
|
|
67
|
+
modulesRegressed: number;
|
|
68
|
+
modulesUnchanged: number;
|
|
69
|
+
};
|
|
70
|
+
moduleChanges: ModuleChange[];
|
|
71
|
+
securityDelta: SecurityDelta;
|
|
72
|
+
functionalityDelta: FunctionalityDelta;
|
|
73
|
+
recommendations: string[];
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Compare two assessment runs
|
|
77
|
+
*/
|
|
78
|
+
export declare function compareAssessments(baseline: MCPDirectoryAssessment, current: MCPDirectoryAssessment): AssessmentDiff;
|
|
79
|
+
//# sourceMappingURL=assessmentDiffer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assessmentDiffer.d.ts","sourceRoot":"","sources":["../../src/lib/assessmentDiffer.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EACV,sBAAsB,EACtB,gBAAgB,EACjB,MAAM,mBAAmB,CAAC;AAE3B;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,UAAU,GAAG,WAAW,GAAG,WAAW,CAAC;AAErE;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,gBAAgB,CAAC;IACjC,aAAa,EAAE,gBAAgB,CAAC;IAChC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,eAAe,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,oBAAoB,EAAE,MAAM,EAAE,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE;QACR,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,OAAO,EAAE;QACP,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,OAAO,EAAE;QACP,aAAa,EAAE,eAAe,CAAC;QAC/B,cAAc,EAAE,gBAAgB,CAAC;QACjC,aAAa,EAAE,gBAAgB,CAAC;QAChC,eAAe,EAAE,MAAM,CAAC;QACxB,gBAAgB,EAAE,MAAM,CAAC;QACzB,gBAAgB,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,aAAa,EAAE,aAAa,CAAC;IAC7B,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,sBAAsB,EAChC,OAAO,EAAE,sBAAsB,GAC9B,cAAc,CA4DhB"}
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assessment Differ
|
|
3
|
+
* Compares two assessment runs to detect regressions or improvements.
|
|
4
|
+
*
|
|
5
|
+
* Use Cases:
|
|
6
|
+
* - Compare bundle v1.0 vs v1.1
|
|
7
|
+
* - Track assessment changes over time
|
|
8
|
+
* - CI/CD regression detection
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Compare two assessment runs
|
|
12
|
+
*/
|
|
13
|
+
export function compareAssessments(baseline, current) {
|
|
14
|
+
const moduleChanges = compareModules(baseline, current);
|
|
15
|
+
const securityDelta = compareSecurityResults(baseline, current);
|
|
16
|
+
const functionalityDelta = compareFunctionalityResults(baseline, current);
|
|
17
|
+
const improved = moduleChanges.filter((m) => m.change === "improved").length;
|
|
18
|
+
const regressed = moduleChanges.filter((m) => m.change === "regressed").length;
|
|
19
|
+
const unchanged = moduleChanges.filter((m) => m.change === "unchanged").length;
|
|
20
|
+
// Determine overall change
|
|
21
|
+
let overallChange = "unchanged";
|
|
22
|
+
if (regressed > 0 ||
|
|
23
|
+
securityDelta.netChange > 0 ||
|
|
24
|
+
functionalityDelta.netChange < 0) {
|
|
25
|
+
overallChange = "regressed";
|
|
26
|
+
}
|
|
27
|
+
else if (improved > 0 ||
|
|
28
|
+
securityDelta.netChange < 0 ||
|
|
29
|
+
functionalityDelta.netChange > 0) {
|
|
30
|
+
overallChange = "improved";
|
|
31
|
+
}
|
|
32
|
+
const recommendations = generateDiffRecommendations(moduleChanges, securityDelta, functionalityDelta);
|
|
33
|
+
return {
|
|
34
|
+
serverName: current.serverName,
|
|
35
|
+
baseline: {
|
|
36
|
+
version: extractVersion(baseline),
|
|
37
|
+
date: baseline.assessmentDate,
|
|
38
|
+
assessorVersion: baseline.assessorVersion,
|
|
39
|
+
},
|
|
40
|
+
current: {
|
|
41
|
+
version: extractVersion(current),
|
|
42
|
+
date: current.assessmentDate,
|
|
43
|
+
assessorVersion: current.assessorVersion,
|
|
44
|
+
},
|
|
45
|
+
summary: {
|
|
46
|
+
overallChange,
|
|
47
|
+
baselineStatus: baseline.overallStatus,
|
|
48
|
+
currentStatus: current.overallStatus,
|
|
49
|
+
modulesImproved: improved,
|
|
50
|
+
modulesRegressed: regressed,
|
|
51
|
+
modulesUnchanged: unchanged,
|
|
52
|
+
},
|
|
53
|
+
moduleChanges,
|
|
54
|
+
securityDelta,
|
|
55
|
+
functionalityDelta,
|
|
56
|
+
recommendations,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Compare individual module results
|
|
61
|
+
*/
|
|
62
|
+
function compareModules(baseline, current) {
|
|
63
|
+
const changes = [];
|
|
64
|
+
// Compare core modules
|
|
65
|
+
const modules = [
|
|
66
|
+
{
|
|
67
|
+
name: "functionality",
|
|
68
|
+
getStatus: (a) => a.functionality?.status,
|
|
69
|
+
getScore: (a) => a.functionality
|
|
70
|
+
? (a.functionality.workingTools / a.functionality.totalTools) * 100
|
|
71
|
+
: undefined,
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
name: "security",
|
|
75
|
+
getStatus: (a) => a.security?.status,
|
|
76
|
+
getScore: (a) => a.security
|
|
77
|
+
? 100 -
|
|
78
|
+
(a.security.vulnerabilities.length /
|
|
79
|
+
Math.max(a.security.promptInjectionTests.length, 1)) *
|
|
80
|
+
100
|
|
81
|
+
: undefined,
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
name: "documentation",
|
|
85
|
+
getStatus: (a) => a.documentation?.status,
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
name: "errorHandling",
|
|
89
|
+
getStatus: (a) => a.errorHandling?.status,
|
|
90
|
+
getScore: (a) => a.errorHandling?.metrics?.mcpComplianceScore,
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
name: "usability",
|
|
94
|
+
getStatus: (a) => a.usability?.status,
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
name: "mcpSpecCompliance",
|
|
98
|
+
getStatus: (a) => a.mcpSpecCompliance?.status,
|
|
99
|
+
getScore: (a) => a.mcpSpecCompliance?.complianceScore,
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
name: "aupCompliance",
|
|
103
|
+
getStatus: (a) => a.aupCompliance?.status,
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
name: "toolAnnotations",
|
|
107
|
+
getStatus: (a) => a.toolAnnotations?.status,
|
|
108
|
+
},
|
|
109
|
+
{
|
|
110
|
+
name: "manifestValidation",
|
|
111
|
+
getStatus: (a) => a.manifestValidation?.status,
|
|
112
|
+
},
|
|
113
|
+
{
|
|
114
|
+
name: "portability",
|
|
115
|
+
getStatus: (a) => a.portability?.status,
|
|
116
|
+
},
|
|
117
|
+
];
|
|
118
|
+
for (const mod of modules) {
|
|
119
|
+
const baselineStatus = mod.getStatus(baseline);
|
|
120
|
+
const currentStatus = mod.getStatus(current);
|
|
121
|
+
// Skip if module wasn't run in either assessment
|
|
122
|
+
if (!baselineStatus && !currentStatus)
|
|
123
|
+
continue;
|
|
124
|
+
const baselineScore = mod.getScore?.(baseline);
|
|
125
|
+
const currentScore = mod.getScore?.(current);
|
|
126
|
+
const change = determineChange(baselineStatus || "NEED_MORE_INFO", currentStatus || "NEED_MORE_INFO", baselineScore, currentScore);
|
|
127
|
+
changes.push({
|
|
128
|
+
module: mod.name,
|
|
129
|
+
baselineStatus: baselineStatus || "NEED_MORE_INFO",
|
|
130
|
+
currentStatus: currentStatus || "NEED_MORE_INFO",
|
|
131
|
+
baselineScore,
|
|
132
|
+
currentScore,
|
|
133
|
+
change,
|
|
134
|
+
details: generateModuleDetails(mod.name, baselineStatus, currentStatus, baselineScore, currentScore),
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
return changes;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Determine if a module improved, regressed, or stayed the same
|
|
141
|
+
*/
|
|
142
|
+
function determineChange(baselineStatus, currentStatus, baselineScore, currentScore) {
|
|
143
|
+
// Status-based change
|
|
144
|
+
const statusOrder = {
|
|
145
|
+
FAIL: 0,
|
|
146
|
+
NEED_MORE_INFO: 1,
|
|
147
|
+
PASS: 2,
|
|
148
|
+
};
|
|
149
|
+
const baselineRank = statusOrder[baselineStatus];
|
|
150
|
+
const currentRank = statusOrder[currentStatus];
|
|
151
|
+
if (currentRank > baselineRank)
|
|
152
|
+
return "improved";
|
|
153
|
+
if (currentRank < baselineRank)
|
|
154
|
+
return "regressed";
|
|
155
|
+
// Score-based change (if statuses are equal)
|
|
156
|
+
if (baselineScore !== undefined && currentScore !== undefined) {
|
|
157
|
+
const scoreDiff = currentScore - baselineScore;
|
|
158
|
+
if (scoreDiff > 5)
|
|
159
|
+
return "improved"; // 5% threshold
|
|
160
|
+
if (scoreDiff < -5)
|
|
161
|
+
return "regressed";
|
|
162
|
+
}
|
|
163
|
+
return "unchanged";
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Generate human-readable details for a module change
|
|
167
|
+
*/
|
|
168
|
+
function generateModuleDetails(_moduleName, baselineStatus, currentStatus, baselineScore, currentScore) {
|
|
169
|
+
const parts = [];
|
|
170
|
+
if (baselineStatus !== currentStatus) {
|
|
171
|
+
parts.push(`${baselineStatus || "N/A"} → ${currentStatus || "N/A"}`);
|
|
172
|
+
}
|
|
173
|
+
if (baselineScore !== undefined && currentScore !== undefined) {
|
|
174
|
+
const diff = currentScore - baselineScore;
|
|
175
|
+
const sign = diff > 0 ? "+" : "";
|
|
176
|
+
parts.push(`Score: ${baselineScore.toFixed(1)}% → ${currentScore.toFixed(1)}% (${sign}${diff.toFixed(1)}%)`);
|
|
177
|
+
}
|
|
178
|
+
if (parts.length === 0) {
|
|
179
|
+
return "No change";
|
|
180
|
+
}
|
|
181
|
+
return parts.join(", ");
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Compare security results between assessments
|
|
185
|
+
*/
|
|
186
|
+
function compareSecurityResults(baseline, current) {
|
|
187
|
+
const baselineVulns = new Set(baseline.security?.vulnerabilities || []);
|
|
188
|
+
const currentVulns = new Set(current.security?.vulnerabilities || []);
|
|
189
|
+
const newVulnerabilities = [];
|
|
190
|
+
const fixedVulnerabilities = [];
|
|
191
|
+
for (const vuln of currentVulns) {
|
|
192
|
+
if (!baselineVulns.has(vuln)) {
|
|
193
|
+
newVulnerabilities.push(vuln);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
for (const vuln of baselineVulns) {
|
|
197
|
+
if (!currentVulns.has(vuln)) {
|
|
198
|
+
fixedVulnerabilities.push(vuln);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
return {
|
|
202
|
+
newVulnerabilities,
|
|
203
|
+
fixedVulnerabilities,
|
|
204
|
+
netChange: newVulnerabilities.length - fixedVulnerabilities.length,
|
|
205
|
+
baselineCount: baselineVulns.size,
|
|
206
|
+
currentCount: currentVulns.size,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Compare functionality results between assessments
|
|
211
|
+
*/
|
|
212
|
+
function compareFunctionalityResults(baseline, current) {
|
|
213
|
+
const baselineBroken = new Set(baseline.functionality?.brokenTools || []);
|
|
214
|
+
const currentBroken = new Set(current.functionality?.brokenTools || []);
|
|
215
|
+
const newBrokenTools = [];
|
|
216
|
+
const fixedTools = [];
|
|
217
|
+
for (const tool of currentBroken) {
|
|
218
|
+
if (!baselineBroken.has(tool)) {
|
|
219
|
+
newBrokenTools.push(tool);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
for (const tool of baselineBroken) {
|
|
223
|
+
if (!currentBroken.has(tool)) {
|
|
224
|
+
fixedTools.push(tool);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
const baselineWorking = baseline.functionality?.workingTools || 0;
|
|
228
|
+
const currentWorking = current.functionality?.workingTools || 0;
|
|
229
|
+
return {
|
|
230
|
+
newBrokenTools,
|
|
231
|
+
fixedTools,
|
|
232
|
+
netChange: currentWorking - baselineWorking,
|
|
233
|
+
baselineWorking,
|
|
234
|
+
currentWorking,
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Generate recommendations based on the diff
|
|
239
|
+
*/
|
|
240
|
+
function generateDiffRecommendations(moduleChanges, securityDelta, functionalityDelta) {
|
|
241
|
+
const recommendations = [];
|
|
242
|
+
// Security regressions are critical
|
|
243
|
+
if (securityDelta.newVulnerabilities.length > 0) {
|
|
244
|
+
recommendations.push(`CRITICAL: ${securityDelta.newVulnerabilities.length} new security vulnerabilities detected`);
|
|
245
|
+
for (const vuln of securityDelta.newVulnerabilities.slice(0, 3)) {
|
|
246
|
+
recommendations.push(` - ${vuln}`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
// Functionality regressions
|
|
250
|
+
if (functionalityDelta.newBrokenTools.length > 0) {
|
|
251
|
+
recommendations.push(`WARNING: ${functionalityDelta.newBrokenTools.length} tool(s) now broken`);
|
|
252
|
+
for (const tool of functionalityDelta.newBrokenTools.slice(0, 3)) {
|
|
253
|
+
recommendations.push(` - ${tool}`);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
// Module regressions
|
|
257
|
+
const regressions = moduleChanges.filter((m) => m.change === "regressed");
|
|
258
|
+
if (regressions.length > 0) {
|
|
259
|
+
recommendations.push(`WARNING: ${regressions.length} module(s) regressed`);
|
|
260
|
+
for (const reg of regressions) {
|
|
261
|
+
recommendations.push(` - ${reg.module}: ${reg.details}`);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
// Positive changes
|
|
265
|
+
if (securityDelta.fixedVulnerabilities.length > 0) {
|
|
266
|
+
recommendations.push(`IMPROVED: ${securityDelta.fixedVulnerabilities.length} vulnerabilities fixed`);
|
|
267
|
+
}
|
|
268
|
+
if (functionalityDelta.fixedTools.length > 0) {
|
|
269
|
+
recommendations.push(`IMPROVED: ${functionalityDelta.fixedTools.length} tool(s) now working`);
|
|
270
|
+
}
|
|
271
|
+
const improvements = moduleChanges.filter((m) => m.change === "improved");
|
|
272
|
+
if (improvements.length > 0) {
|
|
273
|
+
recommendations.push(`IMPROVED: ${improvements.length} module(s) improved`);
|
|
274
|
+
}
|
|
275
|
+
if (recommendations.length === 0) {
|
|
276
|
+
recommendations.push("No significant changes detected between versions");
|
|
277
|
+
}
|
|
278
|
+
return recommendations;
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* Extract version from assessment (if available from manifest)
|
|
282
|
+
*/
|
|
283
|
+
function extractVersion(assessment) {
|
|
284
|
+
// Try to get version from manifest validation
|
|
285
|
+
if (assessment.manifestValidation?.manifestVersion) {
|
|
286
|
+
return assessment.manifestValidation.manifestVersion;
|
|
287
|
+
}
|
|
288
|
+
return undefined;
|
|
289
|
+
}
|
|
@@ -241,6 +241,16 @@ export interface UsabilityMetrics {
|
|
|
241
241
|
overallScore: number;
|
|
242
242
|
};
|
|
243
243
|
}
|
|
244
|
+
/** Tool definition with schema from MCP tools/list response */
|
|
245
|
+
export interface DiscoveredTool {
|
|
246
|
+
name: string;
|
|
247
|
+
description?: string;
|
|
248
|
+
inputSchema?: {
|
|
249
|
+
type: string;
|
|
250
|
+
properties?: Record<string, unknown>;
|
|
251
|
+
required?: string[];
|
|
252
|
+
};
|
|
253
|
+
}
|
|
244
254
|
export interface FunctionalityAssessment {
|
|
245
255
|
totalTools: number;
|
|
246
256
|
testedTools: number;
|
|
@@ -250,6 +260,8 @@ export interface FunctionalityAssessment {
|
|
|
250
260
|
status: AssessmentStatus;
|
|
251
261
|
explanation: string;
|
|
252
262
|
toolResults: ToolTestResult[];
|
|
263
|
+
/** Raw tool definitions with inputSchema from MCP server */
|
|
264
|
+
tools?: DiscoveredTool[];
|
|
253
265
|
}
|
|
254
266
|
export interface SecurityAssessment {
|
|
255
267
|
promptInjectionTests: SecurityTestResult[];
|
|
@@ -476,6 +488,8 @@ export interface MCPDirectoryAssessment {
|
|
|
476
488
|
prohibitedLibraries?: ProhibitedLibrariesAssessment;
|
|
477
489
|
manifestValidation?: ManifestValidationAssessment;
|
|
478
490
|
portability?: PortabilityAssessment;
|
|
491
|
+
externalAPIScanner?: ExternalAPIScannerAssessment;
|
|
492
|
+
authentication?: AuthenticationAssessment;
|
|
479
493
|
overallStatus: AssessmentStatus;
|
|
480
494
|
summary: string;
|
|
481
495
|
recommendations: string[];
|
|
@@ -634,6 +648,18 @@ export interface ManifestJsonSchema {
|
|
|
634
648
|
icon?: string;
|
|
635
649
|
homepage?: string;
|
|
636
650
|
keywords?: string[];
|
|
651
|
+
privacy_policies?: string[];
|
|
652
|
+
}
|
|
653
|
+
/**
|
|
654
|
+
* Privacy Policy URL Validation Result
|
|
655
|
+
* Validates that privacy_policies URLs are accessible
|
|
656
|
+
*/
|
|
657
|
+
export interface PrivacyPolicyValidation {
|
|
658
|
+
url: string;
|
|
659
|
+
accessible: boolean;
|
|
660
|
+
statusCode?: number;
|
|
661
|
+
contentType?: string;
|
|
662
|
+
error?: string;
|
|
637
663
|
}
|
|
638
664
|
export interface ManifestValidationResult {
|
|
639
665
|
field: string;
|
|
@@ -650,6 +676,12 @@ export interface ManifestValidationAssessment {
|
|
|
650
676
|
hasIcon: boolean;
|
|
651
677
|
hasRequiredFields: boolean;
|
|
652
678
|
missingFields: string[];
|
|
679
|
+
/** Privacy policy URL validation results */
|
|
680
|
+
privacyPolicies?: {
|
|
681
|
+
declared: string[];
|
|
682
|
+
validationResults: PrivacyPolicyValidation[];
|
|
683
|
+
allAccessible: boolean;
|
|
684
|
+
};
|
|
653
685
|
status: AssessmentStatus;
|
|
654
686
|
explanation: string;
|
|
655
687
|
recommendations: string[];
|
|
@@ -677,6 +709,41 @@ export interface PortabilityAssessment {
|
|
|
677
709
|
explanation: string;
|
|
678
710
|
recommendations: string[];
|
|
679
711
|
}
|
|
712
|
+
export interface DetectedAPI {
|
|
713
|
+
url: string;
|
|
714
|
+
service: string;
|
|
715
|
+
filePath: string;
|
|
716
|
+
}
|
|
717
|
+
export interface ExternalAPIScannerAssessment {
|
|
718
|
+
detectedAPIs: DetectedAPI[];
|
|
719
|
+
uniqueServices: string[];
|
|
720
|
+
affiliationWarning?: string;
|
|
721
|
+
scannedFiles: number;
|
|
722
|
+
status: AssessmentStatus;
|
|
723
|
+
explanation: string;
|
|
724
|
+
recommendations: string[];
|
|
725
|
+
}
|
|
726
|
+
export type AuthMethod = "oauth" | "api_key" | "none" | "unknown";
|
|
727
|
+
export interface AuthAppropriateness {
|
|
728
|
+
isAppropriate: boolean;
|
|
729
|
+
concerns: string[];
|
|
730
|
+
explanation: string;
|
|
731
|
+
}
|
|
732
|
+
export interface AuthenticationAssessment {
|
|
733
|
+
authMethod: AuthMethod;
|
|
734
|
+
hasLocalDependencies: boolean;
|
|
735
|
+
transportType: string;
|
|
736
|
+
appropriateness: AuthAppropriateness;
|
|
737
|
+
recommendation: string;
|
|
738
|
+
detectedPatterns: {
|
|
739
|
+
oauthIndicators: string[];
|
|
740
|
+
localResourceIndicators: string[];
|
|
741
|
+
apiKeyIndicators: string[];
|
|
742
|
+
};
|
|
743
|
+
status: AssessmentStatus;
|
|
744
|
+
explanation: string;
|
|
745
|
+
recommendations: string[];
|
|
746
|
+
}
|
|
680
747
|
export declare const PROMPT_INJECTION_TESTS: Omit<SecurityTestResult, "vulnerable" | "evidence">[];
|
|
681
748
|
/**
|
|
682
749
|
* Claude Code Bridge Configuration
|
|
@@ -723,6 +790,8 @@ export interface AssessmentConfiguration {
|
|
|
723
790
|
prohibitedLibraries?: boolean;
|
|
724
791
|
manifestValidation?: boolean;
|
|
725
792
|
portability?: boolean;
|
|
793
|
+
externalAPIScanner?: boolean;
|
|
794
|
+
authentication?: boolean;
|
|
726
795
|
};
|
|
727
796
|
}
|
|
728
797
|
/**
|