@dotsetlabs/bellwether 2.1.1 → 2.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/README.md +19 -4
- package/dist/baseline/golden-output.d.ts +0 -4
- package/dist/baseline/golden-output.js +2 -47
- package/dist/cli/commands/baseline-accept.js +14 -45
- package/dist/cli/commands/baseline.js +23 -78
- package/dist/cli/commands/check-formatters.d.ts +10 -0
- package/dist/cli/commands/check-formatters.js +160 -0
- package/dist/cli/commands/check.js +48 -213
- package/dist/cli/commands/contract.js +1 -13
- package/dist/cli/commands/dashboard.d.ts +3 -0
- package/dist/cli/commands/dashboard.js +69 -0
- package/dist/cli/commands/discover.js +24 -2
- package/dist/cli/commands/explore.js +34 -38
- package/dist/cli/commands/watch.js +14 -4
- package/dist/cli/output.d.ts +0 -42
- package/dist/cli/output.js +73 -110
- package/dist/cli/utils/config-loader.d.ts +6 -0
- package/dist/cli/utils/config-loader.js +19 -0
- package/dist/cli/utils/error-hints.d.ts +9 -0
- package/dist/cli/utils/error-hints.js +128 -0
- package/dist/cli/utils/headers.d.ts +12 -0
- package/dist/cli/utils/headers.js +40 -0
- package/dist/cli/utils/path-resolution.d.ts +10 -0
- package/dist/cli/utils/path-resolution.js +27 -0
- package/dist/cli/utils/report-loader.d.ts +9 -0
- package/dist/cli/utils/report-loader.js +31 -0
- package/dist/cli/utils/server-runtime.d.ts +16 -0
- package/dist/cli/utils/server-runtime.js +31 -0
- package/dist/config/defaults.d.ts +2 -1
- package/dist/config/defaults.js +2 -1
- package/dist/config/template.js +12 -0
- package/dist/config/validator.d.ts +38 -18
- package/dist/config/validator.js +10 -0
- package/dist/constants/core.d.ts +2 -42
- package/dist/constants/core.js +11 -50
- package/dist/contract/validator.js +2 -47
- package/dist/dashboard/index.d.ts +3 -0
- package/dist/dashboard/index.js +6 -0
- package/dist/dashboard/runtime/artifact-index.d.ts +45 -0
- package/dist/dashboard/runtime/artifact-index.js +238 -0
- package/dist/dashboard/runtime/command-profiles.d.ts +764 -0
- package/dist/dashboard/runtime/command-profiles.js +691 -0
- package/dist/dashboard/runtime/config-service.d.ts +21 -0
- package/dist/dashboard/runtime/config-service.js +73 -0
- package/dist/dashboard/runtime/job-runner.d.ts +26 -0
- package/dist/dashboard/runtime/job-runner.js +292 -0
- package/dist/dashboard/security/input-validation.d.ts +3 -0
- package/dist/dashboard/security/input-validation.js +27 -0
- package/dist/dashboard/security/localhost-guard.d.ts +5 -0
- package/dist/dashboard/security/localhost-guard.js +52 -0
- package/dist/dashboard/server.d.ts +14 -0
- package/dist/dashboard/server.js +293 -0
- package/dist/dashboard/types.d.ts +55 -0
- package/dist/dashboard/types.js +2 -0
- package/dist/dashboard/ui.d.ts +2 -0
- package/dist/dashboard/ui.js +2264 -0
- package/dist/discovery/discovery.js +20 -1
- package/dist/discovery/types.d.ts +1 -1
- package/dist/docs/contract.js +7 -1
- package/dist/errors/retry.js +15 -1
- package/dist/errors/types.d.ts +10 -0
- package/dist/errors/types.js +28 -0
- package/dist/interview/question-category.d.ts +5 -0
- package/dist/interview/question-category.js +2 -0
- package/dist/interview/question-types.d.ts +80 -0
- package/dist/interview/question-types.js +2 -0
- package/dist/interview/schema-test-generator.d.ts +3 -29
- package/dist/interview/schema-test-generator.js +11 -286
- package/dist/interview/test-fixtures.d.ts +19 -0
- package/dist/interview/test-fixtures.js +2 -0
- package/dist/interview/types.d.ts +5 -80
- package/dist/persona/types.d.ts +3 -5
- package/dist/scenarios/types.d.ts +1 -1
- package/dist/transport/auth-errors.d.ts +15 -0
- package/dist/transport/auth-errors.js +22 -0
- package/dist/transport/http-transport.js +8 -0
- package/dist/transport/mcp-client.d.ts +12 -0
- package/dist/transport/mcp-client.js +92 -4
- package/dist/transport/sse-transport.d.ts +0 -1
- package/dist/transport/sse-transport.js +15 -11
- package/dist/utils/content-type.d.ts +14 -0
- package/dist/utils/content-type.js +37 -0
- package/dist/utils/http-headers.d.ts +9 -0
- package/dist/utils/http-headers.js +34 -0
- package/dist/utils/smart-truncate.js +2 -23
- package/package.json +4 -15
- package/man/bellwether.1 +0 -204
- package/man/bellwether.1.md +0 -148
|
@@ -13,9 +13,9 @@ import { MCPClient } from '../../transport/mcp-client.js';
|
|
|
13
13
|
import { discover } from '../../discovery/discovery.js';
|
|
14
14
|
import { Interviewer } from '../../interview/interviewer.js';
|
|
15
15
|
import { generateContractMd, generateJsonReport } from '../../docs/generator.js';
|
|
16
|
-
import { loadConfig, ConfigNotFoundError,
|
|
16
|
+
import { loadConfig, ConfigNotFoundError, } from '../../config/loader.js';
|
|
17
17
|
import { validateConfigForCheck, getConfigWarnings } from '../../config/validator.js';
|
|
18
|
-
import { createBaseline, loadBaseline, saveBaseline, getToolFingerprints, toToolCapability, compareBaselines, acceptDrift,
|
|
18
|
+
import { createBaseline, loadBaseline, saveBaseline, getToolFingerprints, toToolCapability, compareBaselines, acceptDrift, applySeverityConfig, shouldFailOnDiff, analyzeForIncremental, formatIncrementalSummary, runSecurityTests, parseSecurityCategories, getAllSecurityCategories, } from '../../baseline/index.js';
|
|
19
19
|
import { convertAssertions } from '../../baseline/converter.js';
|
|
20
20
|
import { getMetricsCollector, resetMetricsCollector } from '../../metrics/collector.js';
|
|
21
21
|
import { getGlobalCache, resetGlobalCache } from '../../cache/response-cache.js';
|
|
@@ -29,6 +29,10 @@ import { configureLogger } from '../../logging/logger.js';
|
|
|
29
29
|
import { buildInterviewInsights } from '../../interview/insights.js';
|
|
30
30
|
import { EXIT_CODES, SEVERITY_TO_EXIT_CODE, PATHS, SECURITY_TESTING, CHECK_SAMPLING, WORKFLOW, REPORT_SCHEMAS, PERCENTAGE_CONVERSION, MCP, } from '../../constants.js';
|
|
31
31
|
import { getFeatureFlags, getExcludedFeatureNames } from '../../protocol/index.js';
|
|
32
|
+
import { resolveServerRuntime } from '../utils/server-runtime.js';
|
|
33
|
+
import { resolvePathFromOutputDir, resolvePathFromOutputDirOrCwd, } from '../utils/path-resolution.js';
|
|
34
|
+
import { printCheckErrorHints } from '../utils/error-hints.js';
|
|
35
|
+
import { formatDiffOutput, formatCheckResults } from './check-formatters.js';
|
|
32
36
|
export const checkCommand = new Command('check')
|
|
33
37
|
.description('Check MCP server schema and detect drift (free, fast, deterministic)')
|
|
34
38
|
.allowUnknownOption() // Allow server flags like -y for npx to pass through
|
|
@@ -41,6 +45,7 @@ export const checkCommand = new Command('check')
|
|
|
41
45
|
.option('--format <format>', 'Diff output format: text, json, compact, github, markdown, junit, sarif')
|
|
42
46
|
.option('--min-severity <level>', 'Minimum severity to report (overrides config): none, info, warning, breaking')
|
|
43
47
|
.option('--fail-on-severity <level>', 'Fail threshold (overrides config): none, info, warning, breaking')
|
|
48
|
+
.option('-H, --header <header...>', 'Custom headers for remote MCP requests (e.g., "Authorization: Bearer token")')
|
|
44
49
|
.action(async (serverCommandArg, serverArgs, options) => {
|
|
45
50
|
// Load configuration
|
|
46
51
|
let config;
|
|
@@ -54,19 +59,27 @@ export const checkCommand = new Command('check')
|
|
|
54
59
|
}
|
|
55
60
|
throw error;
|
|
56
61
|
}
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
let
|
|
60
|
-
let
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
62
|
+
let serverCommand;
|
|
63
|
+
let args;
|
|
64
|
+
let transport;
|
|
65
|
+
let remoteUrl;
|
|
66
|
+
let remoteSessionId;
|
|
67
|
+
let remoteHeaders;
|
|
68
|
+
let serverIdentifier;
|
|
69
|
+
try {
|
|
70
|
+
const runtime = resolveServerRuntime(config, serverCommandArg, serverArgs, options.header);
|
|
71
|
+
serverCommand = runtime.serverCommand;
|
|
72
|
+
args = runtime.args;
|
|
73
|
+
transport = runtime.transport;
|
|
74
|
+
remoteUrl = runtime.remoteUrl;
|
|
75
|
+
remoteSessionId = runtime.remoteSessionId;
|
|
76
|
+
remoteHeaders = runtime.remoteHeaders;
|
|
77
|
+
serverIdentifier = runtime.serverIdentifier;
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
output.error(error instanceof Error ? error.message : String(error));
|
|
81
|
+
process.exit(EXIT_CODES.ERROR);
|
|
66
82
|
}
|
|
67
|
-
const transport = config.server.transport ?? 'stdio';
|
|
68
|
-
const remoteUrl = config.server.url?.trim();
|
|
69
|
-
const remoteSessionId = config.server.sessionId?.trim();
|
|
70
83
|
// Validate config for check
|
|
71
84
|
try {
|
|
72
85
|
validateConfigForCheck(config, serverCommand);
|
|
@@ -90,9 +103,16 @@ export const checkCommand = new Command('check')
|
|
|
90
103
|
const effectiveLogLevel = verbose ? logLevel : 'silent';
|
|
91
104
|
configureLogger({ level: effectiveLogLevel });
|
|
92
105
|
}
|
|
93
|
-
// Resolve baseline options from config (--fail-on-drift CLI flag can override)
|
|
94
|
-
|
|
95
|
-
|
|
106
|
+
// Resolve baseline options from config (--fail-on-drift CLI flag can override).
|
|
107
|
+
// Keep path semantics consistent with baseline subcommands:
|
|
108
|
+
// - savePath: relative to output.dir
|
|
109
|
+
// - comparePath: output.dir first, then cwd fallback for existing files
|
|
110
|
+
const baselinePath = config.baseline.comparePath
|
|
111
|
+
? resolvePathFromOutputDirOrCwd(config.baseline.comparePath, outputDir)
|
|
112
|
+
: undefined;
|
|
113
|
+
const saveBaselinePath = config.baseline.savePath
|
|
114
|
+
? resolvePathFromOutputDir(config.baseline.savePath, outputDir)
|
|
115
|
+
: undefined;
|
|
96
116
|
const failOnDrift = options.failOnDrift ? true : config.baseline.failOnDrift;
|
|
97
117
|
// Build severity config (CLI options override config file)
|
|
98
118
|
const severityConfig = {
|
|
@@ -149,9 +169,6 @@ export const checkCommand = new Command('check')
|
|
|
149
169
|
const fullExamples = config.output.examples.full;
|
|
150
170
|
const exampleLength = config.output.examples.maxLength;
|
|
151
171
|
const maxExamplesPerTool = config.output.examples.maxPerTool;
|
|
152
|
-
const serverIdentifier = transport === 'stdio'
|
|
153
|
-
? `${serverCommand} ${args.join(' ')}`.trim()
|
|
154
|
-
: (remoteUrl ?? 'unknown');
|
|
155
172
|
// Display startup banner
|
|
156
173
|
if (!machineReadable) {
|
|
157
174
|
const banner = formatCheckBanner({
|
|
@@ -193,6 +210,7 @@ export const checkCommand = new Command('check')
|
|
|
193
210
|
await mcpClient.connectRemote(remoteUrl, {
|
|
194
211
|
transport,
|
|
195
212
|
sessionId: remoteSessionId || undefined,
|
|
213
|
+
headers: remoteHeaders,
|
|
196
214
|
});
|
|
197
215
|
}
|
|
198
216
|
// Discovery phase
|
|
@@ -255,12 +273,18 @@ export const checkCommand = new Command('check')
|
|
|
255
273
|
toolsDiscovered: discovery.tools.length,
|
|
256
274
|
personasUsed: 0, // No personas in check mode
|
|
257
275
|
});
|
|
258
|
-
|
|
259
|
-
|
|
276
|
+
const hasInterviewTargets = discovery.tools.length > 0 ||
|
|
277
|
+
discovery.prompts.length > 0 ||
|
|
278
|
+
(discovery.resources?.length ?? 0) > 0;
|
|
279
|
+
if (!hasInterviewTargets) {
|
|
280
|
+
output.info('No tools, prompts, or resources found. Nothing to check.');
|
|
260
281
|
metricsCollector.endInterview();
|
|
261
282
|
await mcpClient.disconnect();
|
|
262
283
|
return;
|
|
263
284
|
}
|
|
285
|
+
if (discovery.tools.length === 0) {
|
|
286
|
+
output.info('No tools found; continuing with prompt/resource checks.');
|
|
287
|
+
}
|
|
264
288
|
// Incremental checking - load baseline and determine which tools to test
|
|
265
289
|
let incrementalBaseline = null;
|
|
266
290
|
let incrementalResult = null;
|
|
@@ -852,7 +876,7 @@ export const checkCommand = new Command('check')
|
|
|
852
876
|
output.info('\n--- Drift Report ---');
|
|
853
877
|
}
|
|
854
878
|
// Select formatter based on --format option
|
|
855
|
-
const formattedDiff =
|
|
879
|
+
const formattedDiff = formatDiffOutput(diff, diffFormat, baselinePath);
|
|
856
880
|
if (machineReadable) {
|
|
857
881
|
console.log(formattedDiff);
|
|
858
882
|
}
|
|
@@ -948,21 +972,7 @@ export const checkCommand = new Command('check')
|
|
|
948
972
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
949
973
|
output.error('\n--- Check Failed ---');
|
|
950
974
|
output.error(`Error: ${errorMessage}`);
|
|
951
|
-
|
|
952
|
-
output.error('\nPossible causes:');
|
|
953
|
-
output.error(' - The MCP server is not running');
|
|
954
|
-
output.error(' - The server address/port is incorrect');
|
|
955
|
-
}
|
|
956
|
-
else if (errorMessage.includes('timeout') || errorMessage.includes('Timeout')) {
|
|
957
|
-
output.error('\nPossible causes:');
|
|
958
|
-
output.error(' - The MCP server is taking too long to respond');
|
|
959
|
-
output.error(' - Increase server.timeout in bellwether.yaml');
|
|
960
|
-
}
|
|
961
|
-
else if (errorMessage.includes('ENOENT') || errorMessage.includes('not found')) {
|
|
962
|
-
output.error('\nPossible causes:');
|
|
963
|
-
output.error(' - The server command was not found');
|
|
964
|
-
output.error(' - Check that the command is installed and in PATH');
|
|
965
|
-
}
|
|
975
|
+
printCheckErrorHints(error, transport);
|
|
966
976
|
pendingExitCode = EXIT_CODES.ERROR;
|
|
967
977
|
}
|
|
968
978
|
finally {
|
|
@@ -977,179 +987,4 @@ export const checkCommand = new Command('check')
|
|
|
977
987
|
}
|
|
978
988
|
}
|
|
979
989
|
});
|
|
980
|
-
/**
|
|
981
|
-
* Format a diff using the specified output format.
|
|
982
|
-
*
|
|
983
|
-
* @param diff - The behavioral diff to format
|
|
984
|
-
* @param format - Output format: text, json, compact, github, markdown, junit, sarif
|
|
985
|
-
* @param baselinePath - Path to baseline file (used for SARIF location references)
|
|
986
|
-
* @returns Formatted string
|
|
987
|
-
*/
|
|
988
|
-
function formatDiff(diff, format, baselinePath) {
|
|
989
|
-
switch (format.toLowerCase()) {
|
|
990
|
-
case 'json':
|
|
991
|
-
return formatDiffJson(diff);
|
|
992
|
-
case 'compact':
|
|
993
|
-
return formatDiffCompact(diff);
|
|
994
|
-
case 'github':
|
|
995
|
-
return formatDiffGitHubActions(diff);
|
|
996
|
-
case 'markdown':
|
|
997
|
-
case 'md':
|
|
998
|
-
return formatDiffMarkdown(diff);
|
|
999
|
-
case 'junit':
|
|
1000
|
-
case 'junit-xml':
|
|
1001
|
-
case 'xml':
|
|
1002
|
-
return formatDiffJUnit(diff, 'bellwether-check');
|
|
1003
|
-
case 'sarif':
|
|
1004
|
-
return formatDiffSarif(diff, baselinePath);
|
|
1005
|
-
case 'text':
|
|
1006
|
-
default:
|
|
1007
|
-
return formatDiffText(diff);
|
|
1008
|
-
}
|
|
1009
|
-
}
|
|
1010
|
-
/**
|
|
1011
|
-
* Format check results as JUnit XML (for CI systems that expect test results).
|
|
1012
|
-
* This is used when --format junit is specified but no baseline comparison occurs.
|
|
1013
|
-
*/
|
|
1014
|
-
function formatCheckResultsJUnit(baseline) {
|
|
1015
|
-
const tools = getToolFingerprints(baseline);
|
|
1016
|
-
const lines = [];
|
|
1017
|
-
const securityFailures = tools.filter((t) => t.securityFingerprint?.findings?.some((f) => f.riskLevel === 'critical' || f.riskLevel === 'high')).length;
|
|
1018
|
-
lines.push('<?xml version="1.0" encoding="UTF-8"?>');
|
|
1019
|
-
lines.push('<testsuites>');
|
|
1020
|
-
lines.push(` <testsuite name="bellwether-check" tests="${tools.length}" failures="${securityFailures}" errors="0">`);
|
|
1021
|
-
for (const tool of tools) {
|
|
1022
|
-
const successRate = tool.baselineSuccessRate ?? 1;
|
|
1023
|
-
const status = successRate >= 0.9 ? 'passed' : 'warning';
|
|
1024
|
-
lines.push(` <testcase name="${tool.name}" classname="mcp-tools" time="0">`);
|
|
1025
|
-
lines.push(` <system-out>Success rate: ${(successRate * 100).toFixed(0)}%</system-out>`);
|
|
1026
|
-
if (status === 'warning') {
|
|
1027
|
-
lines.push(` <system-err>Tool has success rate below 90%</system-err>`);
|
|
1028
|
-
}
|
|
1029
|
-
lines.push(' </testcase>');
|
|
1030
|
-
}
|
|
1031
|
-
// Add security findings as test cases if present
|
|
1032
|
-
const securityTools = tools.filter((t) => t.securityFingerprint?.findings?.length);
|
|
1033
|
-
if (securityTools.length > 0) {
|
|
1034
|
-
lines.push(` <!-- Security findings -->`);
|
|
1035
|
-
for (const tool of securityTools) {
|
|
1036
|
-
const findings = tool.securityFingerprint?.findings ?? [];
|
|
1037
|
-
const criticalHigh = findings.filter((f) => f.riskLevel === 'critical' || f.riskLevel === 'high').length;
|
|
1038
|
-
if (criticalHigh > 0) {
|
|
1039
|
-
lines.push(` <testcase name="${tool.name}-security" classname="security">`);
|
|
1040
|
-
lines.push(` <failure message="${criticalHigh} critical/high security findings">`);
|
|
1041
|
-
for (const finding of findings.filter((f) => f.riskLevel === 'critical' || f.riskLevel === 'high')) {
|
|
1042
|
-
lines.push(` ${finding.riskLevel.toUpperCase()}: ${finding.title} (${finding.cweId})`);
|
|
1043
|
-
}
|
|
1044
|
-
lines.push(` </failure>`);
|
|
1045
|
-
lines.push(' </testcase>');
|
|
1046
|
-
}
|
|
1047
|
-
}
|
|
1048
|
-
}
|
|
1049
|
-
lines.push(' </testsuite>');
|
|
1050
|
-
lines.push('</testsuites>');
|
|
1051
|
-
return lines.join('\n');
|
|
1052
|
-
}
|
|
1053
|
-
/**
|
|
1054
|
-
* Format check results as SARIF (for GitHub Code Scanning and other tools).
|
|
1055
|
-
* This is used when --format sarif is specified but no baseline comparison occurs.
|
|
1056
|
-
*/
|
|
1057
|
-
function formatCheckResultsSarif(baseline) {
|
|
1058
|
-
const tools = getToolFingerprints(baseline);
|
|
1059
|
-
const serverUri = baseline.metadata?.serverCommand || baseline.server.name || 'mcp-server';
|
|
1060
|
-
const results = [];
|
|
1061
|
-
// Add results for tools with security findings
|
|
1062
|
-
const securityTools = tools.filter((t) => t.securityFingerprint?.findings?.length);
|
|
1063
|
-
for (const tool of securityTools) {
|
|
1064
|
-
const findings = tool.securityFingerprint?.findings ?? [];
|
|
1065
|
-
for (const finding of findings) {
|
|
1066
|
-
const level = finding.riskLevel === 'critical' || finding.riskLevel === 'high'
|
|
1067
|
-
? 'error'
|
|
1068
|
-
: finding.riskLevel === 'medium'
|
|
1069
|
-
? 'warning'
|
|
1070
|
-
: 'note';
|
|
1071
|
-
results.push({
|
|
1072
|
-
ruleId: finding.cweId || 'BWH-SEC',
|
|
1073
|
-
level,
|
|
1074
|
-
message: { text: `[${tool.name}] ${finding.title}: ${finding.description}` },
|
|
1075
|
-
locations: [
|
|
1076
|
-
{
|
|
1077
|
-
physicalLocation: {
|
|
1078
|
-
artifactLocation: { uri: serverUri },
|
|
1079
|
-
region: { startLine: 1 },
|
|
1080
|
-
},
|
|
1081
|
-
},
|
|
1082
|
-
],
|
|
1083
|
-
});
|
|
1084
|
-
}
|
|
1085
|
-
}
|
|
1086
|
-
// Add results for tools with low success rate
|
|
1087
|
-
for (const tool of tools) {
|
|
1088
|
-
const successRate = tool.baselineSuccessRate ?? 1;
|
|
1089
|
-
if (successRate < 0.9) {
|
|
1090
|
-
results.push({
|
|
1091
|
-
ruleId: 'BWH-REL',
|
|
1092
|
-
level: 'warning',
|
|
1093
|
-
message: {
|
|
1094
|
-
text: `Tool "${tool.name}" has ${(successRate * 100).toFixed(0)}% success rate`,
|
|
1095
|
-
},
|
|
1096
|
-
locations: [
|
|
1097
|
-
{
|
|
1098
|
-
physicalLocation: {
|
|
1099
|
-
artifactLocation: { uri: serverUri },
|
|
1100
|
-
region: { startLine: 1 },
|
|
1101
|
-
},
|
|
1102
|
-
},
|
|
1103
|
-
],
|
|
1104
|
-
});
|
|
1105
|
-
}
|
|
1106
|
-
}
|
|
1107
|
-
const sarif = {
|
|
1108
|
-
$schema: 'https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json',
|
|
1109
|
-
version: '2.1.0',
|
|
1110
|
-
runs: [
|
|
1111
|
-
{
|
|
1112
|
-
tool: {
|
|
1113
|
-
driver: {
|
|
1114
|
-
name: 'bellwether',
|
|
1115
|
-
version: '1.0.0',
|
|
1116
|
-
informationUri: 'https://github.com/dotsetlabs/bellwether',
|
|
1117
|
-
rules: [
|
|
1118
|
-
{
|
|
1119
|
-
id: 'BWH-SEC',
|
|
1120
|
-
name: 'SecurityFinding',
|
|
1121
|
-
shortDescription: { text: 'Security vulnerability detected' },
|
|
1122
|
-
defaultConfiguration: { level: 'warning' },
|
|
1123
|
-
},
|
|
1124
|
-
{
|
|
1125
|
-
id: 'BWH-REL',
|
|
1126
|
-
name: 'LowReliability',
|
|
1127
|
-
shortDescription: { text: 'Tool reliability below threshold' },
|
|
1128
|
-
defaultConfiguration: { level: 'warning' },
|
|
1129
|
-
},
|
|
1130
|
-
],
|
|
1131
|
-
},
|
|
1132
|
-
},
|
|
1133
|
-
results,
|
|
1134
|
-
},
|
|
1135
|
-
],
|
|
1136
|
-
};
|
|
1137
|
-
return JSON.stringify(sarif, null, 2);
|
|
1138
|
-
}
|
|
1139
|
-
/**
|
|
1140
|
-
* Format check results using the specified output format.
|
|
1141
|
-
* Used when no baseline comparison occurs.
|
|
1142
|
-
*/
|
|
1143
|
-
function formatCheckResults(baseline, format) {
|
|
1144
|
-
switch (format.toLowerCase()) {
|
|
1145
|
-
case 'junit':
|
|
1146
|
-
case 'junit-xml':
|
|
1147
|
-
case 'xml':
|
|
1148
|
-
return formatCheckResultsJUnit(baseline);
|
|
1149
|
-
case 'sarif':
|
|
1150
|
-
return formatCheckResultsSarif(baseline);
|
|
1151
|
-
default:
|
|
1152
|
-
return null; // No special formatting needed for other formats
|
|
1153
|
-
}
|
|
1154
|
-
}
|
|
1155
990
|
//# sourceMappingURL=check.js.map
|
|
@@ -13,24 +13,12 @@ import { loadContract, findContractFile, validateContract, generateContract, gen
|
|
|
13
13
|
import { MCPClient } from '../../transport/mcp-client.js';
|
|
14
14
|
import { discover } from '../../discovery/discovery.js';
|
|
15
15
|
import { EXIT_CODES, CONTRACT_TESTING } from '../../constants.js';
|
|
16
|
-
import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
|
|
17
16
|
import * as output from '../output.js';
|
|
17
|
+
import { loadConfigOrExit } from '../utils/config-loader.js';
|
|
18
18
|
/**
|
|
19
19
|
* Default paths for contract files.
|
|
20
20
|
*/
|
|
21
21
|
const DEFAULT_CONTRACT_FILENAMES = CONTRACT_TESTING.CONTRACT_FILENAMES;
|
|
22
|
-
function loadConfigOrExit(configPath) {
|
|
23
|
-
try {
|
|
24
|
-
return loadConfig(configPath);
|
|
25
|
-
}
|
|
26
|
-
catch (error) {
|
|
27
|
-
if (error instanceof ConfigNotFoundError) {
|
|
28
|
-
output.error(error.message);
|
|
29
|
-
process.exit(EXIT_CODES.ERROR);
|
|
30
|
-
}
|
|
31
|
-
throw error;
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
22
|
/**
|
|
35
23
|
* Find or use provided contract path.
|
|
36
24
|
*/
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { Command } from 'commander';
|
|
2
|
+
import { EXIT_CODES } from '../../constants.js';
|
|
3
|
+
import { startDashboard } from '../../dashboard/index.js';
|
|
4
|
+
import * as output from '../output.js';
|
|
5
|
+
const DEFAULT_HOST = '127.0.0.1';
|
|
6
|
+
const DEFAULT_PORT = 7331;
|
|
7
|
+
function parsePort(rawPort) {
|
|
8
|
+
const parsed = Number.parseInt(rawPort, 10);
|
|
9
|
+
if (!Number.isInteger(parsed) || parsed < 1 || parsed > 65535) {
|
|
10
|
+
throw new Error(`Invalid port "${rawPort}". Use a value between 1 and 65535.`);
|
|
11
|
+
}
|
|
12
|
+
return parsed;
|
|
13
|
+
}
|
|
14
|
+
export const dashboardCommand = new Command('dashboard')
|
|
15
|
+
.description('Start local web dashboard for Bellwether')
|
|
16
|
+
.option('--host <host>', 'Host interface to bind', DEFAULT_HOST)
|
|
17
|
+
.option('-p, --port <port>', 'Port to listen on', String(DEFAULT_PORT))
|
|
18
|
+
.action(async (options) => {
|
|
19
|
+
const host = String(options.host ?? DEFAULT_HOST).trim();
|
|
20
|
+
let port;
|
|
21
|
+
try {
|
|
22
|
+
port = parsePort(String(options.port ?? DEFAULT_PORT));
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
output.error(error instanceof Error ? error.message : String(error));
|
|
26
|
+
process.exit(EXIT_CODES.ERROR);
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
try {
|
|
30
|
+
const dashboard = await startDashboard({
|
|
31
|
+
host,
|
|
32
|
+
port,
|
|
33
|
+
cwd: process.cwd(),
|
|
34
|
+
cliEntrypoint: process.argv[1],
|
|
35
|
+
});
|
|
36
|
+
output.info('Bellwether Dashboard');
|
|
37
|
+
output.info(`URL: ${dashboard.url}`);
|
|
38
|
+
output.info('Available profiles: check, explore, validate-config, discover, watch, baseline.save, baseline.compare, baseline.show, baseline.diff, baseline.accept, registry.search, contract.validate, contract.generate, contract.show, golden.save, golden.compare, golden.list, golden.delete');
|
|
39
|
+
output.info('Press Ctrl+C to stop.');
|
|
40
|
+
let shuttingDown = false;
|
|
41
|
+
const shutdown = async (signal) => {
|
|
42
|
+
if (shuttingDown) {
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
shuttingDown = true;
|
|
46
|
+
output.info(`\nReceived ${signal}. Stopping dashboard...`);
|
|
47
|
+
try {
|
|
48
|
+
await dashboard.stop();
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
output.error(error instanceof Error ? error.message : String(error));
|
|
52
|
+
process.exit(EXIT_CODES.ERROR);
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
process.exit(EXIT_CODES.CLEAN);
|
|
56
|
+
};
|
|
57
|
+
process.on('SIGINT', () => {
|
|
58
|
+
void shutdown('SIGINT');
|
|
59
|
+
});
|
|
60
|
+
process.on('SIGTERM', () => {
|
|
61
|
+
void shutdown('SIGTERM');
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
output.error(`Failed to start dashboard: ${error instanceof Error ? error.message : String(error)}`);
|
|
66
|
+
process.exit(EXIT_CODES.ERROR);
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
//# sourceMappingURL=dashboard.js.map
|
|
@@ -4,6 +4,8 @@ import { discover, summarizeDiscovery } from '../../discovery/discovery.js';
|
|
|
4
4
|
import { EXIT_CODES } from '../../constants.js';
|
|
5
5
|
import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
|
|
6
6
|
import * as output from '../output.js';
|
|
7
|
+
import { ServerAuthError } from '../../errors/types.js';
|
|
8
|
+
import { mergeHeaders, parseCliHeaders } from '../utils/headers.js';
|
|
7
9
|
/**
|
|
8
10
|
* Action handler for the discover command.
|
|
9
11
|
*/
|
|
@@ -28,6 +30,16 @@ async function discoverAction(command, args, options) {
|
|
|
28
30
|
const outputJson = options.json ?? config?.discovery?.json ?? false;
|
|
29
31
|
const remoteUrl = options.url ?? config?.discovery?.url;
|
|
30
32
|
const sessionId = options.sessionId ?? config?.discovery?.sessionId;
|
|
33
|
+
const configuredHeaders = mergeHeaders(config?.server?.headers, config?.discovery?.headers);
|
|
34
|
+
let cliHeaders;
|
|
35
|
+
try {
|
|
36
|
+
cliHeaders = parseCliHeaders(options.header);
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
output.error(error instanceof Error ? error.message : String(error));
|
|
40
|
+
process.exit(EXIT_CODES.ERROR);
|
|
41
|
+
}
|
|
42
|
+
const headers = mergeHeaders(configuredHeaders, cliHeaders);
|
|
31
43
|
// Validate transport options
|
|
32
44
|
if (isRemoteTransport && !remoteUrl) {
|
|
33
45
|
output.error(`Error: --url is required when using --transport ${transportType}`);
|
|
@@ -49,10 +61,11 @@ async function discoverAction(command, args, options) {
|
|
|
49
61
|
await client.connectRemote(remoteUrl, {
|
|
50
62
|
transport: transportType,
|
|
51
63
|
sessionId,
|
|
64
|
+
headers,
|
|
52
65
|
});
|
|
53
66
|
}
|
|
54
67
|
else {
|
|
55
|
-
await client.connect(command, args);
|
|
68
|
+
await client.connect(command, args, config?.server?.env);
|
|
56
69
|
}
|
|
57
70
|
output.info('Discovering capabilities...\n');
|
|
58
71
|
const result = await discover(client, isRemoteTransport ? remoteUrl : command, isRemoteTransport ? [] : args);
|
|
@@ -83,7 +96,15 @@ async function discoverAction(command, args, options) {
|
|
|
83
96
|
}
|
|
84
97
|
}
|
|
85
98
|
catch (error) {
|
|
86
|
-
|
|
99
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
100
|
+
output.error(`Discovery failed: ${message}`);
|
|
101
|
+
if (error instanceof ServerAuthError ||
|
|
102
|
+
message.includes('401') ||
|
|
103
|
+
message.includes('403') ||
|
|
104
|
+
message.includes('407') ||
|
|
105
|
+
/unauthorized|forbidden|authentication|authorization/i.test(message)) {
|
|
106
|
+
output.error('Hint: configure discovery.headers/server.headers or pass --header.');
|
|
107
|
+
}
|
|
87
108
|
process.exit(EXIT_CODES.ERROR);
|
|
88
109
|
}
|
|
89
110
|
finally {
|
|
@@ -101,5 +122,6 @@ export const discoverCommand = new Command('discover')
|
|
|
101
122
|
.option('--transport <type>', 'Transport type: stdio, sse, streamable-http')
|
|
102
123
|
.option('--url <url>', 'URL for remote MCP server (requires --transport sse or streamable-http)')
|
|
103
124
|
.option('--session-id <id>', 'Session ID for remote server authentication')
|
|
125
|
+
.option('-H, --header <header...>', 'Custom headers for remote MCP requests (e.g., "Authorization: Bearer token")')
|
|
104
126
|
.action(discoverAction);
|
|
105
127
|
//# sourceMappingURL=discover.js.map
|
|
@@ -13,7 +13,7 @@ import { MCPClient } from '../../transport/mcp-client.js';
|
|
|
13
13
|
import { discover } from '../../discovery/discovery.js';
|
|
14
14
|
import { Interviewer } from '../../interview/interviewer.js';
|
|
15
15
|
import { generateAgentsMd, generateJsonReport } from '../../docs/generator.js';
|
|
16
|
-
import { loadConfig, ConfigNotFoundError,
|
|
16
|
+
import { loadConfig, ConfigNotFoundError, } from '../../config/loader.js';
|
|
17
17
|
import { validateConfigForExplore } from '../../config/validator.js';
|
|
18
18
|
import { CostTracker, estimateInterviewCost, estimateInterviewTime, formatCostAndTimeEstimate, suggestOptimizations, formatOptimizationSuggestions, } from '../../cost/index.js';
|
|
19
19
|
import { getMetricsCollector, resetMetricsCollector } from '../../metrics/collector.js';
|
|
@@ -31,6 +31,8 @@ import { suppressLogs, restoreLogLevel, configureLogger, } from '../../logging/l
|
|
|
31
31
|
import { extractServerContextFromArgs } from '../utils/server-context.js';
|
|
32
32
|
import { isCI } from '../utils/env.js';
|
|
33
33
|
import { buildInterviewInsights } from '../../interview/insights.js';
|
|
34
|
+
import { resolveServerRuntime } from '../utils/server-runtime.js';
|
|
35
|
+
import { printExploreErrorHints } from '../utils/error-hints.js';
|
|
34
36
|
/**
|
|
35
37
|
* Wrapper to parse personas with warning output.
|
|
36
38
|
*/
|
|
@@ -45,6 +47,7 @@ export const exploreCommand = new Command('explore')
|
|
|
45
47
|
.argument('[server-command]', 'Server command (overrides config)')
|
|
46
48
|
.argument('[args...]', 'Server arguments')
|
|
47
49
|
.option('-c, --config <path>', 'Path to config file', PATHS.DEFAULT_CONFIG_FILENAME)
|
|
50
|
+
.option('-H, --header <header...>', 'Custom headers for remote MCP requests (e.g., "Authorization: Bearer token")')
|
|
48
51
|
.action(async (serverCommandArg, serverArgs, options) => {
|
|
49
52
|
// Load configuration
|
|
50
53
|
let config;
|
|
@@ -58,19 +61,27 @@ export const exploreCommand = new Command('explore')
|
|
|
58
61
|
}
|
|
59
62
|
throw error;
|
|
60
63
|
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
let
|
|
64
|
-
let
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
64
|
+
let serverCommand;
|
|
65
|
+
let args;
|
|
66
|
+
let transport;
|
|
67
|
+
let remoteUrl;
|
|
68
|
+
let remoteSessionId;
|
|
69
|
+
let remoteHeaders;
|
|
70
|
+
let serverIdentifier;
|
|
71
|
+
try {
|
|
72
|
+
const runtime = resolveServerRuntime(config, serverCommandArg, serverArgs, options.header);
|
|
73
|
+
serverCommand = runtime.serverCommand;
|
|
74
|
+
args = runtime.args;
|
|
75
|
+
transport = runtime.transport;
|
|
76
|
+
remoteUrl = runtime.remoteUrl;
|
|
77
|
+
remoteSessionId = runtime.remoteSessionId;
|
|
78
|
+
remoteHeaders = runtime.remoteHeaders;
|
|
79
|
+
serverIdentifier = runtime.serverIdentifier;
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
output.error(error instanceof Error ? error.message : String(error));
|
|
83
|
+
process.exit(EXIT_CODES.ERROR);
|
|
70
84
|
}
|
|
71
|
-
const transport = config.server.transport ?? 'stdio';
|
|
72
|
-
const remoteUrl = config.server.url?.trim();
|
|
73
|
-
const remoteSessionId = config.server.sessionId?.trim();
|
|
74
85
|
// Validate config for explore
|
|
75
86
|
try {
|
|
76
87
|
validateConfigForExplore(config, serverCommand);
|
|
@@ -100,9 +111,6 @@ export const exploreCommand = new Command('explore')
|
|
|
100
111
|
const provider = config.llm.provider;
|
|
101
112
|
const model = config.llm.model || undefined;
|
|
102
113
|
// Display startup banner
|
|
103
|
-
const serverIdentifier = transport === 'stdio'
|
|
104
|
-
? `${serverCommand} ${args.join(' ')}`.trim()
|
|
105
|
-
: (remoteUrl ?? 'unknown');
|
|
106
114
|
const banner = formatExploreBanner({
|
|
107
115
|
serverCommand: serverIdentifier,
|
|
108
116
|
provider,
|
|
@@ -184,6 +192,7 @@ export const exploreCommand = new Command('explore')
|
|
|
184
192
|
await mcpClient.connectRemote(remoteUrl, {
|
|
185
193
|
transport,
|
|
186
194
|
sessionId: remoteSessionId || undefined,
|
|
195
|
+
headers: remoteHeaders,
|
|
187
196
|
});
|
|
188
197
|
}
|
|
189
198
|
// Discovery phase
|
|
@@ -211,12 +220,18 @@ export const exploreCommand = new Command('explore')
|
|
|
211
220
|
toolsDiscovered: discovery.tools.length,
|
|
212
221
|
personasUsed: selectedPersonas.length,
|
|
213
222
|
});
|
|
214
|
-
|
|
215
|
-
|
|
223
|
+
const hasInterviewTargets = discovery.tools.length > 0 ||
|
|
224
|
+
discovery.prompts.length > 0 ||
|
|
225
|
+
(discovery.resources?.length ?? 0) > 0;
|
|
226
|
+
if (!hasInterviewTargets) {
|
|
227
|
+
output.info('No tools, prompts, or resources found. Nothing to explore.');
|
|
216
228
|
metricsCollector.endInterview();
|
|
217
229
|
await mcpClient.disconnect();
|
|
218
230
|
return;
|
|
219
231
|
}
|
|
232
|
+
if (discovery.tools.length === 0) {
|
|
233
|
+
output.info('No tools found; continuing with prompt/resource exploration.');
|
|
234
|
+
}
|
|
220
235
|
// Show cost/time estimate (unless in CI)
|
|
221
236
|
if (!isCI()) {
|
|
222
237
|
const costEstimate = estimateInterviewCost(model || 'default', discovery.tools.length, maxQuestions, selectedPersonas.length);
|
|
@@ -480,26 +495,7 @@ export const exploreCommand = new Command('explore')
|
|
|
480
495
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
481
496
|
output.error('\n--- Exploration Failed ---');
|
|
482
497
|
output.error(`Error: ${errorMessage}`);
|
|
483
|
-
|
|
484
|
-
output.error('\nPossible causes:');
|
|
485
|
-
output.error(' - The MCP server is not running');
|
|
486
|
-
output.error(' - The server address/port is incorrect');
|
|
487
|
-
}
|
|
488
|
-
else if (errorMessage.includes('timeout') || errorMessage.includes('Timeout')) {
|
|
489
|
-
output.error('\nPossible causes:');
|
|
490
|
-
output.error(' - The MCP server is taking too long to respond');
|
|
491
|
-
output.error(' - Increase server.timeout in bellwether.yaml');
|
|
492
|
-
}
|
|
493
|
-
else if (errorMessage.includes('ENOENT') || errorMessage.includes('not found')) {
|
|
494
|
-
output.error('\nPossible causes:');
|
|
495
|
-
output.error(' - The server command was not found');
|
|
496
|
-
output.error(' - Check that the command is installed and in PATH');
|
|
497
|
-
}
|
|
498
|
-
else if (errorMessage.includes('API key') || errorMessage.includes('authentication')) {
|
|
499
|
-
output.error('\nPossible causes:');
|
|
500
|
-
output.error(' - Missing or invalid API key');
|
|
501
|
-
output.error(' - Run "bellwether auth" to configure API keys');
|
|
502
|
-
}
|
|
498
|
+
printExploreErrorHints(error, transport);
|
|
503
499
|
pendingExitCode = EXIT_CODES.ERROR;
|
|
504
500
|
}
|
|
505
501
|
finally {
|