@dotsetlabs/bellwether 2.1.2 → 2.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +2 -2
- package/dist/baseline/golden-output.d.ts +0 -4
- package/dist/baseline/golden-output.js +2 -47
- package/dist/cli/commands/baseline-accept.js +14 -45
- package/dist/cli/commands/baseline.js +23 -78
- package/dist/cli/commands/check-formatters.d.ts +10 -0
- package/dist/cli/commands/check-formatters.js +160 -0
- package/dist/cli/commands/check.js +33 -241
- package/dist/cli/commands/contract.js +1 -13
- package/dist/cli/commands/explore.js +19 -66
- package/dist/cli/commands/watch.js +2 -3
- package/dist/cli/output.d.ts +0 -42
- package/dist/cli/output.js +73 -110
- package/dist/cli/utils/config-loader.d.ts +6 -0
- package/dist/cli/utils/config-loader.js +19 -0
- package/dist/cli/utils/error-hints.d.ts +9 -0
- package/dist/cli/utils/error-hints.js +128 -0
- package/dist/cli/utils/headers.js +2 -25
- package/dist/cli/utils/path-resolution.d.ts +10 -0
- package/dist/cli/utils/path-resolution.js +27 -0
- package/dist/cli/utils/report-loader.d.ts +9 -0
- package/dist/cli/utils/report-loader.js +31 -0
- package/dist/cli/utils/server-runtime.d.ts +16 -0
- package/dist/cli/utils/server-runtime.js +31 -0
- package/dist/config/defaults.d.ts +0 -1
- package/dist/config/defaults.js +0 -1
- package/dist/constants/core.d.ts +0 -42
- package/dist/constants/core.js +0 -50
- package/dist/contract/validator.js +2 -47
- package/dist/interview/question-category.d.ts +5 -0
- package/dist/interview/question-category.js +2 -0
- package/dist/interview/question-types.d.ts +80 -0
- package/dist/interview/question-types.js +2 -0
- package/dist/interview/schema-test-generator.d.ts +3 -29
- package/dist/interview/schema-test-generator.js +11 -286
- package/dist/interview/test-fixtures.d.ts +19 -0
- package/dist/interview/test-fixtures.js +2 -0
- package/dist/interview/types.d.ts +5 -80
- package/dist/persona/types.d.ts +3 -5
- package/dist/scenarios/types.d.ts +1 -1
- package/dist/transport/auth-errors.d.ts +15 -0
- package/dist/transport/auth-errors.js +22 -0
- package/dist/transport/http-transport.js +7 -9
- package/dist/transport/mcp-client.d.ts +0 -4
- package/dist/transport/mcp-client.js +13 -37
- package/dist/transport/sse-transport.d.ts +0 -1
- package/dist/transport/sse-transport.js +13 -28
- package/dist/utils/content-type.d.ts +14 -0
- package/dist/utils/content-type.js +37 -0
- package/dist/utils/http-headers.d.ts +9 -0
- package/dist/utils/http-headers.js +34 -0
- package/dist/utils/smart-truncate.js +2 -23
- package/package.json +2 -2
|
@@ -13,9 +13,9 @@ import { MCPClient } from '../../transport/mcp-client.js';
|
|
|
13
13
|
import { discover } from '../../discovery/discovery.js';
|
|
14
14
|
import { Interviewer } from '../../interview/interviewer.js';
|
|
15
15
|
import { generateContractMd, generateJsonReport } from '../../docs/generator.js';
|
|
16
|
-
import { loadConfig, ConfigNotFoundError,
|
|
16
|
+
import { loadConfig, ConfigNotFoundError, } from '../../config/loader.js';
|
|
17
17
|
import { validateConfigForCheck, getConfigWarnings } from '../../config/validator.js';
|
|
18
|
-
import { createBaseline, loadBaseline, saveBaseline, getToolFingerprints, toToolCapability, compareBaselines, acceptDrift,
|
|
18
|
+
import { createBaseline, loadBaseline, saveBaseline, getToolFingerprints, toToolCapability, compareBaselines, acceptDrift, applySeverityConfig, shouldFailOnDiff, analyzeForIncremental, formatIncrementalSummary, runSecurityTests, parseSecurityCategories, getAllSecurityCategories, } from '../../baseline/index.js';
|
|
19
19
|
import { convertAssertions } from '../../baseline/converter.js';
|
|
20
20
|
import { getMetricsCollector, resetMetricsCollector } from '../../metrics/collector.js';
|
|
21
21
|
import { getGlobalCache, resetGlobalCache } from '../../cache/response-cache.js';
|
|
@@ -29,8 +29,10 @@ import { configureLogger } from '../../logging/logger.js';
|
|
|
29
29
|
import { buildInterviewInsights } from '../../interview/insights.js';
|
|
30
30
|
import { EXIT_CODES, SEVERITY_TO_EXIT_CODE, PATHS, SECURITY_TESTING, CHECK_SAMPLING, WORKFLOW, REPORT_SCHEMAS, PERCENTAGE_CONVERSION, MCP, } from '../../constants.js';
|
|
31
31
|
import { getFeatureFlags, getExcludedFeatureNames } from '../../protocol/index.js';
|
|
32
|
-
import {
|
|
33
|
-
import {
|
|
32
|
+
import { resolveServerRuntime } from '../utils/server-runtime.js';
|
|
33
|
+
import { resolvePathFromOutputDir, resolvePathFromOutputDirOrCwd, } from '../utils/path-resolution.js';
|
|
34
|
+
import { printCheckErrorHints } from '../utils/error-hints.js';
|
|
35
|
+
import { formatDiffOutput, formatCheckResults } from './check-formatters.js';
|
|
34
36
|
export const checkCommand = new Command('check')
|
|
35
37
|
.description('Check MCP server schema and detect drift (free, fast, deterministic)')
|
|
36
38
|
.allowUnknownOption() // Allow server flags like -y for npx to pass through
|
|
@@ -57,29 +59,27 @@ export const checkCommand = new Command('check')
|
|
|
57
59
|
}
|
|
58
60
|
throw error;
|
|
59
61
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
let
|
|
63
|
-
let
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
serverCommand = parsed.command;
|
|
68
|
-
args = parsed.args;
|
|
69
|
-
}
|
|
70
|
-
const transport = config.server.transport ?? 'stdio';
|
|
71
|
-
const remoteUrl = config.server.url?.trim();
|
|
72
|
-
const remoteSessionId = config.server.sessionId?.trim();
|
|
73
|
-
const configRemoteHeaders = config.server.headers;
|
|
74
|
-
let cliHeaders;
|
|
62
|
+
let serverCommand;
|
|
63
|
+
let args;
|
|
64
|
+
let transport;
|
|
65
|
+
let remoteUrl;
|
|
66
|
+
let remoteSessionId;
|
|
67
|
+
let remoteHeaders;
|
|
68
|
+
let serverIdentifier;
|
|
75
69
|
try {
|
|
76
|
-
|
|
70
|
+
const runtime = resolveServerRuntime(config, serverCommandArg, serverArgs, options.header);
|
|
71
|
+
serverCommand = runtime.serverCommand;
|
|
72
|
+
args = runtime.args;
|
|
73
|
+
transport = runtime.transport;
|
|
74
|
+
remoteUrl = runtime.remoteUrl;
|
|
75
|
+
remoteSessionId = runtime.remoteSessionId;
|
|
76
|
+
remoteHeaders = runtime.remoteHeaders;
|
|
77
|
+
serverIdentifier = runtime.serverIdentifier;
|
|
77
78
|
}
|
|
78
79
|
catch (error) {
|
|
79
80
|
output.error(error instanceof Error ? error.message : String(error));
|
|
80
81
|
process.exit(EXIT_CODES.ERROR);
|
|
81
82
|
}
|
|
82
|
-
const remoteHeaders = mergeHeaders(configRemoteHeaders, cliHeaders);
|
|
83
83
|
// Validate config for check
|
|
84
84
|
try {
|
|
85
85
|
validateConfigForCheck(config, serverCommand);
|
|
@@ -103,9 +103,16 @@ export const checkCommand = new Command('check')
|
|
|
103
103
|
const effectiveLogLevel = verbose ? logLevel : 'silent';
|
|
104
104
|
configureLogger({ level: effectiveLogLevel });
|
|
105
105
|
}
|
|
106
|
-
// Resolve baseline options from config (--fail-on-drift CLI flag can override)
|
|
107
|
-
|
|
108
|
-
|
|
106
|
+
// Resolve baseline options from config (--fail-on-drift CLI flag can override).
|
|
107
|
+
// Keep path semantics consistent with baseline subcommands:
|
|
108
|
+
// - savePath: relative to output.dir
|
|
109
|
+
// - comparePath: output.dir first, then cwd fallback for existing files
|
|
110
|
+
const baselinePath = config.baseline.comparePath
|
|
111
|
+
? resolvePathFromOutputDirOrCwd(config.baseline.comparePath, outputDir)
|
|
112
|
+
: undefined;
|
|
113
|
+
const saveBaselinePath = config.baseline.savePath
|
|
114
|
+
? resolvePathFromOutputDir(config.baseline.savePath, outputDir)
|
|
115
|
+
: undefined;
|
|
109
116
|
const failOnDrift = options.failOnDrift ? true : config.baseline.failOnDrift;
|
|
110
117
|
// Build severity config (CLI options override config file)
|
|
111
118
|
const severityConfig = {
|
|
@@ -162,9 +169,6 @@ export const checkCommand = new Command('check')
|
|
|
162
169
|
const fullExamples = config.output.examples.full;
|
|
163
170
|
const exampleLength = config.output.examples.maxLength;
|
|
164
171
|
const maxExamplesPerTool = config.output.examples.maxPerTool;
|
|
165
|
-
const serverIdentifier = transport === 'stdio'
|
|
166
|
-
? `${serverCommand} ${args.join(' ')}`.trim()
|
|
167
|
-
: (remoteUrl ?? 'unknown');
|
|
168
172
|
// Display startup banner
|
|
169
173
|
if (!machineReadable) {
|
|
170
174
|
const banner = formatCheckBanner({
|
|
@@ -872,7 +876,7 @@ export const checkCommand = new Command('check')
|
|
|
872
876
|
output.info('\n--- Drift Report ---');
|
|
873
877
|
}
|
|
874
878
|
// Select formatter based on --format option
|
|
875
|
-
const formattedDiff =
|
|
879
|
+
const formattedDiff = formatDiffOutput(diff, diffFormat, baselinePath);
|
|
876
880
|
if (machineReadable) {
|
|
877
881
|
console.log(formattedDiff);
|
|
878
882
|
}
|
|
@@ -968,44 +972,7 @@ export const checkCommand = new Command('check')
|
|
|
968
972
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
969
973
|
output.error('\n--- Check Failed ---');
|
|
970
974
|
output.error(`Error: ${errorMessage}`);
|
|
971
|
-
|
|
972
|
-
if (error instanceof ServerAuthError ||
|
|
973
|
-
errorMessage.includes('401') ||
|
|
974
|
-
errorMessage.includes('403') ||
|
|
975
|
-
errorMessage.includes('407') ||
|
|
976
|
-
/unauthorized|forbidden|authentication|authorization/i.test(errorMessage)) {
|
|
977
|
-
output.error('\nAuthentication failed:');
|
|
978
|
-
output.error(' - Add server.headers.Authorization in bellwether.yaml');
|
|
979
|
-
output.error(' - Or pass --header "Authorization: Bearer $TOKEN"');
|
|
980
|
-
output.error(' - Verify credentials are valid and not expired');
|
|
981
|
-
}
|
|
982
|
-
else if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('Connection refused')) {
|
|
983
|
-
output.error('\nPossible causes:');
|
|
984
|
-
if (isRemoteTransport) {
|
|
985
|
-
output.error(' - The remote MCP server is not reachable');
|
|
986
|
-
output.error(' - The server URL/port is incorrect');
|
|
987
|
-
}
|
|
988
|
-
else {
|
|
989
|
-
output.error(' - The MCP server is not running');
|
|
990
|
-
output.error(' - The server address/port is incorrect');
|
|
991
|
-
}
|
|
992
|
-
}
|
|
993
|
-
else if (isRemoteTransport && errorMessage.includes('HTTP 404')) {
|
|
994
|
-
output.error('\nPossible causes:');
|
|
995
|
-
output.error(' - The remote MCP URL is incorrect');
|
|
996
|
-
output.error(' - For SSE transport, verify the server exposes /sse');
|
|
997
|
-
}
|
|
998
|
-
else if (errorMessage.includes('timeout') || errorMessage.includes('Timeout')) {
|
|
999
|
-
output.error('\nPossible causes:');
|
|
1000
|
-
output.error(' - The MCP server is taking too long to respond');
|
|
1001
|
-
output.error(' - Increase server.timeout in bellwether.yaml');
|
|
1002
|
-
}
|
|
1003
|
-
else if (!isRemoteTransport &&
|
|
1004
|
-
(errorMessage.includes('ENOENT') || errorMessage.includes('not found'))) {
|
|
1005
|
-
output.error('\nPossible causes:');
|
|
1006
|
-
output.error(' - The server command was not found');
|
|
1007
|
-
output.error(' - Check that the command is installed and in PATH');
|
|
1008
|
-
}
|
|
975
|
+
printCheckErrorHints(error, transport);
|
|
1009
976
|
pendingExitCode = EXIT_CODES.ERROR;
|
|
1010
977
|
}
|
|
1011
978
|
finally {
|
|
@@ -1020,179 +987,4 @@ export const checkCommand = new Command('check')
|
|
|
1020
987
|
}
|
|
1021
988
|
}
|
|
1022
989
|
});
|
|
1023
|
-
/**
|
|
1024
|
-
* Format a diff using the specified output format.
|
|
1025
|
-
*
|
|
1026
|
-
* @param diff - The behavioral diff to format
|
|
1027
|
-
* @param format - Output format: text, json, compact, github, markdown, junit, sarif
|
|
1028
|
-
* @param baselinePath - Path to baseline file (used for SARIF location references)
|
|
1029
|
-
* @returns Formatted string
|
|
1030
|
-
*/
|
|
1031
|
-
function formatDiff(diff, format, baselinePath) {
|
|
1032
|
-
switch (format.toLowerCase()) {
|
|
1033
|
-
case 'json':
|
|
1034
|
-
return formatDiffJson(diff);
|
|
1035
|
-
case 'compact':
|
|
1036
|
-
return formatDiffCompact(diff);
|
|
1037
|
-
case 'github':
|
|
1038
|
-
return formatDiffGitHubActions(diff);
|
|
1039
|
-
case 'markdown':
|
|
1040
|
-
case 'md':
|
|
1041
|
-
return formatDiffMarkdown(diff);
|
|
1042
|
-
case 'junit':
|
|
1043
|
-
case 'junit-xml':
|
|
1044
|
-
case 'xml':
|
|
1045
|
-
return formatDiffJUnit(diff, 'bellwether-check');
|
|
1046
|
-
case 'sarif':
|
|
1047
|
-
return formatDiffSarif(diff, baselinePath);
|
|
1048
|
-
case 'text':
|
|
1049
|
-
default:
|
|
1050
|
-
return formatDiffText(diff);
|
|
1051
|
-
}
|
|
1052
|
-
}
|
|
1053
|
-
/**
|
|
1054
|
-
* Format check results as JUnit XML (for CI systems that expect test results).
|
|
1055
|
-
* This is used when --format junit is specified but no baseline comparison occurs.
|
|
1056
|
-
*/
|
|
1057
|
-
function formatCheckResultsJUnit(baseline) {
|
|
1058
|
-
const tools = getToolFingerprints(baseline);
|
|
1059
|
-
const lines = [];
|
|
1060
|
-
const securityFailures = tools.filter((t) => t.securityFingerprint?.findings?.some((f) => f.riskLevel === 'critical' || f.riskLevel === 'high')).length;
|
|
1061
|
-
lines.push('<?xml version="1.0" encoding="UTF-8"?>');
|
|
1062
|
-
lines.push('<testsuites>');
|
|
1063
|
-
lines.push(` <testsuite name="bellwether-check" tests="${tools.length}" failures="${securityFailures}" errors="0">`);
|
|
1064
|
-
for (const tool of tools) {
|
|
1065
|
-
const successRate = tool.baselineSuccessRate ?? 1;
|
|
1066
|
-
const status = successRate >= 0.9 ? 'passed' : 'warning';
|
|
1067
|
-
lines.push(` <testcase name="${tool.name}" classname="mcp-tools" time="0">`);
|
|
1068
|
-
lines.push(` <system-out>Success rate: ${(successRate * 100).toFixed(0)}%</system-out>`);
|
|
1069
|
-
if (status === 'warning') {
|
|
1070
|
-
lines.push(` <system-err>Tool has success rate below 90%</system-err>`);
|
|
1071
|
-
}
|
|
1072
|
-
lines.push(' </testcase>');
|
|
1073
|
-
}
|
|
1074
|
-
// Add security findings as test cases if present
|
|
1075
|
-
const securityTools = tools.filter((t) => t.securityFingerprint?.findings?.length);
|
|
1076
|
-
if (securityTools.length > 0) {
|
|
1077
|
-
lines.push(` <!-- Security findings -->`);
|
|
1078
|
-
for (const tool of securityTools) {
|
|
1079
|
-
const findings = tool.securityFingerprint?.findings ?? [];
|
|
1080
|
-
const criticalHigh = findings.filter((f) => f.riskLevel === 'critical' || f.riskLevel === 'high').length;
|
|
1081
|
-
if (criticalHigh > 0) {
|
|
1082
|
-
lines.push(` <testcase name="${tool.name}-security" classname="security">`);
|
|
1083
|
-
lines.push(` <failure message="${criticalHigh} critical/high security findings">`);
|
|
1084
|
-
for (const finding of findings.filter((f) => f.riskLevel === 'critical' || f.riskLevel === 'high')) {
|
|
1085
|
-
lines.push(` ${finding.riskLevel.toUpperCase()}: ${finding.title} (${finding.cweId})`);
|
|
1086
|
-
}
|
|
1087
|
-
lines.push(` </failure>`);
|
|
1088
|
-
lines.push(' </testcase>');
|
|
1089
|
-
}
|
|
1090
|
-
}
|
|
1091
|
-
}
|
|
1092
|
-
lines.push(' </testsuite>');
|
|
1093
|
-
lines.push('</testsuites>');
|
|
1094
|
-
return lines.join('\n');
|
|
1095
|
-
}
|
|
1096
|
-
/**
|
|
1097
|
-
* Format check results as SARIF (for GitHub Code Scanning and other tools).
|
|
1098
|
-
* This is used when --format sarif is specified but no baseline comparison occurs.
|
|
1099
|
-
*/
|
|
1100
|
-
function formatCheckResultsSarif(baseline) {
|
|
1101
|
-
const tools = getToolFingerprints(baseline);
|
|
1102
|
-
const serverUri = baseline.metadata?.serverCommand || baseline.server.name || 'mcp-server';
|
|
1103
|
-
const results = [];
|
|
1104
|
-
// Add results for tools with security findings
|
|
1105
|
-
const securityTools = tools.filter((t) => t.securityFingerprint?.findings?.length);
|
|
1106
|
-
for (const tool of securityTools) {
|
|
1107
|
-
const findings = tool.securityFingerprint?.findings ?? [];
|
|
1108
|
-
for (const finding of findings) {
|
|
1109
|
-
const level = finding.riskLevel === 'critical' || finding.riskLevel === 'high'
|
|
1110
|
-
? 'error'
|
|
1111
|
-
: finding.riskLevel === 'medium'
|
|
1112
|
-
? 'warning'
|
|
1113
|
-
: 'note';
|
|
1114
|
-
results.push({
|
|
1115
|
-
ruleId: finding.cweId || 'BWH-SEC',
|
|
1116
|
-
level,
|
|
1117
|
-
message: { text: `[${tool.name}] ${finding.title}: ${finding.description}` },
|
|
1118
|
-
locations: [
|
|
1119
|
-
{
|
|
1120
|
-
physicalLocation: {
|
|
1121
|
-
artifactLocation: { uri: serverUri },
|
|
1122
|
-
region: { startLine: 1 },
|
|
1123
|
-
},
|
|
1124
|
-
},
|
|
1125
|
-
],
|
|
1126
|
-
});
|
|
1127
|
-
}
|
|
1128
|
-
}
|
|
1129
|
-
// Add results for tools with low success rate
|
|
1130
|
-
for (const tool of tools) {
|
|
1131
|
-
const successRate = tool.baselineSuccessRate ?? 1;
|
|
1132
|
-
if (successRate < 0.9) {
|
|
1133
|
-
results.push({
|
|
1134
|
-
ruleId: 'BWH-REL',
|
|
1135
|
-
level: 'warning',
|
|
1136
|
-
message: {
|
|
1137
|
-
text: `Tool "${tool.name}" has ${(successRate * 100).toFixed(0)}% success rate`,
|
|
1138
|
-
},
|
|
1139
|
-
locations: [
|
|
1140
|
-
{
|
|
1141
|
-
physicalLocation: {
|
|
1142
|
-
artifactLocation: { uri: serverUri },
|
|
1143
|
-
region: { startLine: 1 },
|
|
1144
|
-
},
|
|
1145
|
-
},
|
|
1146
|
-
],
|
|
1147
|
-
});
|
|
1148
|
-
}
|
|
1149
|
-
}
|
|
1150
|
-
const sarif = {
|
|
1151
|
-
$schema: 'https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json',
|
|
1152
|
-
version: '2.1.0',
|
|
1153
|
-
runs: [
|
|
1154
|
-
{
|
|
1155
|
-
tool: {
|
|
1156
|
-
driver: {
|
|
1157
|
-
name: 'bellwether',
|
|
1158
|
-
version: '1.0.0',
|
|
1159
|
-
informationUri: 'https://github.com/dotsetlabs/bellwether',
|
|
1160
|
-
rules: [
|
|
1161
|
-
{
|
|
1162
|
-
id: 'BWH-SEC',
|
|
1163
|
-
name: 'SecurityFinding',
|
|
1164
|
-
shortDescription: { text: 'Security vulnerability detected' },
|
|
1165
|
-
defaultConfiguration: { level: 'warning' },
|
|
1166
|
-
},
|
|
1167
|
-
{
|
|
1168
|
-
id: 'BWH-REL',
|
|
1169
|
-
name: 'LowReliability',
|
|
1170
|
-
shortDescription: { text: 'Tool reliability below threshold' },
|
|
1171
|
-
defaultConfiguration: { level: 'warning' },
|
|
1172
|
-
},
|
|
1173
|
-
],
|
|
1174
|
-
},
|
|
1175
|
-
},
|
|
1176
|
-
results,
|
|
1177
|
-
},
|
|
1178
|
-
],
|
|
1179
|
-
};
|
|
1180
|
-
return JSON.stringify(sarif, null, 2);
|
|
1181
|
-
}
|
|
1182
|
-
/**
|
|
1183
|
-
* Format check results using the specified output format.
|
|
1184
|
-
* Used when no baseline comparison occurs.
|
|
1185
|
-
*/
|
|
1186
|
-
function formatCheckResults(baseline, format) {
|
|
1187
|
-
switch (format.toLowerCase()) {
|
|
1188
|
-
case 'junit':
|
|
1189
|
-
case 'junit-xml':
|
|
1190
|
-
case 'xml':
|
|
1191
|
-
return formatCheckResultsJUnit(baseline);
|
|
1192
|
-
case 'sarif':
|
|
1193
|
-
return formatCheckResultsSarif(baseline);
|
|
1194
|
-
default:
|
|
1195
|
-
return null; // No special formatting needed for other formats
|
|
1196
|
-
}
|
|
1197
|
-
}
|
|
1198
990
|
//# sourceMappingURL=check.js.map
|
|
@@ -13,24 +13,12 @@ import { loadContract, findContractFile, validateContract, generateContract, gen
|
|
|
13
13
|
import { MCPClient } from '../../transport/mcp-client.js';
|
|
14
14
|
import { discover } from '../../discovery/discovery.js';
|
|
15
15
|
import { EXIT_CODES, CONTRACT_TESTING } from '../../constants.js';
|
|
16
|
-
import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
|
|
17
16
|
import * as output from '../output.js';
|
|
17
|
+
import { loadConfigOrExit } from '../utils/config-loader.js';
|
|
18
18
|
/**
|
|
19
19
|
* Default paths for contract files.
|
|
20
20
|
*/
|
|
21
21
|
const DEFAULT_CONTRACT_FILENAMES = CONTRACT_TESTING.CONTRACT_FILENAMES;
|
|
22
|
-
function loadConfigOrExit(configPath) {
|
|
23
|
-
try {
|
|
24
|
-
return loadConfig(configPath);
|
|
25
|
-
}
|
|
26
|
-
catch (error) {
|
|
27
|
-
if (error instanceof ConfigNotFoundError) {
|
|
28
|
-
output.error(error.message);
|
|
29
|
-
process.exit(EXIT_CODES.ERROR);
|
|
30
|
-
}
|
|
31
|
-
throw error;
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
22
|
/**
|
|
35
23
|
* Find or use provided contract path.
|
|
36
24
|
*/
|
|
@@ -13,7 +13,7 @@ import { MCPClient } from '../../transport/mcp-client.js';
|
|
|
13
13
|
import { discover } from '../../discovery/discovery.js';
|
|
14
14
|
import { Interviewer } from '../../interview/interviewer.js';
|
|
15
15
|
import { generateAgentsMd, generateJsonReport } from '../../docs/generator.js';
|
|
16
|
-
import { loadConfig, ConfigNotFoundError,
|
|
16
|
+
import { loadConfig, ConfigNotFoundError, } from '../../config/loader.js';
|
|
17
17
|
import { validateConfigForExplore } from '../../config/validator.js';
|
|
18
18
|
import { CostTracker, estimateInterviewCost, estimateInterviewTime, formatCostAndTimeEstimate, suggestOptimizations, formatOptimizationSuggestions, } from '../../cost/index.js';
|
|
19
19
|
import { getMetricsCollector, resetMetricsCollector } from '../../metrics/collector.js';
|
|
@@ -31,8 +31,8 @@ import { suppressLogs, restoreLogLevel, configureLogger, } from '../../logging/l
|
|
|
31
31
|
import { extractServerContextFromArgs } from '../utils/server-context.js';
|
|
32
32
|
import { isCI } from '../utils/env.js';
|
|
33
33
|
import { buildInterviewInsights } from '../../interview/insights.js';
|
|
34
|
-
import {
|
|
35
|
-
import {
|
|
34
|
+
import { resolveServerRuntime } from '../utils/server-runtime.js';
|
|
35
|
+
import { printExploreErrorHints } from '../utils/error-hints.js';
|
|
36
36
|
/**
|
|
37
37
|
* Wrapper to parse personas with warning output.
|
|
38
38
|
*/
|
|
@@ -61,29 +61,27 @@ export const exploreCommand = new Command('explore')
|
|
|
61
61
|
}
|
|
62
62
|
throw error;
|
|
63
63
|
}
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
let
|
|
67
|
-
let
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
serverCommand = parsed.command;
|
|
72
|
-
args = parsed.args;
|
|
73
|
-
}
|
|
74
|
-
const transport = config.server.transport ?? 'stdio';
|
|
75
|
-
const remoteUrl = config.server.url?.trim();
|
|
76
|
-
const remoteSessionId = config.server.sessionId?.trim();
|
|
77
|
-
const configRemoteHeaders = config.server.headers;
|
|
78
|
-
let cliHeaders;
|
|
64
|
+
let serverCommand;
|
|
65
|
+
let args;
|
|
66
|
+
let transport;
|
|
67
|
+
let remoteUrl;
|
|
68
|
+
let remoteSessionId;
|
|
69
|
+
let remoteHeaders;
|
|
70
|
+
let serverIdentifier;
|
|
79
71
|
try {
|
|
80
|
-
|
|
72
|
+
const runtime = resolveServerRuntime(config, serverCommandArg, serverArgs, options.header);
|
|
73
|
+
serverCommand = runtime.serverCommand;
|
|
74
|
+
args = runtime.args;
|
|
75
|
+
transport = runtime.transport;
|
|
76
|
+
remoteUrl = runtime.remoteUrl;
|
|
77
|
+
remoteSessionId = runtime.remoteSessionId;
|
|
78
|
+
remoteHeaders = runtime.remoteHeaders;
|
|
79
|
+
serverIdentifier = runtime.serverIdentifier;
|
|
81
80
|
}
|
|
82
81
|
catch (error) {
|
|
83
82
|
output.error(error instanceof Error ? error.message : String(error));
|
|
84
83
|
process.exit(EXIT_CODES.ERROR);
|
|
85
84
|
}
|
|
86
|
-
const remoteHeaders = mergeHeaders(configRemoteHeaders, cliHeaders);
|
|
87
85
|
// Validate config for explore
|
|
88
86
|
try {
|
|
89
87
|
validateConfigForExplore(config, serverCommand);
|
|
@@ -113,9 +111,6 @@ export const exploreCommand = new Command('explore')
|
|
|
113
111
|
const provider = config.llm.provider;
|
|
114
112
|
const model = config.llm.model || undefined;
|
|
115
113
|
// Display startup banner
|
|
116
|
-
const serverIdentifier = transport === 'stdio'
|
|
117
|
-
? `${serverCommand} ${args.join(' ')}`.trim()
|
|
118
|
-
: (remoteUrl ?? 'unknown');
|
|
119
114
|
const banner = formatExploreBanner({
|
|
120
115
|
serverCommand: serverIdentifier,
|
|
121
116
|
provider,
|
|
@@ -500,49 +495,7 @@ export const exploreCommand = new Command('explore')
|
|
|
500
495
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
501
496
|
output.error('\n--- Exploration Failed ---');
|
|
502
497
|
output.error(`Error: ${errorMessage}`);
|
|
503
|
-
|
|
504
|
-
if (error instanceof ServerAuthError ||
|
|
505
|
-
errorMessage.includes('401') ||
|
|
506
|
-
errorMessage.includes('403') ||
|
|
507
|
-
errorMessage.includes('407') ||
|
|
508
|
-
/unauthorized|forbidden|authentication|authorization/i.test(errorMessage)) {
|
|
509
|
-
output.error('\nPossible causes:');
|
|
510
|
-
output.error(' - Missing or invalid remote MCP authentication headers');
|
|
511
|
-
output.error(' - Add server.headers.Authorization or pass --header "Authorization: Bearer $TOKEN"');
|
|
512
|
-
output.error(' - Verify token scopes/permissions');
|
|
513
|
-
}
|
|
514
|
-
else if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('Connection refused')) {
|
|
515
|
-
output.error('\nPossible causes:');
|
|
516
|
-
if (isRemoteTransport) {
|
|
517
|
-
output.error(' - The remote MCP server is not reachable');
|
|
518
|
-
output.error(' - The server URL/port is incorrect');
|
|
519
|
-
}
|
|
520
|
-
else {
|
|
521
|
-
output.error(' - The MCP server is not running');
|
|
522
|
-
output.error(' - The server address/port is incorrect');
|
|
523
|
-
}
|
|
524
|
-
}
|
|
525
|
-
else if (isRemoteTransport && errorMessage.includes('HTTP 404')) {
|
|
526
|
-
output.error('\nPossible causes:');
|
|
527
|
-
output.error(' - The remote MCP URL is incorrect');
|
|
528
|
-
output.error(' - For SSE transport, verify the server exposes /sse');
|
|
529
|
-
}
|
|
530
|
-
else if (errorMessage.includes('timeout') || errorMessage.includes('Timeout')) {
|
|
531
|
-
output.error('\nPossible causes:');
|
|
532
|
-
output.error(' - The MCP server is taking too long to respond');
|
|
533
|
-
output.error(' - Increase server.timeout in bellwether.yaml');
|
|
534
|
-
}
|
|
535
|
-
else if (!isRemoteTransport &&
|
|
536
|
-
(errorMessage.includes('ENOENT') || errorMessage.includes('not found'))) {
|
|
537
|
-
output.error('\nPossible causes:');
|
|
538
|
-
output.error(' - The server command was not found');
|
|
539
|
-
output.error(' - Check that the command is installed and in PATH');
|
|
540
|
-
}
|
|
541
|
-
else if (errorMessage.includes('API key') || errorMessage.includes('authentication')) {
|
|
542
|
-
output.error('\nPossible causes:');
|
|
543
|
-
output.error(' - Missing or invalid API key');
|
|
544
|
-
output.error(' - Run "bellwether auth" to configure API keys');
|
|
545
|
-
}
|
|
498
|
+
printExploreErrorHints(error, transport);
|
|
546
499
|
pendingExitCode = EXIT_CODES.ERROR;
|
|
547
500
|
}
|
|
548
501
|
finally {
|
|
@@ -18,6 +18,7 @@ import { validateConfigForCheck } from '../../config/validator.js';
|
|
|
18
18
|
import { createBaseline, saveBaseline, loadBaseline, compareBaselines, formatDiffText, } from '../../baseline/index.js';
|
|
19
19
|
import { EXIT_CODES } from '../../constants.js';
|
|
20
20
|
import * as output from '../output.js';
|
|
21
|
+
import { resolvePathFromOutputDir } from '../utils/path-resolution.js';
|
|
21
22
|
import { ServerAuthError } from '../../errors/types.js';
|
|
22
23
|
export const watchCommand = new Command('watch')
|
|
23
24
|
.description('Watch for file changes and auto-check (uses bellwether.yaml)')
|
|
@@ -60,9 +61,7 @@ export const watchCommand = new Command('watch')
|
|
|
60
61
|
const minSamples = config.check.sampling.minSamples;
|
|
61
62
|
// Baseline path for watch mode (use savePath or baseline default)
|
|
62
63
|
const baselinePathValue = config.baseline.savePath ?? config.baseline.path;
|
|
63
|
-
const baselinePath = baselinePathValue.
|
|
64
|
-
? baselinePathValue
|
|
65
|
-
: resolve(join(config.output.dir, baselinePathValue));
|
|
64
|
+
const baselinePath = resolve(resolvePathFromOutputDir(baselinePathValue, config.output.dir));
|
|
66
65
|
// Extract settings from config
|
|
67
66
|
const timeout = config.server.timeout;
|
|
68
67
|
const verbose = config.logging.verbose;
|
package/dist/cli/output.d.ts
CHANGED
|
@@ -182,46 +182,4 @@ export declare function createStreamingCallback(prefix?: string): {
|
|
|
182
182
|
onComplete: (text: string, operation: string) => void;
|
|
183
183
|
onError: (error: Error, operation: string) => void;
|
|
184
184
|
};
|
|
185
|
-
/**
|
|
186
|
-
* Diff summary data used for displaying comparison results.
|
|
187
|
-
*/
|
|
188
|
-
export interface DiffSummary {
|
|
189
|
-
severity: string;
|
|
190
|
-
toolsAdded: number;
|
|
191
|
-
toolsRemoved: number;
|
|
192
|
-
toolsModified: number;
|
|
193
|
-
behaviorChanges: number;
|
|
194
|
-
}
|
|
195
|
-
/**
|
|
196
|
-
* Get the label for a severity level.
|
|
197
|
-
*/
|
|
198
|
-
export declare function getSeverityIcon(severity: string): string;
|
|
199
|
-
/**
|
|
200
|
-
* Default export for convenient importing.
|
|
201
|
-
*/
|
|
202
|
-
declare const _default: {
|
|
203
|
-
configureOutput: typeof configureOutput;
|
|
204
|
-
getOutputConfig: typeof getOutputConfig;
|
|
205
|
-
resetOutput: typeof resetOutput;
|
|
206
|
-
isQuiet: typeof isQuiet;
|
|
207
|
-
info: typeof info;
|
|
208
|
-
success: typeof success;
|
|
209
|
-
warn: typeof warn;
|
|
210
|
-
error: typeof error;
|
|
211
|
-
debug: typeof debug;
|
|
212
|
-
newline: typeof newline;
|
|
213
|
-
lines: typeof lines;
|
|
214
|
-
json: typeof json;
|
|
215
|
-
section: typeof section;
|
|
216
|
-
keyValue: typeof keyValue;
|
|
217
|
-
listItem: typeof listItem;
|
|
218
|
-
numberedList: typeof numberedList;
|
|
219
|
-
createOutput: typeof createOutput;
|
|
220
|
-
Output: typeof Output;
|
|
221
|
-
StreamingDisplay: typeof StreamingDisplay;
|
|
222
|
-
createStreamingDisplay: typeof createStreamingDisplay;
|
|
223
|
-
createStreamingCallback: typeof createStreamingCallback;
|
|
224
|
-
getSeverityIcon: typeof getSeverityIcon;
|
|
225
|
-
};
|
|
226
|
-
export default _default;
|
|
227
185
|
//# sourceMappingURL=output.d.ts.map
|