@dotsetlabs/bellwether 2.1.1 → 2.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/CHANGELOG.md +53 -0
  2. package/README.md +19 -4
  3. package/dist/baseline/golden-output.d.ts +0 -4
  4. package/dist/baseline/golden-output.js +2 -47
  5. package/dist/cli/commands/baseline-accept.js +14 -45
  6. package/dist/cli/commands/baseline.js +23 -78
  7. package/dist/cli/commands/check-formatters.d.ts +10 -0
  8. package/dist/cli/commands/check-formatters.js +160 -0
  9. package/dist/cli/commands/check.js +48 -213
  10. package/dist/cli/commands/contract.js +1 -13
  11. package/dist/cli/commands/dashboard.d.ts +3 -0
  12. package/dist/cli/commands/dashboard.js +69 -0
  13. package/dist/cli/commands/discover.js +24 -2
  14. package/dist/cli/commands/explore.js +34 -38
  15. package/dist/cli/commands/watch.js +14 -4
  16. package/dist/cli/output.d.ts +0 -42
  17. package/dist/cli/output.js +73 -110
  18. package/dist/cli/utils/config-loader.d.ts +6 -0
  19. package/dist/cli/utils/config-loader.js +19 -0
  20. package/dist/cli/utils/error-hints.d.ts +9 -0
  21. package/dist/cli/utils/error-hints.js +128 -0
  22. package/dist/cli/utils/headers.d.ts +12 -0
  23. package/dist/cli/utils/headers.js +40 -0
  24. package/dist/cli/utils/path-resolution.d.ts +10 -0
  25. package/dist/cli/utils/path-resolution.js +27 -0
  26. package/dist/cli/utils/report-loader.d.ts +9 -0
  27. package/dist/cli/utils/report-loader.js +31 -0
  28. package/dist/cli/utils/server-runtime.d.ts +16 -0
  29. package/dist/cli/utils/server-runtime.js +31 -0
  30. package/dist/config/defaults.d.ts +2 -1
  31. package/dist/config/defaults.js +2 -1
  32. package/dist/config/template.js +12 -0
  33. package/dist/config/validator.d.ts +38 -18
  34. package/dist/config/validator.js +10 -0
  35. package/dist/constants/core.d.ts +2 -42
  36. package/dist/constants/core.js +11 -50
  37. package/dist/contract/validator.js +2 -47
  38. package/dist/dashboard/index.d.ts +3 -0
  39. package/dist/dashboard/index.js +6 -0
  40. package/dist/dashboard/runtime/artifact-index.d.ts +45 -0
  41. package/dist/dashboard/runtime/artifact-index.js +238 -0
  42. package/dist/dashboard/runtime/command-profiles.d.ts +764 -0
  43. package/dist/dashboard/runtime/command-profiles.js +691 -0
  44. package/dist/dashboard/runtime/config-service.d.ts +21 -0
  45. package/dist/dashboard/runtime/config-service.js +73 -0
  46. package/dist/dashboard/runtime/job-runner.d.ts +26 -0
  47. package/dist/dashboard/runtime/job-runner.js +292 -0
  48. package/dist/dashboard/security/input-validation.d.ts +3 -0
  49. package/dist/dashboard/security/input-validation.js +27 -0
  50. package/dist/dashboard/security/localhost-guard.d.ts +5 -0
  51. package/dist/dashboard/security/localhost-guard.js +52 -0
  52. package/dist/dashboard/server.d.ts +14 -0
  53. package/dist/dashboard/server.js +293 -0
  54. package/dist/dashboard/types.d.ts +55 -0
  55. package/dist/dashboard/types.js +2 -0
  56. package/dist/dashboard/ui.d.ts +2 -0
  57. package/dist/dashboard/ui.js +2264 -0
  58. package/dist/discovery/discovery.js +20 -1
  59. package/dist/discovery/types.d.ts +1 -1
  60. package/dist/docs/contract.js +7 -1
  61. package/dist/errors/retry.js +15 -1
  62. package/dist/errors/types.d.ts +10 -0
  63. package/dist/errors/types.js +28 -0
  64. package/dist/interview/question-category.d.ts +5 -0
  65. package/dist/interview/question-category.js +2 -0
  66. package/dist/interview/question-types.d.ts +80 -0
  67. package/dist/interview/question-types.js +2 -0
  68. package/dist/interview/schema-test-generator.d.ts +3 -29
  69. package/dist/interview/schema-test-generator.js +11 -286
  70. package/dist/interview/test-fixtures.d.ts +19 -0
  71. package/dist/interview/test-fixtures.js +2 -0
  72. package/dist/interview/types.d.ts +5 -80
  73. package/dist/persona/types.d.ts +3 -5
  74. package/dist/scenarios/types.d.ts +1 -1
  75. package/dist/transport/auth-errors.d.ts +15 -0
  76. package/dist/transport/auth-errors.js +22 -0
  77. package/dist/transport/http-transport.js +8 -0
  78. package/dist/transport/mcp-client.d.ts +12 -0
  79. package/dist/transport/mcp-client.js +92 -4
  80. package/dist/transport/sse-transport.d.ts +0 -1
  81. package/dist/transport/sse-transport.js +15 -11
  82. package/dist/utils/content-type.d.ts +14 -0
  83. package/dist/utils/content-type.js +37 -0
  84. package/dist/utils/http-headers.d.ts +9 -0
  85. package/dist/utils/http-headers.js +34 -0
  86. package/dist/utils/smart-truncate.js +2 -23
  87. package/package.json +4 -15
  88. package/man/bellwether.1 +0 -204
  89. package/man/bellwether.1.md +0 -148
@@ -13,9 +13,9 @@ import { MCPClient } from '../../transport/mcp-client.js';
13
13
  import { discover } from '../../discovery/discovery.js';
14
14
  import { Interviewer } from '../../interview/interviewer.js';
15
15
  import { generateContractMd, generateJsonReport } from '../../docs/generator.js';
16
- import { loadConfig, ConfigNotFoundError, parseCommandString, } from '../../config/loader.js';
16
+ import { loadConfig, ConfigNotFoundError, } from '../../config/loader.js';
17
17
  import { validateConfigForCheck, getConfigWarnings } from '../../config/validator.js';
18
- import { createBaseline, loadBaseline, saveBaseline, getToolFingerprints, toToolCapability, compareBaselines, acceptDrift, formatDiffText, formatDiffJson, formatDiffCompact, formatDiffGitHubActions, formatDiffMarkdown, formatDiffJUnit, formatDiffSarif, applySeverityConfig, shouldFailOnDiff, analyzeForIncremental, formatIncrementalSummary, runSecurityTests, parseSecurityCategories, getAllSecurityCategories, } from '../../baseline/index.js';
18
+ import { createBaseline, loadBaseline, saveBaseline, getToolFingerprints, toToolCapability, compareBaselines, acceptDrift, applySeverityConfig, shouldFailOnDiff, analyzeForIncremental, formatIncrementalSummary, runSecurityTests, parseSecurityCategories, getAllSecurityCategories, } from '../../baseline/index.js';
19
19
  import { convertAssertions } from '../../baseline/converter.js';
20
20
  import { getMetricsCollector, resetMetricsCollector } from '../../metrics/collector.js';
21
21
  import { getGlobalCache, resetGlobalCache } from '../../cache/response-cache.js';
@@ -29,6 +29,10 @@ import { configureLogger } from '../../logging/logger.js';
29
29
  import { buildInterviewInsights } from '../../interview/insights.js';
30
30
  import { EXIT_CODES, SEVERITY_TO_EXIT_CODE, PATHS, SECURITY_TESTING, CHECK_SAMPLING, WORKFLOW, REPORT_SCHEMAS, PERCENTAGE_CONVERSION, MCP, } from '../../constants.js';
31
31
  import { getFeatureFlags, getExcludedFeatureNames } from '../../protocol/index.js';
32
+ import { resolveServerRuntime } from '../utils/server-runtime.js';
33
+ import { resolvePathFromOutputDir, resolvePathFromOutputDirOrCwd, } from '../utils/path-resolution.js';
34
+ import { printCheckErrorHints } from '../utils/error-hints.js';
35
+ import { formatDiffOutput, formatCheckResults } from './check-formatters.js';
32
36
  export const checkCommand = new Command('check')
33
37
  .description('Check MCP server schema and detect drift (free, fast, deterministic)')
34
38
  .allowUnknownOption() // Allow server flags like -y for npx to pass through
@@ -41,6 +45,7 @@ export const checkCommand = new Command('check')
41
45
  .option('--format <format>', 'Diff output format: text, json, compact, github, markdown, junit, sarif')
42
46
  .option('--min-severity <level>', 'Minimum severity to report (overrides config): none, info, warning, breaking')
43
47
  .option('--fail-on-severity <level>', 'Fail threshold (overrides config): none, info, warning, breaking')
48
+ .option('-H, --header <header...>', 'Custom headers for remote MCP requests (e.g., "Authorization: Bearer token")')
44
49
  .action(async (serverCommandArg, serverArgs, options) => {
45
50
  // Load configuration
46
51
  let config;
@@ -54,19 +59,27 @@ export const checkCommand = new Command('check')
54
59
  }
55
60
  throw error;
56
61
  }
57
- // Determine server command (CLI arg overrides config)
58
- // If command string contains spaces and no separate args, parse it
59
- let serverCommand = serverCommandArg || config.server.command;
60
- let args = serverArgs.length > 0 ? serverArgs : config.server.args;
61
- // Handle command strings like "npx @package" in config when args is empty
62
- if (!serverCommandArg && args.length === 0 && serverCommand.includes(' ')) {
63
- const parsed = parseCommandString(serverCommand);
64
- serverCommand = parsed.command;
65
- args = parsed.args;
62
+ let serverCommand;
63
+ let args;
64
+ let transport;
65
+ let remoteUrl;
66
+ let remoteSessionId;
67
+ let remoteHeaders;
68
+ let serverIdentifier;
69
+ try {
70
+ const runtime = resolveServerRuntime(config, serverCommandArg, serverArgs, options.header);
71
+ serverCommand = runtime.serverCommand;
72
+ args = runtime.args;
73
+ transport = runtime.transport;
74
+ remoteUrl = runtime.remoteUrl;
75
+ remoteSessionId = runtime.remoteSessionId;
76
+ remoteHeaders = runtime.remoteHeaders;
77
+ serverIdentifier = runtime.serverIdentifier;
78
+ }
79
+ catch (error) {
80
+ output.error(error instanceof Error ? error.message : String(error));
81
+ process.exit(EXIT_CODES.ERROR);
66
82
  }
67
- const transport = config.server.transport ?? 'stdio';
68
- const remoteUrl = config.server.url?.trim();
69
- const remoteSessionId = config.server.sessionId?.trim();
70
83
  // Validate config for check
71
84
  try {
72
85
  validateConfigForCheck(config, serverCommand);
@@ -90,9 +103,16 @@ export const checkCommand = new Command('check')
90
103
  const effectiveLogLevel = verbose ? logLevel : 'silent';
91
104
  configureLogger({ level: effectiveLogLevel });
92
105
  }
93
- // Resolve baseline options from config (--fail-on-drift CLI flag can override)
94
- const baselinePath = config.baseline.comparePath;
95
- const saveBaselinePath = config.baseline.savePath;
106
+ // Resolve baseline options from config (--fail-on-drift CLI flag can override).
107
+ // Keep path semantics consistent with baseline subcommands:
108
+ // - savePath: relative to output.dir
109
+ // - comparePath: output.dir first, then cwd fallback for existing files
110
+ const baselinePath = config.baseline.comparePath
111
+ ? resolvePathFromOutputDirOrCwd(config.baseline.comparePath, outputDir)
112
+ : undefined;
113
+ const saveBaselinePath = config.baseline.savePath
114
+ ? resolvePathFromOutputDir(config.baseline.savePath, outputDir)
115
+ : undefined;
96
116
  const failOnDrift = options.failOnDrift ? true : config.baseline.failOnDrift;
97
117
  // Build severity config (CLI options override config file)
98
118
  const severityConfig = {
@@ -149,9 +169,6 @@ export const checkCommand = new Command('check')
149
169
  const fullExamples = config.output.examples.full;
150
170
  const exampleLength = config.output.examples.maxLength;
151
171
  const maxExamplesPerTool = config.output.examples.maxPerTool;
152
- const serverIdentifier = transport === 'stdio'
153
- ? `${serverCommand} ${args.join(' ')}`.trim()
154
- : (remoteUrl ?? 'unknown');
155
172
  // Display startup banner
156
173
  if (!machineReadable) {
157
174
  const banner = formatCheckBanner({
@@ -193,6 +210,7 @@ export const checkCommand = new Command('check')
193
210
  await mcpClient.connectRemote(remoteUrl, {
194
211
  transport,
195
212
  sessionId: remoteSessionId || undefined,
213
+ headers: remoteHeaders,
196
214
  });
197
215
  }
198
216
  // Discovery phase
@@ -255,12 +273,18 @@ export const checkCommand = new Command('check')
255
273
  toolsDiscovered: discovery.tools.length,
256
274
  personasUsed: 0, // No personas in check mode
257
275
  });
258
- if (discovery.tools.length === 0) {
259
- output.info('No tools found. Nothing to check.');
276
+ const hasInterviewTargets = discovery.tools.length > 0 ||
277
+ discovery.prompts.length > 0 ||
278
+ (discovery.resources?.length ?? 0) > 0;
279
+ if (!hasInterviewTargets) {
280
+ output.info('No tools, prompts, or resources found. Nothing to check.');
260
281
  metricsCollector.endInterview();
261
282
  await mcpClient.disconnect();
262
283
  return;
263
284
  }
285
+ if (discovery.tools.length === 0) {
286
+ output.info('No tools found; continuing with prompt/resource checks.');
287
+ }
264
288
  // Incremental checking - load baseline and determine which tools to test
265
289
  let incrementalBaseline = null;
266
290
  let incrementalResult = null;
@@ -852,7 +876,7 @@ export const checkCommand = new Command('check')
852
876
  output.info('\n--- Drift Report ---');
853
877
  }
854
878
  // Select formatter based on --format option
855
- const formattedDiff = formatDiff(diff, diffFormat, baselinePath);
879
+ const formattedDiff = formatDiffOutput(diff, diffFormat, baselinePath);
856
880
  if (machineReadable) {
857
881
  console.log(formattedDiff);
858
882
  }
@@ -948,21 +972,7 @@ export const checkCommand = new Command('check')
948
972
  const errorMessage = error instanceof Error ? error.message : String(error);
949
973
  output.error('\n--- Check Failed ---');
950
974
  output.error(`Error: ${errorMessage}`);
951
- if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('Connection refused')) {
952
- output.error('\nPossible causes:');
953
- output.error(' - The MCP server is not running');
954
- output.error(' - The server address/port is incorrect');
955
- }
956
- else if (errorMessage.includes('timeout') || errorMessage.includes('Timeout')) {
957
- output.error('\nPossible causes:');
958
- output.error(' - The MCP server is taking too long to respond');
959
- output.error(' - Increase server.timeout in bellwether.yaml');
960
- }
961
- else if (errorMessage.includes('ENOENT') || errorMessage.includes('not found')) {
962
- output.error('\nPossible causes:');
963
- output.error(' - The server command was not found');
964
- output.error(' - Check that the command is installed and in PATH');
965
- }
975
+ printCheckErrorHints(error, transport);
966
976
  pendingExitCode = EXIT_CODES.ERROR;
967
977
  }
968
978
  finally {
@@ -977,179 +987,4 @@ export const checkCommand = new Command('check')
977
987
  }
978
988
  }
979
989
  });
980
- /**
981
- * Format a diff using the specified output format.
982
- *
983
- * @param diff - The behavioral diff to format
984
- * @param format - Output format: text, json, compact, github, markdown, junit, sarif
985
- * @param baselinePath - Path to baseline file (used for SARIF location references)
986
- * @returns Formatted string
987
- */
988
- function formatDiff(diff, format, baselinePath) {
989
- switch (format.toLowerCase()) {
990
- case 'json':
991
- return formatDiffJson(diff);
992
- case 'compact':
993
- return formatDiffCompact(diff);
994
- case 'github':
995
- return formatDiffGitHubActions(diff);
996
- case 'markdown':
997
- case 'md':
998
- return formatDiffMarkdown(diff);
999
- case 'junit':
1000
- case 'junit-xml':
1001
- case 'xml':
1002
- return formatDiffJUnit(diff, 'bellwether-check');
1003
- case 'sarif':
1004
- return formatDiffSarif(diff, baselinePath);
1005
- case 'text':
1006
- default:
1007
- return formatDiffText(diff);
1008
- }
1009
- }
1010
- /**
1011
- * Format check results as JUnit XML (for CI systems that expect test results).
1012
- * This is used when --format junit is specified but no baseline comparison occurs.
1013
- */
1014
- function formatCheckResultsJUnit(baseline) {
1015
- const tools = getToolFingerprints(baseline);
1016
- const lines = [];
1017
- const securityFailures = tools.filter((t) => t.securityFingerprint?.findings?.some((f) => f.riskLevel === 'critical' || f.riskLevel === 'high')).length;
1018
- lines.push('<?xml version="1.0" encoding="UTF-8"?>');
1019
- lines.push('<testsuites>');
1020
- lines.push(` <testsuite name="bellwether-check" tests="${tools.length}" failures="${securityFailures}" errors="0">`);
1021
- for (const tool of tools) {
1022
- const successRate = tool.baselineSuccessRate ?? 1;
1023
- const status = successRate >= 0.9 ? 'passed' : 'warning';
1024
- lines.push(` <testcase name="${tool.name}" classname="mcp-tools" time="0">`);
1025
- lines.push(` <system-out>Success rate: ${(successRate * 100).toFixed(0)}%</system-out>`);
1026
- if (status === 'warning') {
1027
- lines.push(` <system-err>Tool has success rate below 90%</system-err>`);
1028
- }
1029
- lines.push(' </testcase>');
1030
- }
1031
- // Add security findings as test cases if present
1032
- const securityTools = tools.filter((t) => t.securityFingerprint?.findings?.length);
1033
- if (securityTools.length > 0) {
1034
- lines.push(` <!-- Security findings -->`);
1035
- for (const tool of securityTools) {
1036
- const findings = tool.securityFingerprint?.findings ?? [];
1037
- const criticalHigh = findings.filter((f) => f.riskLevel === 'critical' || f.riskLevel === 'high').length;
1038
- if (criticalHigh > 0) {
1039
- lines.push(` <testcase name="${tool.name}-security" classname="security">`);
1040
- lines.push(` <failure message="${criticalHigh} critical/high security findings">`);
1041
- for (const finding of findings.filter((f) => f.riskLevel === 'critical' || f.riskLevel === 'high')) {
1042
- lines.push(` ${finding.riskLevel.toUpperCase()}: ${finding.title} (${finding.cweId})`);
1043
- }
1044
- lines.push(` </failure>`);
1045
- lines.push(' </testcase>');
1046
- }
1047
- }
1048
- }
1049
- lines.push(' </testsuite>');
1050
- lines.push('</testsuites>');
1051
- return lines.join('\n');
1052
- }
1053
- /**
1054
- * Format check results as SARIF (for GitHub Code Scanning and other tools).
1055
- * This is used when --format sarif is specified but no baseline comparison occurs.
1056
- */
1057
- function formatCheckResultsSarif(baseline) {
1058
- const tools = getToolFingerprints(baseline);
1059
- const serverUri = baseline.metadata?.serverCommand || baseline.server.name || 'mcp-server';
1060
- const results = [];
1061
- // Add results for tools with security findings
1062
- const securityTools = tools.filter((t) => t.securityFingerprint?.findings?.length);
1063
- for (const tool of securityTools) {
1064
- const findings = tool.securityFingerprint?.findings ?? [];
1065
- for (const finding of findings) {
1066
- const level = finding.riskLevel === 'critical' || finding.riskLevel === 'high'
1067
- ? 'error'
1068
- : finding.riskLevel === 'medium'
1069
- ? 'warning'
1070
- : 'note';
1071
- results.push({
1072
- ruleId: finding.cweId || 'BWH-SEC',
1073
- level,
1074
- message: { text: `[${tool.name}] ${finding.title}: ${finding.description}` },
1075
- locations: [
1076
- {
1077
- physicalLocation: {
1078
- artifactLocation: { uri: serverUri },
1079
- region: { startLine: 1 },
1080
- },
1081
- },
1082
- ],
1083
- });
1084
- }
1085
- }
1086
- // Add results for tools with low success rate
1087
- for (const tool of tools) {
1088
- const successRate = tool.baselineSuccessRate ?? 1;
1089
- if (successRate < 0.9) {
1090
- results.push({
1091
- ruleId: 'BWH-REL',
1092
- level: 'warning',
1093
- message: {
1094
- text: `Tool "${tool.name}" has ${(successRate * 100).toFixed(0)}% success rate`,
1095
- },
1096
- locations: [
1097
- {
1098
- physicalLocation: {
1099
- artifactLocation: { uri: serverUri },
1100
- region: { startLine: 1 },
1101
- },
1102
- },
1103
- ],
1104
- });
1105
- }
1106
- }
1107
- const sarif = {
1108
- $schema: 'https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json',
1109
- version: '2.1.0',
1110
- runs: [
1111
- {
1112
- tool: {
1113
- driver: {
1114
- name: 'bellwether',
1115
- version: '1.0.0',
1116
- informationUri: 'https://github.com/dotsetlabs/bellwether',
1117
- rules: [
1118
- {
1119
- id: 'BWH-SEC',
1120
- name: 'SecurityFinding',
1121
- shortDescription: { text: 'Security vulnerability detected' },
1122
- defaultConfiguration: { level: 'warning' },
1123
- },
1124
- {
1125
- id: 'BWH-REL',
1126
- name: 'LowReliability',
1127
- shortDescription: { text: 'Tool reliability below threshold' },
1128
- defaultConfiguration: { level: 'warning' },
1129
- },
1130
- ],
1131
- },
1132
- },
1133
- results,
1134
- },
1135
- ],
1136
- };
1137
- return JSON.stringify(sarif, null, 2);
1138
- }
1139
- /**
1140
- * Format check results using the specified output format.
1141
- * Used when no baseline comparison occurs.
1142
- */
1143
- function formatCheckResults(baseline, format) {
1144
- switch (format.toLowerCase()) {
1145
- case 'junit':
1146
- case 'junit-xml':
1147
- case 'xml':
1148
- return formatCheckResultsJUnit(baseline);
1149
- case 'sarif':
1150
- return formatCheckResultsSarif(baseline);
1151
- default:
1152
- return null; // No special formatting needed for other formats
1153
- }
1154
- }
1155
990
  //# sourceMappingURL=check.js.map
@@ -13,24 +13,12 @@ import { loadContract, findContractFile, validateContract, generateContract, gen
13
13
  import { MCPClient } from '../../transport/mcp-client.js';
14
14
  import { discover } from '../../discovery/discovery.js';
15
15
  import { EXIT_CODES, CONTRACT_TESTING } from '../../constants.js';
16
- import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
17
16
  import * as output from '../output.js';
17
+ import { loadConfigOrExit } from '../utils/config-loader.js';
18
18
  /**
19
19
  * Default paths for contract files.
20
20
  */
21
21
  const DEFAULT_CONTRACT_FILENAMES = CONTRACT_TESTING.CONTRACT_FILENAMES;
22
- function loadConfigOrExit(configPath) {
23
- try {
24
- return loadConfig(configPath);
25
- }
26
- catch (error) {
27
- if (error instanceof ConfigNotFoundError) {
28
- output.error(error.message);
29
- process.exit(EXIT_CODES.ERROR);
30
- }
31
- throw error;
32
- }
33
- }
34
22
  /**
35
23
  * Find or use provided contract path.
36
24
  */
@@ -0,0 +1,3 @@
1
+ import { Command } from 'commander';
2
+ export declare const dashboardCommand: Command;
3
+ //# sourceMappingURL=dashboard.d.ts.map
@@ -0,0 +1,69 @@
1
+ import { Command } from 'commander';
2
+ import { EXIT_CODES } from '../../constants.js';
3
+ import { startDashboard } from '../../dashboard/index.js';
4
+ import * as output from '../output.js';
5
+ const DEFAULT_HOST = '127.0.0.1';
6
+ const DEFAULT_PORT = 7331;
7
+ function parsePort(rawPort) {
8
+ const parsed = Number.parseInt(rawPort, 10);
9
+ if (!Number.isInteger(parsed) || parsed < 1 || parsed > 65535) {
10
+ throw new Error(`Invalid port "${rawPort}". Use a value between 1 and 65535.`);
11
+ }
12
+ return parsed;
13
+ }
14
+ export const dashboardCommand = new Command('dashboard')
15
+ .description('Start local web dashboard for Bellwether')
16
+ .option('--host <host>', 'Host interface to bind', DEFAULT_HOST)
17
+ .option('-p, --port <port>', 'Port to listen on', String(DEFAULT_PORT))
18
+ .action(async (options) => {
19
+ const host = String(options.host ?? DEFAULT_HOST).trim();
20
+ let port;
21
+ try {
22
+ port = parsePort(String(options.port ?? DEFAULT_PORT));
23
+ }
24
+ catch (error) {
25
+ output.error(error instanceof Error ? error.message : String(error));
26
+ process.exit(EXIT_CODES.ERROR);
27
+ return;
28
+ }
29
+ try {
30
+ const dashboard = await startDashboard({
31
+ host,
32
+ port,
33
+ cwd: process.cwd(),
34
+ cliEntrypoint: process.argv[1],
35
+ });
36
+ output.info('Bellwether Dashboard');
37
+ output.info(`URL: ${dashboard.url}`);
38
+ output.info('Available profiles: check, explore, validate-config, discover, watch, baseline.save, baseline.compare, baseline.show, baseline.diff, baseline.accept, registry.search, contract.validate, contract.generate, contract.show, golden.save, golden.compare, golden.list, golden.delete');
39
+ output.info('Press Ctrl+C to stop.');
40
+ let shuttingDown = false;
41
+ const shutdown = async (signal) => {
42
+ if (shuttingDown) {
43
+ return;
44
+ }
45
+ shuttingDown = true;
46
+ output.info(`\nReceived ${signal}. Stopping dashboard...`);
47
+ try {
48
+ await dashboard.stop();
49
+ }
50
+ catch (error) {
51
+ output.error(error instanceof Error ? error.message : String(error));
52
+ process.exit(EXIT_CODES.ERROR);
53
+ return;
54
+ }
55
+ process.exit(EXIT_CODES.CLEAN);
56
+ };
57
+ process.on('SIGINT', () => {
58
+ void shutdown('SIGINT');
59
+ });
60
+ process.on('SIGTERM', () => {
61
+ void shutdown('SIGTERM');
62
+ });
63
+ }
64
+ catch (error) {
65
+ output.error(`Failed to start dashboard: ${error instanceof Error ? error.message : String(error)}`);
66
+ process.exit(EXIT_CODES.ERROR);
67
+ }
68
+ });
69
+ //# sourceMappingURL=dashboard.js.map
@@ -4,6 +4,8 @@ import { discover, summarizeDiscovery } from '../../discovery/discovery.js';
4
4
  import { EXIT_CODES } from '../../constants.js';
5
5
  import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
6
6
  import * as output from '../output.js';
7
+ import { ServerAuthError } from '../../errors/types.js';
8
+ import { mergeHeaders, parseCliHeaders } from '../utils/headers.js';
7
9
  /**
8
10
  * Action handler for the discover command.
9
11
  */
@@ -28,6 +30,16 @@ async function discoverAction(command, args, options) {
28
30
  const outputJson = options.json ?? config?.discovery?.json ?? false;
29
31
  const remoteUrl = options.url ?? config?.discovery?.url;
30
32
  const sessionId = options.sessionId ?? config?.discovery?.sessionId;
33
+ const configuredHeaders = mergeHeaders(config?.server?.headers, config?.discovery?.headers);
34
+ let cliHeaders;
35
+ try {
36
+ cliHeaders = parseCliHeaders(options.header);
37
+ }
38
+ catch (error) {
39
+ output.error(error instanceof Error ? error.message : String(error));
40
+ process.exit(EXIT_CODES.ERROR);
41
+ }
42
+ const headers = mergeHeaders(configuredHeaders, cliHeaders);
31
43
  // Validate transport options
32
44
  if (isRemoteTransport && !remoteUrl) {
33
45
  output.error(`Error: --url is required when using --transport ${transportType}`);
@@ -49,10 +61,11 @@ async function discoverAction(command, args, options) {
49
61
  await client.connectRemote(remoteUrl, {
50
62
  transport: transportType,
51
63
  sessionId,
64
+ headers,
52
65
  });
53
66
  }
54
67
  else {
55
- await client.connect(command, args);
68
+ await client.connect(command, args, config?.server?.env);
56
69
  }
57
70
  output.info('Discovering capabilities...\n');
58
71
  const result = await discover(client, isRemoteTransport ? remoteUrl : command, isRemoteTransport ? [] : args);
@@ -83,7 +96,15 @@ async function discoverAction(command, args, options) {
83
96
  }
84
97
  }
85
98
  catch (error) {
86
- output.error(`Discovery failed: ${error instanceof Error ? error.message : String(error)}`);
99
+ const message = error instanceof Error ? error.message : String(error);
100
+ output.error(`Discovery failed: ${message}`);
101
+ if (error instanceof ServerAuthError ||
102
+ message.includes('401') ||
103
+ message.includes('403') ||
104
+ message.includes('407') ||
105
+ /unauthorized|forbidden|authentication|authorization/i.test(message)) {
106
+ output.error('Hint: configure discovery.headers/server.headers or pass --header.');
107
+ }
87
108
  process.exit(EXIT_CODES.ERROR);
88
109
  }
89
110
  finally {
@@ -101,5 +122,6 @@ export const discoverCommand = new Command('discover')
101
122
  .option('--transport <type>', 'Transport type: stdio, sse, streamable-http')
102
123
  .option('--url <url>', 'URL for remote MCP server (requires --transport sse or streamable-http)')
103
124
  .option('--session-id <id>', 'Session ID for remote server authentication')
125
+ .option('-H, --header <header...>', 'Custom headers for remote MCP requests (e.g., "Authorization: Bearer token")')
104
126
  .action(discoverAction);
105
127
  //# sourceMappingURL=discover.js.map
@@ -13,7 +13,7 @@ import { MCPClient } from '../../transport/mcp-client.js';
13
13
  import { discover } from '../../discovery/discovery.js';
14
14
  import { Interviewer } from '../../interview/interviewer.js';
15
15
  import { generateAgentsMd, generateJsonReport } from '../../docs/generator.js';
16
- import { loadConfig, ConfigNotFoundError, parseCommandString, } from '../../config/loader.js';
16
+ import { loadConfig, ConfigNotFoundError, } from '../../config/loader.js';
17
17
  import { validateConfigForExplore } from '../../config/validator.js';
18
18
  import { CostTracker, estimateInterviewCost, estimateInterviewTime, formatCostAndTimeEstimate, suggestOptimizations, formatOptimizationSuggestions, } from '../../cost/index.js';
19
19
  import { getMetricsCollector, resetMetricsCollector } from '../../metrics/collector.js';
@@ -31,6 +31,8 @@ import { suppressLogs, restoreLogLevel, configureLogger, } from '../../logging/l
31
31
  import { extractServerContextFromArgs } from '../utils/server-context.js';
32
32
  import { isCI } from '../utils/env.js';
33
33
  import { buildInterviewInsights } from '../../interview/insights.js';
34
+ import { resolveServerRuntime } from '../utils/server-runtime.js';
35
+ import { printExploreErrorHints } from '../utils/error-hints.js';
34
36
  /**
35
37
  * Wrapper to parse personas with warning output.
36
38
  */
@@ -45,6 +47,7 @@ export const exploreCommand = new Command('explore')
45
47
  .argument('[server-command]', 'Server command (overrides config)')
46
48
  .argument('[args...]', 'Server arguments')
47
49
  .option('-c, --config <path>', 'Path to config file', PATHS.DEFAULT_CONFIG_FILENAME)
50
+ .option('-H, --header <header...>', 'Custom headers for remote MCP requests (e.g., "Authorization: Bearer token")')
48
51
  .action(async (serverCommandArg, serverArgs, options) => {
49
52
  // Load configuration
50
53
  let config;
@@ -58,19 +61,27 @@ export const exploreCommand = new Command('explore')
58
61
  }
59
62
  throw error;
60
63
  }
61
- // Determine server command (CLI arg overrides config)
62
- // If command string contains spaces and no separate args, parse it
63
- let serverCommand = serverCommandArg || config.server.command;
64
- let args = serverArgs.length > 0 ? serverArgs : config.server.args;
65
- // Handle command strings like "npx @package" in config when args is empty
66
- if (!serverCommandArg && args.length === 0 && serverCommand.includes(' ')) {
67
- const parsed = parseCommandString(serverCommand);
68
- serverCommand = parsed.command;
69
- args = parsed.args;
64
+ let serverCommand;
65
+ let args;
66
+ let transport;
67
+ let remoteUrl;
68
+ let remoteSessionId;
69
+ let remoteHeaders;
70
+ let serverIdentifier;
71
+ try {
72
+ const runtime = resolveServerRuntime(config, serverCommandArg, serverArgs, options.header);
73
+ serverCommand = runtime.serverCommand;
74
+ args = runtime.args;
75
+ transport = runtime.transport;
76
+ remoteUrl = runtime.remoteUrl;
77
+ remoteSessionId = runtime.remoteSessionId;
78
+ remoteHeaders = runtime.remoteHeaders;
79
+ serverIdentifier = runtime.serverIdentifier;
80
+ }
81
+ catch (error) {
82
+ output.error(error instanceof Error ? error.message : String(error));
83
+ process.exit(EXIT_CODES.ERROR);
70
84
  }
71
- const transport = config.server.transport ?? 'stdio';
72
- const remoteUrl = config.server.url?.trim();
73
- const remoteSessionId = config.server.sessionId?.trim();
74
85
  // Validate config for explore
75
86
  try {
76
87
  validateConfigForExplore(config, serverCommand);
@@ -100,9 +111,6 @@ export const exploreCommand = new Command('explore')
100
111
  const provider = config.llm.provider;
101
112
  const model = config.llm.model || undefined;
102
113
  // Display startup banner
103
- const serverIdentifier = transport === 'stdio'
104
- ? `${serverCommand} ${args.join(' ')}`.trim()
105
- : (remoteUrl ?? 'unknown');
106
114
  const banner = formatExploreBanner({
107
115
  serverCommand: serverIdentifier,
108
116
  provider,
@@ -184,6 +192,7 @@ export const exploreCommand = new Command('explore')
184
192
  await mcpClient.connectRemote(remoteUrl, {
185
193
  transport,
186
194
  sessionId: remoteSessionId || undefined,
195
+ headers: remoteHeaders,
187
196
  });
188
197
  }
189
198
  // Discovery phase
@@ -211,12 +220,18 @@ export const exploreCommand = new Command('explore')
211
220
  toolsDiscovered: discovery.tools.length,
212
221
  personasUsed: selectedPersonas.length,
213
222
  });
214
- if (discovery.tools.length === 0) {
215
- output.info('No tools found. Nothing to explore.');
223
+ const hasInterviewTargets = discovery.tools.length > 0 ||
224
+ discovery.prompts.length > 0 ||
225
+ (discovery.resources?.length ?? 0) > 0;
226
+ if (!hasInterviewTargets) {
227
+ output.info('No tools, prompts, or resources found. Nothing to explore.');
216
228
  metricsCollector.endInterview();
217
229
  await mcpClient.disconnect();
218
230
  return;
219
231
  }
232
+ if (discovery.tools.length === 0) {
233
+ output.info('No tools found; continuing with prompt/resource exploration.');
234
+ }
220
235
  // Show cost/time estimate (unless in CI)
221
236
  if (!isCI()) {
222
237
  const costEstimate = estimateInterviewCost(model || 'default', discovery.tools.length, maxQuestions, selectedPersonas.length);
@@ -480,26 +495,7 @@ export const exploreCommand = new Command('explore')
480
495
  const errorMessage = error instanceof Error ? error.message : String(error);
481
496
  output.error('\n--- Exploration Failed ---');
482
497
  output.error(`Error: ${errorMessage}`);
483
- if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('Connection refused')) {
484
- output.error('\nPossible causes:');
485
- output.error(' - The MCP server is not running');
486
- output.error(' - The server address/port is incorrect');
487
- }
488
- else if (errorMessage.includes('timeout') || errorMessage.includes('Timeout')) {
489
- output.error('\nPossible causes:');
490
- output.error(' - The MCP server is taking too long to respond');
491
- output.error(' - Increase server.timeout in bellwether.yaml');
492
- }
493
- else if (errorMessage.includes('ENOENT') || errorMessage.includes('not found')) {
494
- output.error('\nPossible causes:');
495
- output.error(' - The server command was not found');
496
- output.error(' - Check that the command is installed and in PATH');
497
- }
498
- else if (errorMessage.includes('API key') || errorMessage.includes('authentication')) {
499
- output.error('\nPossible causes:');
500
- output.error(' - Missing or invalid API key');
501
- output.error(' - Run "bellwether auth" to configure API keys');
502
- }
498
+ printExploreErrorHints(error, transport);
503
499
  pendingExitCode = EXIT_CODES.ERROR;
504
500
  }
505
501
  finally {