@dotsetlabs/bellwether 2.1.0 → 2.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/CHANGELOG.md +35 -0
  2. package/README.md +48 -31
  3. package/dist/cli/commands/check.js +49 -6
  4. package/dist/cli/commands/dashboard.d.ts +3 -0
  5. package/dist/cli/commands/dashboard.js +69 -0
  6. package/dist/cli/commands/discover.js +24 -2
  7. package/dist/cli/commands/explore.js +49 -6
  8. package/dist/cli/commands/watch.js +12 -1
  9. package/dist/cli/index.js +27 -34
  10. package/dist/cli/utils/headers.d.ts +12 -0
  11. package/dist/cli/utils/headers.js +63 -0
  12. package/dist/config/defaults.d.ts +2 -0
  13. package/dist/config/defaults.js +2 -0
  14. package/dist/config/template.js +12 -0
  15. package/dist/config/validator.d.ts +38 -18
  16. package/dist/config/validator.js +10 -0
  17. package/dist/constants/core.d.ts +4 -2
  18. package/dist/constants/core.js +13 -2
  19. package/dist/dashboard/index.d.ts +3 -0
  20. package/dist/dashboard/index.js +6 -0
  21. package/dist/dashboard/runtime/artifact-index.d.ts +45 -0
  22. package/dist/dashboard/runtime/artifact-index.js +238 -0
  23. package/dist/dashboard/runtime/command-profiles.d.ts +764 -0
  24. package/dist/dashboard/runtime/command-profiles.js +691 -0
  25. package/dist/dashboard/runtime/config-service.d.ts +21 -0
  26. package/dist/dashboard/runtime/config-service.js +73 -0
  27. package/dist/dashboard/runtime/job-runner.d.ts +26 -0
  28. package/dist/dashboard/runtime/job-runner.js +292 -0
  29. package/dist/dashboard/security/input-validation.d.ts +3 -0
  30. package/dist/dashboard/security/input-validation.js +27 -0
  31. package/dist/dashboard/security/localhost-guard.d.ts +5 -0
  32. package/dist/dashboard/security/localhost-guard.js +52 -0
  33. package/dist/dashboard/server.d.ts +14 -0
  34. package/dist/dashboard/server.js +293 -0
  35. package/dist/dashboard/types.d.ts +55 -0
  36. package/dist/dashboard/types.js +2 -0
  37. package/dist/dashboard/ui.d.ts +2 -0
  38. package/dist/dashboard/ui.js +2264 -0
  39. package/dist/discovery/discovery.js +20 -1
  40. package/dist/discovery/types.d.ts +1 -1
  41. package/dist/docs/contract.js +7 -1
  42. package/dist/errors/retry.js +15 -1
  43. package/dist/errors/types.d.ts +10 -0
  44. package/dist/errors/types.js +28 -0
  45. package/dist/logging/logger.js +5 -2
  46. package/dist/transport/env-filter.d.ts +6 -0
  47. package/dist/transport/env-filter.js +76 -0
  48. package/dist/transport/http-transport.js +10 -0
  49. package/dist/transport/mcp-client.d.ts +16 -9
  50. package/dist/transport/mcp-client.js +119 -88
  51. package/dist/transport/sse-transport.js +19 -0
  52. package/dist/version.js +2 -2
  53. package/package.json +5 -15
  54. package/man/bellwether.1 +0 -204
  55. package/man/bellwether.1.md +0 -148
package/CHANGELOG.md CHANGED
@@ -7,6 +7,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [2.1.2] - 2026-02-16
11
+
12
+ ### Added
13
+
14
+ - **Remote MCP header auth support** across config, transport, and CLI:
15
+ - `server.headers` and `discovery.headers` configuration
16
+ - `-H/--header` overrides on `check`, `explore`, and `discover`
17
+ - `ServerAuthError` classification with auth-aware retry behavior
18
+ - **Header parsing utilities and tests** for validated, case-insensitive CLI/config header merging.
19
+
20
+ ### Changed
21
+
22
+ - **Remote diagnostics and guidance**: improved auth/connection error hints in `check`, `explore`, `discover`, and `watch`.
23
+ - **Capability handling**: `check`/`explore` now continue when prompts/resources exist even if no tools are exposed.
24
+ - **Documentation refresh** across README + website guides/CLI references to align with current auth, config, and command behavior.
25
+ - **Release consistency tooling**: simplified consistency validation by removing checks tied to deleted policy files.
26
+
27
+ ### Fixed
28
+
29
+ - **Remote preflight stream cleanup**: preflight now cancels response bodies to avoid leaving open remote streams before transport initialization.
30
+ - **Broken documentation links and stale examples**: removed/updated outdated references and invalid CLI examples.
31
+
32
+ ### Removed
33
+
34
+ - **Man page generation and distribution** (`man/`, `scripts/generate-manpage.sh`, `man:generate` script).
35
+ - **Husky/lint-staged workflow** and related hook files.
36
+ - **Repository files no longer maintained** (`ROADMAP.md`, `SECURITY.md`, `CODE_OF_CONDUCT.md`).
37
+
38
+ ## [2.1.1] - 2026-02-14
39
+
40
+ ### Changed
41
+
42
+ - **Product focus tightening**: Clarified the core workflow (`init`, `check`, `baseline`) and repositioned advanced commands as opt-in in CLI/docs.
43
+ - **Release quality hardening**: Added stronger consistency checks and documentation alignment to reduce drift between code behavior and published guidance.
44
+
10
45
  ## [2.1.0] - 2026-02-11
11
46
 
12
47
  ### Changed
package/README.md CHANGED
@@ -31,6 +31,15 @@ bellwether check
31
31
 
32
32
  That's it. No API keys. No LLM costs. Runs in seconds.
33
33
 
34
+ ## Product Focus
35
+
36
+ Bellwether is intentionally opinionated:
37
+
38
+ - **Core workflow (default)**: `init` -> `check` -> `baseline`
39
+ - **Advanced workflow (opt-in)**: `explore`, `watch`, `discover`, `golden`, `contract`, `registry`
40
+
41
+ If you only need CI-safe drift detection, you can stay entirely in the core workflow.
42
+
34
43
  ## Two Modes
35
44
 
36
45
  | Mode | Purpose | Cost | When to Use |
@@ -85,45 +94,29 @@ jobs:
85
94
 
86
95
  Comparisons are **protocol-version-aware** — version-specific fields (annotations, titles, output schemas, etc.) are only compared when both baselines support the relevant MCP protocol version.
87
96
 
88
- ## Commands
89
-
90
- ### Essential Commands
91
-
92
- ```bash
93
- bellwether init <server-command> # Create config
94
- bellwether check # Detect drift (free, deterministic)
95
- bellwether baseline save # Save baseline to compare against
96
- bellwether baseline compare # Compare current vs saved baseline
97
- ```
98
-
99
- ### Explore Command (Optional)
97
+ ## Command Tiers
100
98
 
101
- ```bash
102
- bellwether init --preset local npx @mcp/server # Uses Ollama (free)
103
- bellwether explore # LLM-powered testing
104
- ```
99
+ ### Core Commands (Recommended)
105
100
 
106
- Requires LLM (Ollama for free local, or OpenAI/Anthropic). Generates `AGENTS.md` with behavioral documentation.
101
+ | Command | Purpose |
102
+ |:--------|:--------|
103
+ | `init` | Create `bellwether.yaml` |
104
+ | `check` | Deterministic schema drift detection |
105
+ | `baseline save` | Save snapshot for future comparisons |
106
+ | `baseline compare` | Compare latest check output to saved baseline |
107
107
 
108
- ### All Commands
108
+ ### Advanced Commands (Optional)
109
109
 
110
110
  | Command | Purpose |
111
111
  |:--------|:--------|
112
- | `init` | Create `bellwether.yaml` config |
113
- | `check` | Schema drift detection (free) |
114
- | `explore` | LLM behavioral testing |
115
- | `baseline save` | Save test results as baseline |
116
- | `baseline compare` | Compare against baseline |
117
- | `baseline show` | Display baseline contents |
118
- | `baseline accept` | Accept drift as intentional |
119
- | `baseline diff` | Compare two baselines |
120
- | `discover` | Show server capabilities |
112
+ | `explore` | LLM behavioral testing and `AGENTS.md` generation |
121
113
  | `watch` | Continuous checking on file changes |
114
+ | `discover` | Capability inspection without tests |
122
115
  | `registry` | Search MCP Registry |
123
116
  | `golden` | Golden output regression testing |
124
- | `contract` | Contract validation (generate/validate/show) |
117
+ | `contract` | Contract validation and generation |
125
118
  | `auth` | Manage LLM provider API keys |
126
- | `validate-config` | Validate bellwether.yaml without running tests |
119
+ | `validate-config` | Validate `bellwether.yaml` without running tests |
127
120
 
128
121
  ## CI/CD Exit Codes
129
122
 
@@ -139,9 +132,9 @@ Requires LLM (Ollama for free local, or OpenAI/Anthropic). Generates `AGENTS.md`
139
132
  ## GitHub Action
140
133
 
141
134
  ```yaml
142
- - uses: dotsetlabs/bellwether@v2.0.0
135
+ - uses: dotsetlabs/bellwether@v2.1.2
143
136
  with:
144
- version: '2.0.0'
137
+ version: '2.1.2'
145
138
  server-command: 'npx @mcp/your-server'
146
139
  baseline-path: './bellwether-baseline.json'
147
140
  fail-on-severity: 'warning'
@@ -157,6 +150,22 @@ bellwether init --preset ci npx @mcp/server # Optimized for CI/CD
157
150
  bellwether init --preset local npx @mcp/server # Local Ollama (free)
158
151
  ```
159
152
 
153
+ For remote MCP servers that require auth headers, configure:
154
+
155
+ ```yaml
156
+ server:
157
+ transport: sse
158
+ url: "https://api.example.com/mcp"
159
+ headers:
160
+ Authorization: "Bearer ${MCP_SERVER_TOKEN}"
161
+ ```
162
+
163
+ Or use one-off CLI overrides:
164
+
165
+ ```bash
166
+ bellwether check -H "Authorization: Bearer $MCP_SERVER_TOKEN"
167
+ ```
168
+
160
169
  ## Environment Variables
161
170
 
162
171
  | Variable | Description |
@@ -170,8 +179,16 @@ bellwether init --preset local npx @mcp/server # Local Ollama (free)
170
179
  **[docs.bellwether.sh](https://docs.bellwether.sh)** — Full reference for configuration and commands.
171
180
 
172
181
  - [Quick Start](https://docs.bellwether.sh/quickstart)
182
+ - [Core vs Advanced](https://docs.bellwether.sh/concepts/core-vs-advanced)
173
183
  - [CLI Reference](https://docs.bellwether.sh/cli/init)
174
184
  - [CI/CD Integration](https://docs.bellwether.sh/guides/ci-cd)
185
+ - [Golden Paths](https://docs.bellwether.sh/guides/golden-paths)
186
+ - [Compatibility Policy](https://docs.bellwether.sh/concepts/compatibility-policy)
187
+
188
+ ## Project Governance
189
+
190
+ - [Changelog](./CHANGELOG.md)
191
+ - [Releases](https://github.com/dotsetlabs/bellwether/releases)
175
192
 
176
193
  ## Community
177
194
 
@@ -29,6 +29,8 @@ import { configureLogger } from '../../logging/logger.js';
29
29
  import { buildInterviewInsights } from '../../interview/insights.js';
30
30
  import { EXIT_CODES, SEVERITY_TO_EXIT_CODE, PATHS, SECURITY_TESTING, CHECK_SAMPLING, WORKFLOW, REPORT_SCHEMAS, PERCENTAGE_CONVERSION, MCP, } from '../../constants.js';
31
31
  import { getFeatureFlags, getExcludedFeatureNames } from '../../protocol/index.js';
32
+ import { ServerAuthError } from '../../errors/types.js';
33
+ import { mergeHeaders, parseCliHeaders } from '../utils/headers.js';
32
34
  export const checkCommand = new Command('check')
33
35
  .description('Check MCP server schema and detect drift (free, fast, deterministic)')
34
36
  .allowUnknownOption() // Allow server flags like -y for npx to pass through
@@ -41,6 +43,7 @@ export const checkCommand = new Command('check')
41
43
  .option('--format <format>', 'Diff output format: text, json, compact, github, markdown, junit, sarif')
42
44
  .option('--min-severity <level>', 'Minimum severity to report (overrides config): none, info, warning, breaking')
43
45
  .option('--fail-on-severity <level>', 'Fail threshold (overrides config): none, info, warning, breaking')
46
+ .option('-H, --header <header...>', 'Custom headers for remote MCP requests (e.g., "Authorization: Bearer token")')
44
47
  .action(async (serverCommandArg, serverArgs, options) => {
45
48
  // Load configuration
46
49
  let config;
@@ -67,6 +70,16 @@ export const checkCommand = new Command('check')
67
70
  const transport = config.server.transport ?? 'stdio';
68
71
  const remoteUrl = config.server.url?.trim();
69
72
  const remoteSessionId = config.server.sessionId?.trim();
73
+ const configRemoteHeaders = config.server.headers;
74
+ let cliHeaders;
75
+ try {
76
+ cliHeaders = parseCliHeaders(options.header);
77
+ }
78
+ catch (error) {
79
+ output.error(error instanceof Error ? error.message : String(error));
80
+ process.exit(EXIT_CODES.ERROR);
81
+ }
82
+ const remoteHeaders = mergeHeaders(configRemoteHeaders, cliHeaders);
70
83
  // Validate config for check
71
84
  try {
72
85
  validateConfigForCheck(config, serverCommand);
@@ -193,6 +206,7 @@ export const checkCommand = new Command('check')
193
206
  await mcpClient.connectRemote(remoteUrl, {
194
207
  transport,
195
208
  sessionId: remoteSessionId || undefined,
209
+ headers: remoteHeaders,
196
210
  });
197
211
  }
198
212
  // Discovery phase
@@ -255,12 +269,18 @@ export const checkCommand = new Command('check')
255
269
  toolsDiscovered: discovery.tools.length,
256
270
  personasUsed: 0, // No personas in check mode
257
271
  });
258
- if (discovery.tools.length === 0) {
259
- output.info('No tools found. Nothing to check.');
272
+ const hasInterviewTargets = discovery.tools.length > 0 ||
273
+ discovery.prompts.length > 0 ||
274
+ (discovery.resources?.length ?? 0) > 0;
275
+ if (!hasInterviewTargets) {
276
+ output.info('No tools, prompts, or resources found. Nothing to check.');
260
277
  metricsCollector.endInterview();
261
278
  await mcpClient.disconnect();
262
279
  return;
263
280
  }
281
+ if (discovery.tools.length === 0) {
282
+ output.info('No tools found; continuing with prompt/resource checks.');
283
+ }
264
284
  // Incremental checking - load baseline and determine which tools to test
265
285
  let incrementalBaseline = null;
266
286
  let incrementalResult = null;
@@ -948,17 +968,40 @@ export const checkCommand = new Command('check')
948
968
  const errorMessage = error instanceof Error ? error.message : String(error);
949
969
  output.error('\n--- Check Failed ---');
950
970
  output.error(`Error: ${errorMessage}`);
951
- if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('Connection refused')) {
971
+ const isRemoteTransport = transport !== 'stdio';
972
+ if (error instanceof ServerAuthError ||
973
+ errorMessage.includes('401') ||
974
+ errorMessage.includes('403') ||
975
+ errorMessage.includes('407') ||
976
+ /unauthorized|forbidden|authentication|authorization/i.test(errorMessage)) {
977
+ output.error('\nAuthentication failed:');
978
+ output.error(' - Add server.headers.Authorization in bellwether.yaml');
979
+ output.error(' - Or pass --header "Authorization: Bearer $TOKEN"');
980
+ output.error(' - Verify credentials are valid and not expired');
981
+ }
982
+ else if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('Connection refused')) {
983
+ output.error('\nPossible causes:');
984
+ if (isRemoteTransport) {
985
+ output.error(' - The remote MCP server is not reachable');
986
+ output.error(' - The server URL/port is incorrect');
987
+ }
988
+ else {
989
+ output.error(' - The MCP server is not running');
990
+ output.error(' - The server address/port is incorrect');
991
+ }
992
+ }
993
+ else if (isRemoteTransport && errorMessage.includes('HTTP 404')) {
952
994
  output.error('\nPossible causes:');
953
- output.error(' - The MCP server is not running');
954
- output.error(' - The server address/port is incorrect');
995
+ output.error(' - The remote MCP URL is incorrect');
996
+ output.error(' - For SSE transport, verify the server exposes /sse');
955
997
  }
956
998
  else if (errorMessage.includes('timeout') || errorMessage.includes('Timeout')) {
957
999
  output.error('\nPossible causes:');
958
1000
  output.error(' - The MCP server is taking too long to respond');
959
1001
  output.error(' - Increase server.timeout in bellwether.yaml');
960
1002
  }
961
- else if (errorMessage.includes('ENOENT') || errorMessage.includes('not found')) {
1003
+ else if (!isRemoteTransport &&
1004
+ (errorMessage.includes('ENOENT') || errorMessage.includes('not found'))) {
962
1005
  output.error('\nPossible causes:');
963
1006
  output.error(' - The server command was not found');
964
1007
  output.error(' - Check that the command is installed and in PATH');
@@ -0,0 +1,3 @@
1
+ import { Command } from 'commander';
2
+ export declare const dashboardCommand: Command;
3
+ //# sourceMappingURL=dashboard.d.ts.map
@@ -0,0 +1,69 @@
1
+ import { Command } from 'commander';
2
+ import { EXIT_CODES } from '../../constants.js';
3
+ import { startDashboard } from '../../dashboard/index.js';
4
+ import * as output from '../output.js';
5
+ const DEFAULT_HOST = '127.0.0.1';
6
+ const DEFAULT_PORT = 7331;
7
+ function parsePort(rawPort) {
8
+ const parsed = Number.parseInt(rawPort, 10);
9
+ if (!Number.isInteger(parsed) || parsed < 1 || parsed > 65535) {
10
+ throw new Error(`Invalid port "${rawPort}". Use a value between 1 and 65535.`);
11
+ }
12
+ return parsed;
13
+ }
14
+ export const dashboardCommand = new Command('dashboard')
15
+ .description('Start local web dashboard for Bellwether')
16
+ .option('--host <host>', 'Host interface to bind', DEFAULT_HOST)
17
+ .option('-p, --port <port>', 'Port to listen on', String(DEFAULT_PORT))
18
+ .action(async (options) => {
19
+ const host = String(options.host ?? DEFAULT_HOST).trim();
20
+ let port;
21
+ try {
22
+ port = parsePort(String(options.port ?? DEFAULT_PORT));
23
+ }
24
+ catch (error) {
25
+ output.error(error instanceof Error ? error.message : String(error));
26
+ process.exit(EXIT_CODES.ERROR);
27
+ return;
28
+ }
29
+ try {
30
+ const dashboard = await startDashboard({
31
+ host,
32
+ port,
33
+ cwd: process.cwd(),
34
+ cliEntrypoint: process.argv[1],
35
+ });
36
+ output.info('Bellwether Dashboard');
37
+ output.info(`URL: ${dashboard.url}`);
38
+ output.info('Available profiles: check, explore, validate-config, discover, watch, baseline.save, baseline.compare, baseline.show, baseline.diff, baseline.accept, registry.search, contract.validate, contract.generate, contract.show, golden.save, golden.compare, golden.list, golden.delete');
39
+ output.info('Press Ctrl+C to stop.');
40
+ let shuttingDown = false;
41
+ const shutdown = async (signal) => {
42
+ if (shuttingDown) {
43
+ return;
44
+ }
45
+ shuttingDown = true;
46
+ output.info(`\nReceived ${signal}. Stopping dashboard...`);
47
+ try {
48
+ await dashboard.stop();
49
+ }
50
+ catch (error) {
51
+ output.error(error instanceof Error ? error.message : String(error));
52
+ process.exit(EXIT_CODES.ERROR);
53
+ return;
54
+ }
55
+ process.exit(EXIT_CODES.CLEAN);
56
+ };
57
+ process.on('SIGINT', () => {
58
+ void shutdown('SIGINT');
59
+ });
60
+ process.on('SIGTERM', () => {
61
+ void shutdown('SIGTERM');
62
+ });
63
+ }
64
+ catch (error) {
65
+ output.error(`Failed to start dashboard: ${error instanceof Error ? error.message : String(error)}`);
66
+ process.exit(EXIT_CODES.ERROR);
67
+ }
68
+ });
69
+ //# sourceMappingURL=dashboard.js.map
@@ -4,6 +4,8 @@ import { discover, summarizeDiscovery } from '../../discovery/discovery.js';
4
4
  import { EXIT_CODES } from '../../constants.js';
5
5
  import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
6
6
  import * as output from '../output.js';
7
+ import { ServerAuthError } from '../../errors/types.js';
8
+ import { mergeHeaders, parseCliHeaders } from '../utils/headers.js';
7
9
  /**
8
10
  * Action handler for the discover command.
9
11
  */
@@ -28,6 +30,16 @@ async function discoverAction(command, args, options) {
28
30
  const outputJson = options.json ?? config?.discovery?.json ?? false;
29
31
  const remoteUrl = options.url ?? config?.discovery?.url;
30
32
  const sessionId = options.sessionId ?? config?.discovery?.sessionId;
33
+ const configuredHeaders = mergeHeaders(config?.server?.headers, config?.discovery?.headers);
34
+ let cliHeaders;
35
+ try {
36
+ cliHeaders = parseCliHeaders(options.header);
37
+ }
38
+ catch (error) {
39
+ output.error(error instanceof Error ? error.message : String(error));
40
+ process.exit(EXIT_CODES.ERROR);
41
+ }
42
+ const headers = mergeHeaders(configuredHeaders, cliHeaders);
31
43
  // Validate transport options
32
44
  if (isRemoteTransport && !remoteUrl) {
33
45
  output.error(`Error: --url is required when using --transport ${transportType}`);
@@ -49,10 +61,11 @@ async function discoverAction(command, args, options) {
49
61
  await client.connectRemote(remoteUrl, {
50
62
  transport: transportType,
51
63
  sessionId,
64
+ headers,
52
65
  });
53
66
  }
54
67
  else {
55
- await client.connect(command, args);
68
+ await client.connect(command, args, config?.server?.env);
56
69
  }
57
70
  output.info('Discovering capabilities...\n');
58
71
  const result = await discover(client, isRemoteTransport ? remoteUrl : command, isRemoteTransport ? [] : args);
@@ -83,7 +96,15 @@ async function discoverAction(command, args, options) {
83
96
  }
84
97
  }
85
98
  catch (error) {
86
- output.error(`Discovery failed: ${error instanceof Error ? error.message : String(error)}`);
99
+ const message = error instanceof Error ? error.message : String(error);
100
+ output.error(`Discovery failed: ${message}`);
101
+ if (error instanceof ServerAuthError ||
102
+ message.includes('401') ||
103
+ message.includes('403') ||
104
+ message.includes('407') ||
105
+ /unauthorized|forbidden|authentication|authorization/i.test(message)) {
106
+ output.error('Hint: configure discovery.headers/server.headers or pass --header.');
107
+ }
87
108
  process.exit(EXIT_CODES.ERROR);
88
109
  }
89
110
  finally {
@@ -101,5 +122,6 @@ export const discoverCommand = new Command('discover')
101
122
  .option('--transport <type>', 'Transport type: stdio, sse, streamable-http')
102
123
  .option('--url <url>', 'URL for remote MCP server (requires --transport sse or streamable-http)')
103
124
  .option('--session-id <id>', 'Session ID for remote server authentication')
125
+ .option('-H, --header <header...>', 'Custom headers for remote MCP requests (e.g., "Authorization: Bearer token")')
104
126
  .action(discoverAction);
105
127
  //# sourceMappingURL=discover.js.map
@@ -31,6 +31,8 @@ import { suppressLogs, restoreLogLevel, configureLogger, } from '../../logging/l
31
31
  import { extractServerContextFromArgs } from '../utils/server-context.js';
32
32
  import { isCI } from '../utils/env.js';
33
33
  import { buildInterviewInsights } from '../../interview/insights.js';
34
+ import { ServerAuthError } from '../../errors/types.js';
35
+ import { mergeHeaders, parseCliHeaders } from '../utils/headers.js';
34
36
  /**
35
37
  * Wrapper to parse personas with warning output.
36
38
  */
@@ -45,6 +47,7 @@ export const exploreCommand = new Command('explore')
45
47
  .argument('[server-command]', 'Server command (overrides config)')
46
48
  .argument('[args...]', 'Server arguments')
47
49
  .option('-c, --config <path>', 'Path to config file', PATHS.DEFAULT_CONFIG_FILENAME)
50
+ .option('-H, --header <header...>', 'Custom headers for remote MCP requests (e.g., "Authorization: Bearer token")')
48
51
  .action(async (serverCommandArg, serverArgs, options) => {
49
52
  // Load configuration
50
53
  let config;
@@ -71,6 +74,16 @@ export const exploreCommand = new Command('explore')
71
74
  const transport = config.server.transport ?? 'stdio';
72
75
  const remoteUrl = config.server.url?.trim();
73
76
  const remoteSessionId = config.server.sessionId?.trim();
77
+ const configRemoteHeaders = config.server.headers;
78
+ let cliHeaders;
79
+ try {
80
+ cliHeaders = parseCliHeaders(options.header);
81
+ }
82
+ catch (error) {
83
+ output.error(error instanceof Error ? error.message : String(error));
84
+ process.exit(EXIT_CODES.ERROR);
85
+ }
86
+ const remoteHeaders = mergeHeaders(configRemoteHeaders, cliHeaders);
74
87
  // Validate config for explore
75
88
  try {
76
89
  validateConfigForExplore(config, serverCommand);
@@ -184,6 +197,7 @@ export const exploreCommand = new Command('explore')
184
197
  await mcpClient.connectRemote(remoteUrl, {
185
198
  transport,
186
199
  sessionId: remoteSessionId || undefined,
200
+ headers: remoteHeaders,
187
201
  });
188
202
  }
189
203
  // Discovery phase
@@ -211,12 +225,18 @@ export const exploreCommand = new Command('explore')
211
225
  toolsDiscovered: discovery.tools.length,
212
226
  personasUsed: selectedPersonas.length,
213
227
  });
214
- if (discovery.tools.length === 0) {
215
- output.info('No tools found. Nothing to explore.');
228
+ const hasInterviewTargets = discovery.tools.length > 0 ||
229
+ discovery.prompts.length > 0 ||
230
+ (discovery.resources?.length ?? 0) > 0;
231
+ if (!hasInterviewTargets) {
232
+ output.info('No tools, prompts, or resources found. Nothing to explore.');
216
233
  metricsCollector.endInterview();
217
234
  await mcpClient.disconnect();
218
235
  return;
219
236
  }
237
+ if (discovery.tools.length === 0) {
238
+ output.info('No tools found; continuing with prompt/resource exploration.');
239
+ }
220
240
  // Show cost/time estimate (unless in CI)
221
241
  if (!isCI()) {
222
242
  const costEstimate = estimateInterviewCost(model || 'default', discovery.tools.length, maxQuestions, selectedPersonas.length);
@@ -480,17 +500,40 @@ export const exploreCommand = new Command('explore')
480
500
  const errorMessage = error instanceof Error ? error.message : String(error);
481
501
  output.error('\n--- Exploration Failed ---');
482
502
  output.error(`Error: ${errorMessage}`);
483
- if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('Connection refused')) {
503
+ const isRemoteTransport = transport !== 'stdio';
504
+ if (error instanceof ServerAuthError ||
505
+ errorMessage.includes('401') ||
506
+ errorMessage.includes('403') ||
507
+ errorMessage.includes('407') ||
508
+ /unauthorized|forbidden|authentication|authorization/i.test(errorMessage)) {
509
+ output.error('\nPossible causes:');
510
+ output.error(' - Missing or invalid remote MCP authentication headers');
511
+ output.error(' - Add server.headers.Authorization or pass --header "Authorization: Bearer $TOKEN"');
512
+ output.error(' - Verify token scopes/permissions');
513
+ }
514
+ else if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('Connection refused')) {
515
+ output.error('\nPossible causes:');
516
+ if (isRemoteTransport) {
517
+ output.error(' - The remote MCP server is not reachable');
518
+ output.error(' - The server URL/port is incorrect');
519
+ }
520
+ else {
521
+ output.error(' - The MCP server is not running');
522
+ output.error(' - The server address/port is incorrect');
523
+ }
524
+ }
525
+ else if (isRemoteTransport && errorMessage.includes('HTTP 404')) {
484
526
  output.error('\nPossible causes:');
485
- output.error(' - The MCP server is not running');
486
- output.error(' - The server address/port is incorrect');
527
+ output.error(' - The remote MCP URL is incorrect');
528
+ output.error(' - For SSE transport, verify the server exposes /sse');
487
529
  }
488
530
  else if (errorMessage.includes('timeout') || errorMessage.includes('Timeout')) {
489
531
  output.error('\nPossible causes:');
490
532
  output.error(' - The MCP server is taking too long to respond');
491
533
  output.error(' - Increase server.timeout in bellwether.yaml');
492
534
  }
493
- else if (errorMessage.includes('ENOENT') || errorMessage.includes('not found')) {
535
+ else if (!isRemoteTransport &&
536
+ (errorMessage.includes('ENOENT') || errorMessage.includes('not found'))) {
494
537
  output.error('\nPossible causes:');
495
538
  output.error(' - The server command was not found');
496
539
  output.error(' - Check that the command is installed and in PATH');
@@ -18,6 +18,7 @@ import { validateConfigForCheck } from '../../config/validator.js';
18
18
  import { createBaseline, saveBaseline, loadBaseline, compareBaselines, formatDiffText, } from '../../baseline/index.js';
19
19
  import { EXIT_CODES } from '../../constants.js';
20
20
  import * as output from '../output.js';
21
+ import { ServerAuthError } from '../../errors/types.js';
21
22
  export const watchCommand = new Command('watch')
22
23
  .description('Watch for file changes and auto-check (uses bellwether.yaml)')
23
24
  .argument('[server-command]', 'Server command (overrides config)')
@@ -42,6 +43,7 @@ export const watchCommand = new Command('watch')
42
43
  const transport = config.server.transport ?? 'stdio';
43
44
  const remoteUrl = config.server.url?.trim();
44
45
  const remoteSessionId = config.server.sessionId?.trim();
46
+ const remoteHeaders = config.server.headers;
45
47
  // Validate config for check mode (watch only does check, not explore)
46
48
  try {
47
49
  validateConfigForCheck(config, serverCommand);
@@ -97,6 +99,7 @@ export const watchCommand = new Command('watch')
97
99
  await mcpClient.connectRemote(remoteUrl, {
98
100
  transport,
99
101
  sessionId: remoteSessionId || undefined,
102
+ headers: remoteHeaders,
100
103
  });
101
104
  }
102
105
  const discovery = await discover(mcpClient, transport === 'stdio' ? serverCommand : (remoteUrl ?? serverCommand), transport === 'stdio' ? args : []);
@@ -168,7 +171,15 @@ export const watchCommand = new Command('watch')
168
171
  output.info(`Baseline updated: ${newBaseline.hash.slice(0, 8)}`);
169
172
  }
170
173
  catch (error) {
171
- output.error(`Test failed: ${error instanceof Error ? error.message : String(error)}`);
174
+ const message = error instanceof Error ? error.message : String(error);
175
+ output.error(`Test failed: ${message}`);
176
+ if (error instanceof ServerAuthError ||
177
+ message.includes('401') ||
178
+ message.includes('403') ||
179
+ message.includes('407') ||
180
+ /unauthorized|forbidden|authentication|authorization/i.test(message)) {
181
+ output.error('Hint: configure server.headers.Authorization in bellwether.yaml');
182
+ }
172
183
  }
173
184
  finally {
174
185
  await mcpClient.disconnect();
package/dist/cli/index.js CHANGED
@@ -86,42 +86,37 @@ Bellwether - MCP Server Validation & Documentation
86
86
  const examples = `
87
87
  Examples:
88
88
 
89
- Initialize configuration:
90
- $ bellwether init # Create bellwether.yaml
91
- $ bellwether init --preset ci # Optimized for CI/CD
92
- $ bellwether init --preset local # Local LLM with Ollama
89
+ Core workflow (recommended):
90
+ $ bellwether init npx @mcp/my-server # Create bellwether.yaml
91
+ $ bellwether check # Free, deterministic drift detection
92
+ $ bellwether baseline save # Save baseline snapshot
93
+ $ bellwether check --fail-on-drift # CI/CD gating
93
94
 
94
- Check for drift (free, fast, deterministic):
95
- $ bellwether check npx @mcp/my-server # Validate schemas
96
- $ bellwether baseline save # Save baseline
97
- $ bellwether baseline compare ./bellwether-baseline.json # Detect drift
98
-
99
- Explore behavior (LLM-powered):
100
- $ bellwether explore npx @mcp/my-server # Generate AGENTS.md documentation
101
-
102
- Discover server capabilities:
103
- $ bellwether discover npx @mcp/server-postgres
104
-
105
- Search MCP Registry:
106
- $ bellwether registry filesystem
95
+ Advanced workflow (opt-in):
96
+ $ bellwether explore # LLM behavioral exploration
97
+ $ bellwether discover # Quick capability inspection
98
+ $ bellwether watch # Continuous checking
107
99
 
108
100
  Documentation: https://docs.bellwether.sh
109
101
  `;
110
102
  program
111
103
  .name('bellwether')
112
104
  .description(`${banner}
113
- Check MCP servers for drift. Explore behavior. Generate documentation.
105
+ Deterministic MCP drift detection with an optional advanced analysis layer.
114
106
 
115
- Commands:
116
- check - Schema validation and drift detection (free, fast, deterministic)
117
- explore - LLM-powered behavioral exploration and documentation
118
- discover - Quick capability discovery (no tests)
119
- registry - Search the MCP Registry
120
- baseline - Manage baselines (save/compare/accept/diff/show)
121
- golden - Golden output regression testing
122
- contract - Contract validation (generate/validate/show)
123
- watch - Continuous checking on file changes
124
- auth - Manage LLM provider API keys
107
+ Core commands (default path):
108
+ init - Create bellwether.yaml configuration
109
+ check - Schema validation and drift detection (free, deterministic)
110
+ baseline - Save and compare baseline snapshots
111
+
112
+ Advanced commands (opt-in):
113
+ explore - LLM-powered behavioral exploration and documentation
114
+ discover - Quick capability discovery (no tests)
115
+ watch - Continuous checking on file changes
116
+ registry - Search the MCP Registry
117
+ golden - Golden output regression testing
118
+ contract - Contract validation (generate/validate/show)
119
+ auth - Manage LLM provider API keys
125
120
  validate-config - Validate bellwether.yaml without running tests
126
121
 
127
122
  For more information on a specific command, use:
@@ -155,19 +150,17 @@ For more information on a specific command, use:
155
150
  }
156
151
  })
157
152
  .addHelpText('after', examples);
158
- // Add command groups for better organization
159
- program.addHelpText('beforeAll', '\nCore Commands:');
160
- // Core commands - check and explore
153
+ program.addCommand(initCommand.description('Create a new bellwether.yaml configuration file'));
161
154
  program.addCommand(checkCommand.description('Check MCP server schema and detect drift (free, fast, deterministic)'));
155
+ program.addCommand(baselineCommand.description('Manage baselines for drift detection (save, compare, show, diff)'));
156
+ // Advanced commands
162
157
  program.addCommand(exploreCommand.description('Explore MCP server behavior with LLM-powered testing'));
163
158
  program.addCommand(watchCommand.description('Watch for MCP server changes and auto-check'));
164
159
  program.addCommand(discoverCommand.description('Discover MCP server capabilities (tools, prompts, resources)'));
165
- program.addCommand(initCommand.description('Create a new bellwether.yaml configuration file'));
166
- program.addCommand(authCommand.description('Manage LLM provider API keys (keychain storage)'));
167
- program.addCommand(baselineCommand.description('Manage baselines for drift detection (save, compare, show, diff)'));
168
160
  program.addCommand(goldenCommand.description('Manage golden outputs for tool validation (save, compare, list, delete)'));
169
161
  program.addCommand(registryCommand.description('Search the MCP Registry for servers'));
170
162
  program.addCommand(contractCommand.description('Validate MCP servers against contract definitions (validate, generate, show)'));
163
+ program.addCommand(authCommand.description('Manage LLM provider API keys (keychain storage)'));
171
164
  program.addCommand(validateConfigCommand.description('Validate bellwether.yaml configuration (no tests)'));
172
165
  // Custom help formatting
173
166
  program.configureHelp({