@dotsetlabs/bellwether 0.10.0

package/dist/cli/commands/watch.js

@@ -0,0 +1,253 @@
+/**
+ * Watch command - watch for file changes and auto-check.
+ *
+ * Uses bellwether.yaml for configuration.
+ * Only watch-specific options are available as flags.
+ *
+ * Note: Watch mode only runs schema validation (check mode).
+ * For LLM-powered exploration, use 'bellwether explore' directly.
+ */
+import { Command } from 'commander';
+import { existsSync, readdirSync, statSync } from 'fs';
+import { resolve, join } from 'path';
+import { MCPClient } from '../../transport/mcp-client.js';
+import { discover } from '../../discovery/discovery.js';
+import { Interviewer } from '../../interview/interviewer.js';
+import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
+import { validateConfigForCheck } from '../../config/validator.js';
+import { createBaseline, saveBaseline, loadBaseline, compareBaselines, formatDiffText, } from '../../baseline/index.js';
+import { EXIT_CODES, CHECK_SAMPLING } from '../../constants.js';
+import * as output from '../output.js';
+export const watchCommand = new Command('watch')
+    .description('Watch for file changes and auto-check (uses bellwether.yaml)')
+    .argument('[server-command]', 'Server command (overrides config)')
+    .argument('[args...]', 'Server arguments')
+    .option('-c, --config <path>', 'Path to config file')
+    .action(async (serverCommandArg, serverArgs, options) => {
+    // Load configuration (required)
+    let config;
+    try {
+        config = loadConfig(options.config);
+    }
+    catch (error) {
+        if (error instanceof ConfigNotFoundError) {
+            output.error(error.message);
+            process.exit(EXIT_CODES.ERROR);
+        }
+        throw error;
+    }
+    // Determine server command (CLI arg overrides config)
+    const serverCommand = serverCommandArg || config.server.command;
+    const args = serverArgs.length > 0 ? serverArgs : config.server.args;
+    // Validate config for check mode (watch only does check, not explore)
+    try {
+        validateConfigForCheck(config, serverCommand);
+    }
+    catch (error) {
+        output.error(error instanceof Error ? error.message : String(error));
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Get watch settings from config
+    const watchPath = resolve(config.watch.path);
+    const interval = config.watch.interval;
+    const extensions = config.watch.extensions;
+    const onDriftCommand = config.watch.onDrift;
+    const minSamplesForConfidence = CHECK_SAMPLING.SAMPLES_FOR_CONFIDENCE[config.check.sampling.targetConfidence];
+    const minSamples = Math.max(config.check.sampling.minSamples, minSamplesForConfidence);
+    // Baseline path for watch mode (use savePath or baseline default)
+    const baselinePathValue = config.baseline.savePath ?? config.baseline.path;
+    const baselinePath = baselinePathValue.startsWith('/')
+        ? baselinePathValue
+        : resolve(join(config.output.dir, baselinePathValue));
+    // Extract settings from config
+    const timeout = config.server.timeout;
+    const verbose = config.logging.verbose;
+    output.info('Bellwether Watch Mode\n');
+    output.info(`Server: ${serverCommand} ${args.join(' ')}`);
+    output.info(`Mode: check (schema validation)`);
+    output.info(`Watching: ${watchPath}`);
+    output.info(`Baseline: ${baselinePath}`);
+    output.info(`Poll interval: ${interval}ms`);
+    output.info('');
+    output.info('Note: Watch mode runs schema validation only. Use "bellwether explore" for LLM analysis.');
+    output.info('');
+    // Track last baseline hash to detect changes
+    let lastBaselineHash = null;
+    if (existsSync(baselinePath)) {
+        const baseline = loadBaseline(baselinePath);
+        lastBaselineHash = baseline.integrityHash;
+        output.info(`Loaded existing baseline: ${lastBaselineHash.slice(0, 8)}`);
+    }
+    // Track watched file modification times
+    const fileModTimes = new Map();
+    async function runTest() {
+        const mcpClient = new MCPClient({ timeout });
+        try {
+            output.info('\n--- Running Test ---');
+            output.info(`[${new Date().toLocaleTimeString()}] Starting test...`);
+            await mcpClient.connect(serverCommand, args, config.server.env);
+            const discovery = await discover(mcpClient, serverCommand, args);
+            output.info(`Found ${discovery.tools.length} tools`);
+            if (discovery.tools.length === 0) {
+                output.info('No tools found. Skipping.');
+                await mcpClient.disconnect();
+                return;
+            }
+            const fullServerCommand = `${serverCommand} ${args.join(' ')}`.trim();
+            // Watch mode uses check (no LLM) for fast, deterministic drift detection
+            const interviewer = new Interviewer(null, {
+                maxQuestionsPerTool: minSamples,
+                timeout,
+                skipErrorTests: false,
+                model: 'check',
+                personas: [],
+                checkMode: true, // Required when passing null for LLM
+                parallelTools: config.check.parallel,
+                toolConcurrency: config.check.parallelWorkers,
+                serverCommand: fullServerCommand,
+            });
+            const progressCallback = (progress) => {
+                process.stdout.write(`\rChecking: ${progress.toolsCompleted + 1}/${progress.totalTools} tools`.padEnd(60));
+            };
+            const result = await interviewer.interview(mcpClient, discovery, progressCallback);
+            output.info('\n');
+            // Create and compare baseline
+            const newBaseline = createBaseline(result, fullServerCommand);
+            if (lastBaselineHash && existsSync(baselinePath)) {
+                const previousBaseline = loadBaseline(baselinePath);
+                const diff = compareBaselines(previousBaseline, newBaseline, {});
+                if (diff.severity !== 'none') {
+                    output.info('\n--- Behavioral Drift Detected ---');
+                    output.info(formatDiffText(diff));
+                    // Run on-drift command if configured
+                    if (onDriftCommand) {
+                        output.info(`\nRunning: ${onDriftCommand}`);
+                        const { spawnSync } = await import('child_process');
+                        try {
+                            // Parse command safely - split on spaces but respect quoted strings
+                            const parts = onDriftCommand.match(/(?:[^\s"]+|"[^"]*")+/g);
+                            if (!parts || parts.length === 0) {
+                                throw new Error('Empty on-drift command');
+                            }
+                            const [cmd, ...rest] = parts;
+                            const cmdArgs = rest.map((arg) => arg.replace(/^"|"$/g, ''));
+                            // Use spawnSync without shell to prevent command injection
+                            const cmdResult = spawnSync(cmd, cmdArgs, { stdio: 'inherit' });
+                            if (cmdResult.error) {
+                                throw cmdResult.error;
+                            }
+                            if (cmdResult.status !== 0) {
+                                output.error(`On-drift command exited with code ${cmdResult.status}`);
+                            }
+                        }
+                        catch (e) {
+                            output.error('On-drift command failed: ' + (e instanceof Error ? e.message : String(e)));
+                        }
+                    }
+                }
+                else {
+                    output.info('No drift detected.');
+                }
+            }
+            // Save new baseline
+            saveBaseline(newBaseline, baselinePath);
+            lastBaselineHash = newBaseline.integrityHash;
+            output.info(`Baseline updated: ${newBaseline.integrityHash.slice(0, 8)}`);
+        }
+        catch (error) {
+            output.error('Test failed: ' + (error instanceof Error ? error.message : String(error)));
+        }
+        finally {
+            await mcpClient.disconnect();
+        }
+    }
+    function checkForChanges() {
+        // Simple file watcher - check if any source files changed
+        let changed = false;
+        function walkDir(dir) {
+            try {
+                const entries = readdirSync(dir, { withFileTypes: true });
+                for (const entry of entries) {
+                    const fullPath = join(dir, entry.name);
+                    // Skip common directories
+                    if (entry.isDirectory()) {
+                        if (entry.name === 'node_modules' || entry.name === '.git' || entry.name === 'dist') {
+                            continue;
+                        }
+                        walkDir(fullPath);
+                    }
+                    else if (entry.isFile()) {
+                        if (extensions.some(ext => entry.name.endsWith(ext))) {
+                            const stat = statSync(fullPath);
+                            const modTime = stat.mtimeMs;
+                            const lastMod = fileModTimes.get(fullPath);
+                            if (lastMod === undefined) {
+                                fileModTimes.set(fullPath, modTime);
+                            }
+                            else if (modTime > lastMod) {
+                                fileModTimes.set(fullPath, modTime);
+                                output.info(`\nFile changed: ${fullPath}`);
+                                changed = true;
+                            }
+                        }
+                    }
+                }
+            }
+            catch (error) {
+                if (verbose) {
+                    output.error(`Warning: Error scanning ${dir}: ` + (error instanceof Error ? error.message : String(error)));
+                }
+            }
+        }
+        walkDir(watchPath);
+        return changed;
+    }
+    // Initial test run
+    await runTest();
+    output.info('\nWatching for changes... (Press Ctrl+C to exit)\n');
+    // Track current interval for proper cleanup
+    let currentInterval = null;
+    let isRunningInterview = false;
+    /**
+     * Poll for file changes and run test when changes detected.
+     */
+    async function pollForChanges() {
+        if (isRunningInterview) {
+            return;
+        }
+        try {
+            if (checkForChanges()) {
+                isRunningInterview = true;
+                await runTest();
+                output.info('\nWatching for changes... (Press Ctrl+C to exit)\n');
+            }
+        }
+        catch (error) {
+            output.error('Watch polling error: ' + (error instanceof Error ? error.message : String(error)));
+        }
+        finally {
+            isRunningInterview = false;
+        }
+    }
+    // Start polling interval
+    currentInterval = setInterval(() => {
+        pollForChanges().catch((error) => {
+            output.error('Unexpected polling error: ' + (error instanceof Error ? error.message : String(error)));
+        });
+    }, interval);
+    // Handle exit
+    const cleanup = () => {
+        output.info('\n\nExiting watch mode.');
+        if (currentInterval) {
+            clearInterval(currentInterval);
+            currentInterval = null;
+        }
+        // Remove signal handlers to prevent accumulation
+        process.removeListener('SIGINT', cleanup);
+        process.removeListener('SIGTERM', cleanup);
+        process.exit(EXIT_CODES.CLEAN);
+    };
+    process.on('SIGINT', cleanup);
+    process.on('SIGTERM', cleanup);
+});
+//# sourceMappingURL=watch.js.map

package/dist/cli/index.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/cli/index.js

@@ -0,0 +1,178 @@
+#!/usr/bin/env node
+import { config } from 'dotenv';
+import { existsSync } from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+// Load global ~/.bellwether/.env first (if exists)
+const globalEnvPath = join(homedir(), '.bellwether', '.env');
+if (existsSync(globalEnvPath)) {
+    config({ path: globalEnvPath });
+}
+// Then load project .env (overrides global settings)
+config();
+// Load credentials from keychain if not already in env
+// This is done async but we await it before parsing commands
+async function loadKeychainCredentials() {
+    // Only load from keychain if env vars aren't already set
+    if (!process.env.OPENAI_API_KEY || !process.env.ANTHROPIC_API_KEY) {
+        try {
+            const { getKeychainService } = await import('../auth/keychain.js');
+            const keychain = getKeychainService();
+            // Load OpenAI key if not in env
+            if (!process.env.OPENAI_API_KEY) {
+                const key = await keychain.getApiKey('openai');
+                if (key) {
+                    process.env.OPENAI_API_KEY = key;
+                }
+            }
+            // Load Anthropic key if not in env
+            if (!process.env.ANTHROPIC_API_KEY) {
+                const key = await keychain.getApiKey('anthropic');
+                if (key) {
+                    process.env.ANTHROPIC_API_KEY = key;
+                }
+            }
+        }
+        catch {
+            // Keychain not available - continue without it
+        }
+    }
+}
+import { Command } from 'commander';
+import { checkCommand } from './commands/check.js';
+import { exploreCommand } from './commands/explore.js';
+import { discoverCommand } from './commands/discover.js';
+import { watchCommand } from './commands/watch.js';
+import { initCommand } from './commands/init.js';
+import { authCommand } from './commands/auth.js';
+import { baselineCommand } from './commands/baseline.js';
+import { goldenCommand } from './commands/golden.js';
+import { registryCommand } from './commands/registry.js';
+import { verifyCommand } from './commands/verify.js';
+import { contractCommand } from './commands/contract.js';
+import { validateConfigCommand } from './commands/validate-config.js';
+import { badgeCommand } from './commands/cloud/badge.js';
+import { diffCommand } from './commands/cloud/diff.js';
+import { historyCommand } from './commands/cloud/history.js';
+import { linkCommand } from './commands/cloud/link.js';
+import { loginCommand } from './commands/cloud/login.js';
+import { projectsCommand } from './commands/cloud/projects.js';
+import { teamsCommand } from './commands/cloud/teams.js';
+import { uploadCommand } from './commands/cloud/upload.js';
+import { configureLogger } from '../logging/logger.js';
+import { VERSION } from '../version.js';
+import { findConfigFile } from '../config/validator.js';
+import { ConfigNotFoundError } from '../config/loader.js';
+import { EXIT_CODES, PATHS } from '../constants.js';
+const program = new Command();
+// ASCII art banner for help
+const banner = `
+Bellwether - MCP Server Validation & Documentation
+`;
+// Extended help with examples
+const examples = `
+Examples:
+
+  Initialize configuration:
+    $ bellwether init                       # Create bellwether.yaml
+    $ bellwether init --preset ci           # Optimized for CI/CD
+    $ bellwether init --preset local        # Local LLM with Ollama
+
+  Check for drift (free, fast, deterministic):
+    $ bellwether check npx @mcp/my-server   # Validate schemas
+    $ bellwether baseline save              # Save baseline
+    $ bellwether baseline compare ./bellwether-baseline.json  # Detect drift
+
+  Explore behavior (LLM-powered):
+    $ bellwether explore npx @mcp/my-server # Generate AGENTS.md documentation
+
+  Discover server capabilities:
+    $ bellwether discover npx @mcp/server-postgres
+
+  Search MCP Registry:
+    $ bellwether registry filesystem
+
+  Cloud workflow:
+    $ bellwether login                      # Authenticate with Bellwether Cloud
+    $ bellwether teams                      # List your teams
+    $ bellwether link my-project            # Link to cloud project
+    $ bellwether upload                     # Upload baseline
+    $ bellwether history                    # View version history
+
+Documentation: https://docs.bellwether.sh
+`;
+program
+    .name('bellwether')
+    .description(`${banner}
+Check MCP servers for drift. Explore behavior. Generate documentation.
+
+Commands:
+  check    - Schema validation and drift detection (free, fast, deterministic)
+  explore  - LLM-powered behavioral exploration and documentation
+
+For more information on a specific command, use:
+  bellwether <command> --help`)
+    .version(VERSION)
+    .option('--log-level <level>', 'Log level: debug, info, warn, error, silent')
+    .option('--log-file <path>', 'Write logs to file instead of stderr')
+    .hook('preAction', (thisCommand, actionCommand) => {
+    const activeCommand = actionCommand ?? thisCommand;
+    const commandName = activeCommand.name();
+    const opts = activeCommand.opts();
+    if (commandName !== 'init' && commandName !== 'validate-config') {
+        const configPath = opts.config;
+        const found = findConfigFile(configPath);
+        if (!found) {
+            const searchedPaths = configPath
+                ? [configPath]
+                : PATHS.CONFIG_FILENAMES.map((name) => join(process.cwd(), name));
+            console.error(new ConfigNotFoundError(searchedPaths).message);
+            process.exit(EXIT_CODES.ERROR);
+        }
+    }
+    if (opts.logLevel || opts.logFile) {
+        process.env.BELLWETHER_LOG_OVERRIDE = '1';
+        configureLogger({
+            level: opts.logLevel,
+            file: opts.logFile,
+        });
+    }
+})
+    .addHelpText('after', examples);
+// Add command groups for better organization
+program.addHelpText('beforeAll', '\nCore Commands:');
+// Core commands - check and explore
+program.addCommand(checkCommand.description('Check MCP server schema and detect drift (free, fast, deterministic)'));
+program.addCommand(exploreCommand.description('Explore MCP server behavior with LLM-powered testing'));
+program.addCommand(watchCommand.description('Watch for MCP server changes and auto-check'));
+program.addCommand(discoverCommand.description('Discover MCP server capabilities (tools, prompts, resources)'));
+program.addCommand(initCommand.description('Create a new bellwether.yaml configuration file'));
+program.addCommand(authCommand.description('Manage LLM provider API keys (keychain storage)'));
+program.addCommand(baselineCommand.description('Manage baselines for drift detection (save, compare, show, diff)'));
+program.addCommand(goldenCommand.description('Manage golden outputs for tool validation (save, compare, list, delete)'));
+program.addCommand(registryCommand.description('Search the MCP Registry for servers'));
+program.addCommand(verifyCommand.description('Generate verification report for Verified by Bellwether program'));
+program.addCommand(contractCommand.description('Validate MCP servers against contract definitions (validate, generate, show)'));
+program.addCommand(validateConfigCommand.description('Validate bellwether.yaml configuration (no tests)'));
+// Cloud commands - sync with Bellwether Cloud
+program.addCommand(loginCommand.description('Authenticate with Bellwether Cloud'));
+program.addCommand(teamsCommand.description('Manage team selection for cloud operations'));
+program.addCommand(linkCommand.description('Link local project to Bellwether Cloud project'));
+program.addCommand(projectsCommand.description('List your Bellwether Cloud projects'));
+program.addCommand(uploadCommand.description('Upload baseline to Bellwether Cloud'));
+program.addCommand(historyCommand.description('View baseline version history'));
+program.addCommand(diffCommand.description('Compare two baseline versions'));
+program.addCommand(badgeCommand.description('Get embeddable verification badge for your project'));
+// Custom help formatting
+program.configureHelp({
+    sortSubcommands: false, // Keep our custom order
+    subcommandTerm: (cmd) => cmd.name() + ' ' + cmd.usage(),
+});
+// Load keychain credentials, then parse commands
+loadKeychainCredentials().then(() => {
+    program.parse();
+}).catch(() => {
+    // If keychain loading fails, still parse commands
+    program.parse();
+});
+//# sourceMappingURL=index.js.map

package/dist/cli/interactive.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Interactive mode utilities for CLI commands.
+ *
+ * Provides user prompts for server command, persona selection,
+ * output format, and test control.
+ */
+import type { BellwetherConfig } from '../config/loader.js';
+/**
+ * Interactive configuration gathered from prompts.
+ */
+export interface InteractiveConfig {
+    serverCommand: string;
+    serverArgs: string[];
+    selectedPersonas: string[];
+    outputFormat: 'markdown' | 'json' | 'both';
+    outputDir: string;
+    saveBaseline: boolean;
+    baselinePath?: string;
+    compareBaseline?: string;
+    maxQuestions?: number;
+}
+/**
+ * Prompt user for interactive configuration.
+ */
+export declare function promptForConfig(existingConfig: BellwetherConfig, providedCommand?: string, providedArgs?: string[]): Promise<InteractiveConfig>;
+/**
+ * Pause controller for mid-test pause/resume.
+ */
+export interface PauseController {
+    isPaused: boolean;
+    pause(): void;
+    resume(): void;
+    waitIfPaused(): Promise<void>;
+}
+/**
+ * Create a pause controller for test control.
+ */
+export declare function createPauseController(): PauseController;
+/**
+ * Setup keyboard listener for pause/resume during test.
+ */
+export declare function setupInteractiveKeyboard(controller: PauseController): () => void;
+/**
+ * Display a summary of the interactive configuration.
+ */
+export declare function displayConfigSummary(config: InteractiveConfig): void;
+//# sourceMappingURL=interactive.d.ts.map