@dotsetlabs/bellwether 0.10.0

package/dist/cli/commands/auth.js

@@ -0,0 +1,352 @@
+/**
+ * Auth command for managing LLM provider API keys.
+ *
+ * Provides secure storage via system keychain with fallback to file-based storage.
+ * Separate from the `login` command which handles Bellwether Cloud authentication.
+ */
+import { Command } from 'commander';
+import * as readline from 'readline';
+import { DEFAULT_MODELS } from '../../llm/client.js';
+import { getKeychainService } from '../../auth/keychain.js';
+import { getAuthStatus } from '../../auth/credentials.js';
+import { EXIT_CODES } from '../../constants.js';
+import * as output from '../output.js';
+/**
+ * Provider display names and info.
+ */
+const PROVIDER_INFO = {
+    openai: {
+        name: 'OpenAI',
+        url: 'https://platform.openai.com/api-keys',
+        envVar: 'OPENAI_API_KEY',
+    },
+    anthropic: {
+        name: 'Anthropic',
+        url: 'https://console.anthropic.com/settings/keys',
+        envVar: 'ANTHROPIC_API_KEY',
+    },
+};
+/**
+ * Create a readline interface for prompts.
+ */
+function createPrompt() {
+    return readline.createInterface({
+        input: process.stdin,
+        output: process.stdout,
+    });
+}
+/**
+ * Prompt user for input (hidden for passwords).
+ */
+async function prompt(rl, question, hidden = false) {
+    return new Promise((resolve) => {
+        if (hidden && process.stdin.isTTY) {
+            // Hide input for sensitive data
+            const stdin = process.stdin;
+            const stdout = process.stdout;
+            stdout.write(question);
+            stdin.setRawMode(true);
+            stdin.resume();
+            stdin.setEncoding('utf8');
+            let input = '';
+            const onData = (char) => {
+                const charCode = char.charCodeAt(0);
+                if (charCode === 13 || charCode === 10) {
+                    // Enter
+                    stdin.setRawMode(false);
+                    stdin.removeListener('data', onData);
+                    stdout.write('\n');
+                    resolve(input);
+                }
+                else if (charCode === 127 || charCode === 8) {
+                    // Backspace
+                    if (input.length > 0) {
+                        input = input.slice(0, -1);
+                        stdout.clearLine(0);
+                        stdout.cursorTo(0);
+                        stdout.write(question + '*'.repeat(input.length));
+                    }
+                }
+                else if (charCode === 3) {
+                    // Ctrl+C
+                    process.exit(EXIT_CODES.CLEAN);
+                }
+                else if (charCode >= 32) {
+                    // Printable character
+                    input += char;
+                    stdout.write('*');
+                }
+            };
+            stdin.on('data', onData);
+        }
+        else {
+            // Fallback for non-TTY
+            rl.question(question, resolve);
+        }
+    });
+}
+/**
+ * Prompt user to select from a list.
+ */
+async function promptSelect(rl, question, options) {
+    output.info(question);
+    options.forEach((opt, i) => {
+        output.info(`  ${i + 1}. ${opt.label}`);
+    });
+    const answer = await prompt(rl, `Enter choice (1-${options.length}): `);
+    const index = parseInt(answer, 10) - 1;
+    if (index >= 0 && index < options.length) {
+        return options[index].value;
+    }
+    output.info('Invalid choice. Please try again.\n');
+    return promptSelect(rl, question, options);
+}
+/**
+ * Validate API key format.
+ */
+function validateApiKey(provider, key) {
+    if (!key || key.trim().length === 0) {
+        return { valid: false, error: 'API key cannot be empty' };
+    }
+    if (provider === 'openai') {
+        // OpenAI keys start with sk-
+        if (!key.startsWith('sk-')) {
+            return { valid: false, error: 'OpenAI API keys should start with "sk-"' };
+        }
+        if (key.length < 20) {
+            return { valid: false, error: 'API key appears too short' };
+        }
+    }
+    if (provider === 'anthropic') {
+        // Anthropic keys start with sk-ant-
+        if (!key.startsWith('sk-ant-')) {
+            return { valid: false, error: 'Anthropic API keys should start with "sk-ant-"' };
+        }
+        if (key.length < 20) {
+            return { valid: false, error: 'API key appears too short' };
+        }
+    }
+    return { valid: true };
+}
+/**
+ * Interactive auth setup.
+ */
+async function interactiveSetup() {
+    const rl = createPrompt();
+    output.info('Bellwether Authentication Setup');
+    output.info('================================\n');
+    // Check current status
+    const status = await getAuthStatus();
+    const configuredProviders = status.filter(s => s.provider !== 'ollama' && s.configured);
+    if (configuredProviders.length > 0) {
+        output.info('Currently configured:');
+        for (const s of configuredProviders) {
+            const source = s.source === 'keychain' ? 'keychain' :
+                s.source === 'env' ? `env (${s.envVar})` : s.source;
+            output.info(`  - ${PROVIDER_INFO[s.provider]?.name ?? s.provider}: ${source}`);
+        }
+        output.newline();
+    }
+    // Select provider
+    const provider = await promptSelect(rl, 'Which LLM provider would you like to configure?', [
+        { value: 'anthropic', label: 'Anthropic Claude (recommended)' },
+        { value: 'openai', label: 'OpenAI' },
+    ]);
+    const info = PROVIDER_INFO[provider];
+    output.info(`\nGet your ${info.name} API key from:`);
+    output.info(`  ${info.url}\n`);
+    // Get API key
+    const apiKey = await prompt(rl, `Enter your ${info.name} API key: `, true);
+    // Validate
+    const validation = validateApiKey(provider, apiKey);
+    if (!validation.valid) {
+        output.error(`\nError: ${validation.error}`);
+        rl.close();
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Check keychain availability
+    const keychain = getKeychainService();
+    const hasSecureKeychain = await keychain.isSecureKeychainAvailable();
+    let storageChoice;
+    if (hasSecureKeychain) {
+        storageChoice = await promptSelect(rl, '\nWhere would you like to store the API key?', [
+            { value: 'keychain', label: 'System keychain (recommended - most secure)' },
+            { value: 'env', label: `Environment file (~/.bellwether/.env)` },
+        ]);
+    }
+    else {
+        output.info('\nNote: System keychain not available. Using file-based storage.');
+        storageChoice = 'env';
+    }
+    // Store the key
+    try {
+        if (storageChoice === 'keychain') {
+            await keychain.setApiKey(provider, apiKey);
+            output.success(`\n\u2713 API key stored in system keychain`);
+        }
+        else {
+            // Store in ~/.bellwether/.env
+            const fs = await import('fs');
+            const path = await import('path');
+            const os = await import('os');
+            const envDir = path.join(os.homedir(), '.bellwether');
+            const envPath = path.join(envDir, '.env');
+            if (!fs.existsSync(envDir)) {
+                fs.mkdirSync(envDir, { recursive: true, mode: 0o700 });
+            }
+            // Read existing .env or create new
+            let envContent = '';
+            if (fs.existsSync(envPath)) {
+                envContent = fs.readFileSync(envPath, 'utf-8');
+            }
+            // Update or add the key
+            const envVar = info.envVar;
+            const lines = envContent.split('\n').filter(line => !line.startsWith(`${envVar}=`));
+            lines.push(`${envVar}=${apiKey}`);
+            fs.writeFileSync(envPath, lines.filter(l => l).join('\n') + '\n', { mode: 0o600 });
+            output.success(`\n\u2713 API key stored in ~/.bellwether/.env`);
+        }
+        output.info(`\nYou're all set! Bellwether will now use ${info.name} for tests.`);
+        output.info(`\nDefault model: ${DEFAULT_MODELS[provider]}`);
+        output.info('\nTry it out:');
+        output.info('  bellwether explore npx @modelcontextprotocol/server-memory');
+    }
+    catch (error) {
+        output.error(`\nFailed to store API key: ${error instanceof Error ? error.message : error}`);
+        process.exit(EXIT_CODES.ERROR);
+    }
+    rl.close();
+}
+/**
+ * Show auth status.
+ */
+async function showStatus() {
+    output.info('Bellwether Authentication Status');
+    output.info('=================================\n');
+    const status = await getAuthStatus();
+    for (const s of status) {
+        if (s.provider === 'ollama') {
+            output.info('Ollama:');
+            output.info('  Status: No API key required (local)');
+            output.info('  Model:  qwen3:8b');
+            output.newline();
+            continue;
+        }
+        const info = PROVIDER_INFO[s.provider];
+        if (!info)
+            continue;
+        output.info(`${info.name}:`);
+        if (s.configured) {
+            let sourceDesc;
+            switch (s.source) {
+                case 'keychain':
+                    sourceDesc = 'System keychain';
+                    break;
+                case 'env':
+                    sourceDesc = `Environment variable (${s.envVar})`;
+                    break;
+                case 'project-env':
+                    sourceDesc = `Project .env file (${s.envVar})`;
+                    break;
+                case 'global-env':
+                    sourceDesc = `Global .env file (~/.bellwether/.env)`;
+                    break;
+                default:
+                    sourceDesc = s.source;
+            }
+            output.info(`  Status: \u2713 Configured`);
+            output.info(`  Source: ${sourceDesc}`);
+            output.info(`  Model:  ${DEFAULT_MODELS[s.provider]}`);
+        }
+        else {
+            output.info(`  Status: \u2717 Not configured`);
+            output.info(`  Setup:  Run \`bellwether auth\` or set ${info.envVar}`);
+        }
+        output.newline();
+    }
+    // Show priority order
+    output.info('Credential resolution order:');
+    output.info('  1. Environment variables (highest priority)');
+    output.info('  2. System keychain');
+    output.info('  3. ~/.bellwether/.env file');
+    output.info('  4. Project .env file');
+}
+/**
+ * Add a provider.
+ */
+async function addProvider(providerArg) {
+    const rl = createPrompt();
+    let provider;
+    if (providerArg && (providerArg === 'openai' || providerArg === 'anthropic')) {
+        provider = providerArg;
+    }
+    else {
+        provider = await promptSelect(rl, 'Which provider?', [
+            { value: 'anthropic', label: 'Anthropic (recommended)' },
+            { value: 'openai', label: 'OpenAI' },
+        ]);
+    }
+    const info = PROVIDER_INFO[provider];
+    output.info(`\nGet your API key from: ${info.url}\n`);
+    const apiKey = await prompt(rl, `Enter ${info.name} API key: `, true);
+    const validation = validateApiKey(provider, apiKey);
+    if (!validation.valid) {
+        output.error(`\nError: ${validation.error}`);
+        rl.close();
+        process.exit(EXIT_CODES.ERROR);
+    }
+    const keychain = getKeychainService();
+    await keychain.setApiKey(provider, apiKey);
+    output.success(`\n\u2713 ${info.name} API key stored in keychain`);
+    rl.close();
+}
+/**
+ * Remove a provider.
+ */
+async function removeProvider(providerArg) {
+    const rl = createPrompt();
+    let provider;
+    if (providerArg && (providerArg === 'openai' || providerArg === 'anthropic')) {
+        provider = providerArg;
+    }
+    else {
+        provider = await promptSelect(rl, 'Which provider to remove?', [
+            { value: 'anthropic', label: 'Anthropic' },
+            { value: 'openai', label: 'OpenAI' },
+        ]);
+    }
+    const keychain = getKeychainService();
+    const deleted = await keychain.deleteApiKey(provider);
+    if (deleted) {
+        output.success(`\n\u2713 ${PROVIDER_INFO[provider].name} API key removed from keychain`);
+    }
+    else {
+        output.info(`\nNo ${PROVIDER_INFO[provider].name} API key found in keychain`);
+    }
+    rl.close();
+}
+/**
+ * The auth command.
+ */
+export const authCommand = new Command('auth')
+    .description('Manage LLM provider API keys')
+    .addCommand(new Command('status')
+    .description('Show authentication status for all providers')
+    .action(showStatus))
+    .addCommand(new Command('add')
+    .description('Add or update an API key')
+    .argument('[provider]', 'Provider name (openai, anthropic)')
+    .action(addProvider))
+    .addCommand(new Command('remove')
+    .description('Remove an API key from keychain')
+    .argument('[provider]', 'Provider name (openai, anthropic)')
+    .action(removeProvider))
+    .addCommand(new Command('clear')
+    .description('Remove all stored API keys')
+    .action(async () => {
+    const keychain = getKeychainService();
+    await keychain.clearAll();
+    output.success('All API keys removed from keychain.');
+}))
+    .action(interactiveSetup); // Default action is interactive setup
+//# sourceMappingURL=auth.js.map

package/dist/cli/commands/badge.d.ts

@@ -0,0 +1,3 @@
+import { Command } from 'commander';
+export declare const badgeCommand: Command;
+//# sourceMappingURL=badge.d.ts.map

package/dist/cli/commands/badge.js

@@ -0,0 +1,74 @@
+import { Command } from 'commander';
+import { createCloudClient } from '../../cloud/client.js';
+import { getLinkedProject } from '../../cloud/auth.js';
+import { EXIT_CODES } from '../../constants.js';
+import * as output from '../output.js';
+export const badgeCommand = new Command('badge')
+    .description('Get embeddable badge for your project')
+    .option('-p, --project <id>', 'Project ID (uses linked project if not specified)')
+    .option('--json', 'Output as JSON')
+    .option('--markdown', 'Output markdown only')
+    .option('--url', 'Output badge URL only')
+    .action(async (options) => {
+    // Get project ID
+    let projectId = options.project;
+    if (!projectId) {
+        const link = getLinkedProject();
+        if (!link) {
+            output.error('No project specified and no linked project found.');
+            output.error('Run `bellwether link <project>` first or use --project <id>');
+            process.exit(EXIT_CODES.ERROR);
+        }
+        projectId = link.projectId;
+    }
+    // Create cloud client
+    const client = createCloudClient();
+    try {
+        const badge = await client.getBadgeInfo(projectId);
+        if (!badge) {
+            output.error(`Project not found: ${projectId}`);
+            process.exit(EXIT_CODES.ERROR);
+        }
+        if (options.json) {
+            output.info(JSON.stringify(badge, null, 2));
+            return;
+        }
+        if (options.markdown) {
+            output.info(badge.markdown);
+            return;
+        }
+        if (options.url) {
+            output.info(badge.badgeUrl);
+            return;
+        }
+        // Default: formatted output
+        output.info('');
+        output.info(`Project: ${badge.projectName}`);
+        output.info(`Status:  ${badge.statusText}`);
+        if (badge.lastVerified) {
+            output.info(`Verified: ${new Date(badge.lastVerified).toLocaleString()}`);
+        }
+        if (badge.latestVersion) {
+            output.info(`Version: v${badge.latestVersion}`);
+        }
+        output.info('');
+        output.info('Badge URL:');
+        output.info(`  ${badge.badgeUrl}`);
+        output.info('');
+        output.info('Add to your README.md:');
+        output.info('');
+        output.info(`  ${badge.markdown}`);
+        output.info('');
+        output.info('Or with HTML:');
+        output.info('');
+        // Extract base URL from badge URL for consistency
+        const baseUrl = badge.badgeUrl.replace(/\/badge\/.*$/, '');
+        output.info(`  <a href="${baseUrl}/projects/${projectId}"><img src="${badge.badgeUrl}" alt="Bellwether"></a>`);
+        output.info('');
+    }
+    catch (error) {
+        output.error('Failed to get badge info: ' + (error instanceof Error ? error.message : String(error)));
+        process.exit(EXIT_CODES.ERROR);
+    }
+});
+//# sourceMappingURL=badge.js.map

package/dist/cli/commands/baseline-accept.d.ts

@@ -0,0 +1,15 @@
+/**
+ * baseline accept command - accept detected drift as intentional changes.
+ *
+ * This command allows users to acknowledge that detected drift was intentional
+ * (e.g., when adding new features, updating tool behavior, etc.) and update
+ * the baseline to reflect the new expected state.
+ *
+ * Usage:
+ *   bellwether baseline accept              # Accept drift and update baseline
+ *   bellwether baseline accept --reason "Added new delete tool"
+ *   bellwether baseline accept --dry-run    # Show what would be accepted
+ */
+import { Command } from 'commander';
+export declare const acceptCommand: Command;
+//# sourceMappingURL=baseline-accept.d.ts.map

package/dist/cli/commands/baseline-accept.js

@@ -0,0 +1,178 @@
+/**
+ * baseline accept command - accept detected drift as intentional changes.
+ *
+ * This command allows users to acknowledge that detected drift was intentional
+ * (e.g., when adding new features, updating tool behavior, etc.) and update
+ * the baseline to reflect the new expected state.
+ *
+ * Usage:
+ *   bellwether baseline accept              # Accept drift and update baseline
+ *   bellwether baseline accept --reason "Added new delete tool"
+ *   bellwether baseline accept --dry-run    # Show what would be accepted
+ */
+import { Command } from 'commander';
+import { existsSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { createBaseline, saveBaseline, loadBaseline, compareBaselines, acceptDrift, formatDiffText, } from '../../baseline/index.js';
+import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
+import { EXIT_CODES } from '../../constants.js';
+import * as output from '../output.js';
+function loadConfigOrExit(configPath) {
+    try {
+        return loadConfig(configPath);
+    }
+    catch (error) {
+        if (error instanceof ConfigNotFoundError) {
+            output.error(error.message);
+            process.exit(EXIT_CODES.ERROR);
+        }
+        throw error;
+    }
+}
+/**
+ * Load interview result from JSON report.
+ */
+function loadInterviewResult(reportPath) {
+    if (!existsSync(reportPath)) {
+        throw new Error(`Test report not found: ${reportPath}\n\n` +
+            'Run `bellwether check` first to generate a report.\n' +
+            'Configure in bellwether.yaml:\n' +
+            '  output:\n' +
+            '    format: json  # or "both" for JSON + markdown');
+    }
+    const content = readFileSync(reportPath, 'utf-8');
+    let result;
+    try {
+        result = JSON.parse(content);
+    }
+    catch (error) {
+        throw new Error(`Invalid JSON in report file ${reportPath}: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+    // Validate that this is a check mode result
+    if (result.metadata.model && result.metadata.model !== 'check') {
+        throw new Error(`Baseline operations only work with check mode results.\n\n` +
+            `The report at ${reportPath} was created with explore mode.\n` +
+            'Run `bellwether check` to generate a check mode report first.');
+    }
+    return result;
+}
+export const acceptCommand = new Command('accept')
+    .description('Accept detected drift as intentional and update the baseline')
+    .option('-c, --config <path>', 'Path to config file')
+    .option('--report <path>', 'Path to test report JSON file')
+    .option('--baseline <path>', 'Path to baseline file')
+    .option('--reason <reason>', 'Reason for accepting the drift')
+    .option('--accepted-by <name>', 'Who is accepting the drift (for audit trail)')
+    .option('--dry-run', 'Show what would be accepted without making changes')
+    .option('-f, --force', 'Skip confirmation prompt')
+    .action(async (options) => {
+    const config = loadConfigOrExit(options.config);
+    const outputDir = config.output.dir;
+    const resolvedBaselinePath = options.baseline ?? config.baseline.comparePath ?? config.baseline.path;
+    if (!resolvedBaselinePath) {
+        output.error('No baseline path provided. Set baseline.path or baseline.comparePath in config, or pass --baseline.');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Determine paths
+    const baselinePath = resolvedBaselinePath.startsWith('/')
+        ? resolvedBaselinePath
+        : join(outputDir, resolvedBaselinePath);
+    const reportPath = options.report || join(outputDir, config.output.files.checkReport);
+    // Load the existing baseline
+    if (!existsSync(baselinePath)) {
+        output.error(`Baseline not found: ${baselinePath}`);
+        output.error('\nNo baseline exists to compare against.');
+        output.error('Run `bellwether check` followed by `bellwether baseline save` first.');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    let previousBaseline;
+    try {
+        previousBaseline = loadBaseline(baselinePath);
+    }
+    catch (error) {
+        output.error(`Failed to load baseline: ${error instanceof Error ? error.message : error}`);
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Load the current test results
+    let result;
+    try {
+        result = loadInterviewResult(reportPath);
+    }
+    catch (error) {
+        output.error(error instanceof Error ? error.message : String(error));
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Create current baseline from test results
+    const serverCommand = result.metadata.serverCommand || 'unknown';
+    const currentBaseline = createBaseline(result, serverCommand);
+    // Compare baselines
+    const diff = compareBaselines(previousBaseline, currentBaseline);
+    // Check if there's any drift to accept
+    if (diff.severity === 'none') {
+        output.success('No drift detected. Baseline is already up to date.');
+        return;
+    }
+    // Show the drift that will be accepted
+    output.info('=== Drift to Accept ===');
+    output.newline();
+    output.info(formatDiffText(diff));
+    output.newline();
+    // Show summary
+    output.info('--- Summary ---');
+    output.info(`Severity: ${diff.severity}`);
+    if (diff.toolsAdded.length > 0) {
+        output.info(`Tools added: ${diff.toolsAdded.join(', ')}`);
+    }
+    if (diff.toolsRemoved.length > 0) {
+        output.warn(`Tools removed: ${diff.toolsRemoved.join(', ')}`);
+    }
+    if (diff.toolsModified.length > 0) {
+        output.info(`Tools modified: ${diff.toolsModified.map((t) => t.tool).join(', ')}`);
+    }
+    output.info(`Breaking changes: ${diff.breakingCount}`);
+    output.info(`Warnings: ${diff.warningCount}`);
+    output.info(`Info: ${diff.infoCount}`);
+    output.newline();
+    // Dry run mode - just show what would happen
+    if (options.dryRun) {
+        output.info('--- Dry Run Mode ---');
+        output.info('Would update baseline with acceptance metadata:');
+        output.info(`  Accepted by: ${options.acceptedBy || '(not specified)'}`);
+        output.info(`  Reason: ${options.reason || '(not specified)'}`);
+        output.info(`  Baseline path: ${baselinePath}`);
+        return;
+    }
+    // For breaking changes without --force, show a warning
+    if (diff.severity === 'breaking' && !options.force) {
+        output.warn('');
+        output.warn('⚠️  This will accept BREAKING changes!');
+        output.warn('');
+        output.warn('Breaking changes may affect downstream consumers of this MCP server.');
+        output.warn('Make sure you have updated any dependent systems accordingly.');
+        output.warn('');
+        output.warn('To proceed, run again with --force');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Accept the drift
+    const acceptedBaseline = acceptDrift(currentBaseline, diff, {
+        acceptedBy: options.acceptedBy,
+        reason: options.reason,
+    });
+    // Save the updated baseline
+    saveBaseline(acceptedBaseline, baselinePath);
+    output.success(`Drift accepted and baseline updated: ${baselinePath}`);
+    output.newline();
+    // Show acceptance details
+    output.info('Acceptance recorded:');
+    output.info(`  Accepted at: ${acceptedBaseline.acceptance?.acceptedAt}`);
+    if (options.acceptedBy) {
+        output.info(`  Accepted by: ${options.acceptedBy}`);
+    }
+    if (options.reason) {
+        output.info(`  Reason: ${options.reason}`);
+    }
+    output.newline();
+    output.info('The baseline now reflects the current server state.');
+    output.info('Future `bellwether check` runs will compare against this new baseline.');
+});
+//# sourceMappingURL=baseline-accept.js.map

package/dist/cli/commands/baseline-migrate.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Baseline migrate command - upgrade baselines to current format version.
+ *
+ * This command migrates baseline files from older format versions to the
+ * current format version, ensuring compatibility with the latest CLI features.
+ */
+import { Command } from 'commander';
+/**
+ * Create the migrate command.
+ */
+export declare const migrateCommand: Command;
+//# sourceMappingURL=baseline-migrate.d.ts.map