@dotsetlabs/bellwether 0.10.0

package/dist/cli/commands/upload.js

@@ -0,0 +1,223 @@
+/**
+ * Upload command for uploading baselines to Bellwether Cloud.
+ *
+ * Can read baseline path from bellwether.yaml config.
+ */
+import { Command } from 'commander';
+import { existsSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { getSessionToken, getLinkedProject } from '../../cloud/auth.js';
+import { createCloudClient } from '../../cloud/client.js';
+import { loadBaseline } from '../../baseline/saver.js';
+import { convertToCloudBaseline } from '../../baseline/converter.js';
+import { loadConfig } from '../../config/loader.js';
+import { PATHS, EXIT_CODES } from '../../constants.js';
+import * as output from '../output.js';
+import { getSeverityIcon } from '../output.js';
+/**
+ * Default baseline file name.
+ */
+const DEFAULT_BASELINE_FILE = PATHS.DEFAULT_BASELINE_FILE;
+/**
+ * Try to get baseline config from config file.
+ * Returns output directory and optional baseline path from config.
+ */
+function getBaselineConfigFromFile(configPath) {
+    try {
+        const config = loadConfig(configPath);
+        return {
+            outputDir: config.output.dir,
+            baselinePath: config.baseline.path,
+        };
+    }
+    catch {
+        return { outputDir: '.' };
+    }
+}
+export const uploadCommand = new Command('upload')
+    .description('Upload a baseline to Bellwether Cloud')
+    .argument('[baseline]', `Path to baseline JSON file (default: ${DEFAULT_BASELINE_FILE})`)
+    .option('-b, --baseline <path>', 'Path to baseline JSON file')
+    .option('-c, --config <path>', 'Path to config file')
+    .option('-p, --project <id>', 'Project ID to upload to (uses linked project if not specified)')
+    .option('--ci', 'CI mode - output URL only, exit 1 on breaking drift')
+    .option('--session <session>', 'Session token (overrides stored/env session)')
+    .option('--fail-on-drift', 'Exit with error if any behavioral drift detected')
+    .action(async (baselineArg, options) => {
+    // Get config settings
+    const configSettings = getBaselineConfigFromFile(options.config);
+    const outputDir = configSettings.outputDir;
+    // Determine baseline path with priority:
+    // 1. Positional argument
+    // 2. --baseline flag
+    // 3. baseline.path from config
+    // 4. Default file in output.dir
+    // 5. Default file in current directory
+    let baselinePath = baselineArg ?? options.baseline ?? configSettings.baselinePath;
+    if (!baselinePath) {
+        const configBaseline = join(outputDir, DEFAULT_BASELINE_FILE);
+        if (existsSync(configBaseline)) {
+            baselinePath = configBaseline;
+        }
+        else {
+            baselinePath = DEFAULT_BASELINE_FILE;
+        }
+    }
+    const isCiMode = options.ci;
+    // Get session
+    const sessionToken = options.session ?? getSessionToken();
+    if (!sessionToken) {
+        if (isCiMode) {
+            output.error('BELLWETHER_SESSION not set');
+            process.exit(EXIT_CODES.ERROR);
+        }
+        output.error('Not authenticated. Run `bellwether login` first or set BELLWETHER_SESSION.');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Check baseline file exists
+    if (!existsSync(baselinePath)) {
+        if (isCiMode) {
+            output.error(`Baseline not found: ${baselinePath}`);
+            process.exit(EXIT_CODES.ERROR);
+        }
+        output.error(`Baseline file not found: ${baselinePath}`);
+        output.error('\nCreate a baseline first:');
+        output.error('  1. Run `bellwether check` (with output.format: json in config)');
+        output.error('  2. Run `bellwether baseline save`');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Determine project ID
+    let projectId = options.project;
+    if (!projectId) {
+        const link = getLinkedProject();
+        if (link) {
+            projectId = link.projectId;
+            if (!isCiMode) {
+                output.info(`Using linked project: ${link.projectName}`);
+            }
+        }
+    }
+    if (!projectId) {
+        if (isCiMode) {
+            output.error('No project specified');
+            process.exit(EXIT_CODES.ERROR);
+        }
+        output.error('No project specified.');
+        output.error('\nEither:');
+        output.error('  - Use --project <id> to specify a project');
+        output.error('  - Run `bellwether link` to link this directory to a project');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Load and convert baseline
+    let cloudBaseline;
+    try {
+        // Try loading as cloud baseline first
+        const content = readFileSync(baselinePath, 'utf-8');
+        const parsed = JSON.parse(content);
+        if (parsed.version === '1.0' && parsed.metadata?.formatVersion === '1.0') {
+            // Already in cloud format
+            cloudBaseline = parsed;
+        }
+        else {
+            // Convert from local format
+            const localBaseline = loadBaseline(baselinePath);
+            cloudBaseline = convertToCloudBaseline(localBaseline);
+        }
+    }
+    catch (error) {
+        if (isCiMode) {
+            output.error(`Failed to load baseline: ${error instanceof Error ? error.message : error}`);
+            process.exit(EXIT_CODES.ERROR);
+        }
+        output.error('Failed to load baseline: ' + (error instanceof Error ? error.message : String(error)));
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Create client and upload
+    const client = createCloudClient({ sessionToken });
+    if (!client.isAuthenticated()) {
+        if (isCiMode) {
+            output.error('Authentication failed');
+            process.exit(EXIT_CODES.ERROR);
+        }
+        output.error('Authentication failed. Run `bellwether login` to re-authenticate.');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    if (!isCiMode) {
+        output.info(`Uploading baseline to project ${projectId}...`);
+    }
+    try {
+        const result = await client.uploadBaseline(projectId, cloudBaseline);
+        if (isCiMode) {
+            // CI mode - minimal output
+            output.info(result.viewUrl);
+            // Check for drift
+            if (result.version > 1) {
+                const diff = await client.getLatestDiff(projectId);
+                if (diff) {
+                    if (diff.severity === 'breaking') {
+                        output.error('Breaking changes detected');
+                        process.exit(EXIT_CODES.ERROR);
+                    }
+                    if (options.failOnDrift && diff.severity !== 'none') {
+                        output.error(`Behavioral drift detected: ${diff.severity}`);
+                        process.exit(EXIT_CODES.ERROR);
+                    }
+                }
+            }
+        }
+        else {
+            // Interactive mode - detailed output
+            output.info(`\nUpload successful!`);
+            output.info(`Version: ${result.version}`);
+            output.info(`View:    ${result.viewUrl}`);
+            if (result.diffUrl) {
+                output.info(`Diff:    ${result.diffUrl}`);
+                // Show diff summary
+                const diff = await client.getLatestDiff(projectId);
+                if (diff) {
+                    output.info('\nChanges from previous version:');
+                    printDiffSummary(diff);
+                    if (diff.severity === 'breaking') {
+                        output.info('\n⚠️  Breaking changes detected!');
+                    }
+                }
+            }
+            else {
+                output.info('\nThis is the first baseline for this project.');
+            }
+        }
+    }
+    catch (error) {
+        if (isCiMode) {
+            output.error(`Upload failed: ${error instanceof Error ? error.message : error}`);
+            process.exit(EXIT_CODES.ERROR);
+        }
+        output.error('Upload failed: ' + (error instanceof Error ? error.message : String(error)));
+        process.exit(EXIT_CODES.ERROR);
+    }
+});
+/**
+ * Print a diff summary in human-readable format (verbose format).
+ */
+function printDiffSummary(diff) {
+    output.info(`  Severity: ${getSeverityIcon(diff.severity)} ${diff.severity}`);
+    if (diff.toolsAdded > 0) {
+        output.info(`  Tools added: +${diff.toolsAdded}`);
+    }
+    if (diff.toolsRemoved > 0) {
+        output.info(`  Tools removed: -${diff.toolsRemoved}`);
+    }
+    if (diff.toolsModified > 0) {
+        output.info(`  Tools modified: ~${diff.toolsModified}`);
+    }
+    if (diff.behaviorChanges > 0) {
+        output.info(`  Behavior changes: ${diff.behaviorChanges}`);
+    }
+    if (diff.toolsAdded === 0 &&
+        diff.toolsRemoved === 0 &&
+        diff.toolsModified === 0 &&
+        diff.behaviorChanges === 0) {
+        output.info('  No changes detected');
+    }
+}
+//# sourceMappingURL=upload.js.map

package/dist/cli/commands/validate-config.d.ts

@@ -0,0 +1,6 @@
+/**
+ * validate-config command - Validate bellwether.yaml without running tests.
+ */
+import { Command } from 'commander';
+export declare const validateConfigCommand: Command;
+//# sourceMappingURL=validate-config.d.ts.map

package/dist/cli/commands/validate-config.js

@@ -0,0 +1,35 @@
+/**
+ * validate-config command - Validate bellwether.yaml without running tests.
+ */
+import { Command } from 'commander';
+import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
+import { getConfigWarnings } from '../../config/validator.js';
+import { EXIT_CODES, PATHS } from '../../constants.js';
+import * as output from '../output.js';
+export const validateConfigCommand = new Command('validate-config')
+    .description('Validate bellwether.yaml configuration without running tests')
+    .option('-c, --config <path>', 'Path to config file', PATHS.DEFAULT_CONFIG_FILENAME)
+    .action((options) => {
+    try {
+        const config = loadConfig(options.config);
+        output.success('Configuration is valid.');
+        const warnings = getConfigWarnings(config);
+        if (warnings.length > 0) {
+            output.warn('Configuration warnings:');
+            for (const warning of warnings) {
+                output.warn(`  - ${warning}`);
+            }
+        }
+        process.exit(EXIT_CODES.CLEAN);
+    }
+    catch (error) {
+        if (error instanceof ConfigNotFoundError) {
+            output.error(error.message);
+        }
+        else {
+            output.error(error instanceof Error ? error.message : String(error));
+        }
+        process.exit(EXIT_CODES.ERROR);
+    }
+});
+//# sourceMappingURL=validate-config.js.map

package/dist/cli/commands/verify.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Verify command - generate verification reports for the Verified by Bellwether program.
+ */
+import { Command } from 'commander';
+/**
+ * Create a new verify command instance.
+ * Useful for testing where fresh command instances are needed.
+ */
+export declare function createVerifyCommand(): Command;
+export declare const verifyCommand: Command;
+//# sourceMappingURL=verify.d.ts.map

package/dist/cli/commands/verify.js

@@ -0,0 +1,283 @@
+/**
+ * Verify command - generate verification reports for the Verified by Bellwether program.
+ */
+import { Command } from 'commander';
+import chalk from 'chalk';
+import { writeFile } from 'fs/promises';
+import { join } from 'path';
+import { generateVerificationReport, generateBadgeUrl, generateBadgeMarkdown, } from '../../verification/index.js';
+import { Interviewer, DEFAULT_CONFIG } from '../../interview/interviewer.js';
+import { MCPClient } from '../../transport/mcp-client.js';
+import { discover } from '../../discovery/discovery.js';
+import { createLLMClient, DEFAULT_MODELS } from '../../llm/index.js';
+import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
+import { CostTracker } from '../../cost/index.js';
+import { BUILTIN_PERSONAS } from '../../persona/builtins.js';
+import { EXIT_CODES, PATHS } from '../../constants.js';
+import * as output from '../output.js';
+import { InterviewProgressBar } from '../utils/progress.js';
+import { createCloudClient } from '../../cloud/client.js';
+import { getLinkedProject, getSessionToken } from '../../cloud/auth.js';
+// Convert BUILTIN_PERSONAS record to array
+const ALL_PERSONAS = Object.values(BUILTIN_PERSONAS);
+/**
+ * Create a new verify command instance.
+ * Useful for testing where fresh command instances are needed.
+ */
+export function createVerifyCommand() {
+    return new Command('verify')
+        .description('Generate a verification report for the Verified by Bellwether program')
+        .argument('[server-command]', 'Server command (overrides config)')
+        .argument('[args...]', 'Server arguments')
+        .option('-c, --config <path>', 'Path to config file', PATHS.DEFAULT_CONFIG_FILENAME)
+        .option('-o, --output <dir>', 'Output directory')
+        .option('--server-id <id>', 'Server identifier (namespace/name)')
+        .option('--version <version>', 'Server version to verify')
+        .option('--tier <tier>', 'Target verification tier (bronze, silver, gold, platinum)')
+        .option('--security', 'Include security testing (required for gold+ tiers)')
+        .option('--json', 'Output verification result as JSON')
+        .option('--badge-only', 'Only output badge URL')
+        .option('-p, --project <id>', 'Project ID to submit verification to (requires login)')
+        .action(async (serverCommandArg, serverArgs, options) => {
+        await handleVerify(serverCommandArg, serverArgs, options);
+    });
+}
+export const verifyCommand = createVerifyCommand();
+async function handleVerify(serverCommandArg, serverArgs, options) {
+    output.info(chalk.bold('\n🔒 Bellwether Verification\n'));
+    // Load configuration
+    let bellwetherConfig;
+    try {
+        bellwetherConfig = loadConfig(options.config);
+    }
+    catch (error) {
+        if (error instanceof ConfigNotFoundError) {
+            output.error(error.message);
+            process.exit(EXIT_CODES.ERROR);
+        }
+        throw error;
+    }
+    // Determine server command (CLI arg overrides config)
+    const serverCommand = serverCommandArg || bellwetherConfig.server.command;
+    const args = serverArgs.length > 0 ? serverArgs : bellwetherConfig.server.args;
+    if (!serverCommand) {
+        output.error('Error: No server command provided.');
+        output.error('Either specify a server command as an argument or configure it in bellwether.yaml');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Get LLM settings from config
+    const provider = bellwetherConfig.llm.provider;
+    const model = bellwetherConfig.llm.model || undefined;
+    const outputDir = options.output ?? bellwetherConfig.output.dir;
+    const serverTimeout = bellwetherConfig.server.timeout;
+    const serverEnv = bellwetherConfig.server.env;
+    const targetTier = (options.tier ?? bellwetherConfig.verify.tier);
+    const includeSecurity = options.security ? true : bellwetherConfig.verify.security;
+    const outputJson = options.json ? true : bellwetherConfig.verify.json;
+    const badgeOnly = options.badgeOnly ? true : bellwetherConfig.verify.badgeOnly;
+    // Initialize cost tracker
+    const effectiveModel = model || DEFAULT_MODELS[provider];
+    const costTracker = new CostTracker(effectiveModel);
+    // Create LLM client with usage tracking
+    let llm;
+    try {
+        llm = createLLMClient({
+            provider: provider,
+            model,
+            onUsage: (inputTokens, outputTokens) => {
+                costTracker.addUsage(inputTokens, outputTokens);
+            },
+        });
+    }
+    catch {
+        output.error(chalk.red('Error: Could not create LLM client. Check your API keys.'));
+        process.exit(EXIT_CODES.ERROR);
+    }
+    output.info(chalk.gray(`Using model: ${effectiveModel}`));
+    // Connect to server
+    output.info(chalk.gray(`Connecting to ${serverCommand} ${args.join(' ')}...`));
+    const client = new MCPClient({ timeout: serverTimeout });
+    try {
+        await client.connect(serverCommand, args, serverEnv);
+        const discovery = await discover(client, serverCommand, args);
+        output.info(chalk.green(`✓ Connected to ${discovery.serverInfo.name} v${discovery.serverInfo.version}`));
+        output.info(chalk.gray(`  ${discovery.tools.length} tools, ${discovery.prompts.length} prompts, ${(discovery.resources ?? []).length} resources`));
+        output.newline();
+        // Determine personas based on tier and security option
+        const personas = selectPersonasForTier(targetTier, includeSecurity);
+        output.info(chalk.gray(`Target tier: ${targetTier}`));
+        output.info(chalk.gray(`Using personas: ${personas.map((p) => p.name).join(', ')}`));
+        output.newline();
+        // Run interview
+        output.info(chalk.bold('Running verification test...\n'));
+        const interviewer = new Interviewer(llm, {
+            ...DEFAULT_CONFIG,
+            personas,
+            maxQuestionsPerTool: targetTier === 'platinum' ? 5 : targetTier === 'gold' ? 4 : 3,
+        });
+        // Set up progress bar
+        const progressBar = new InterviewProgressBar({ enabled: !output.isQuiet() });
+        const progressCallback = (progress) => {
+            if (progress.phase === 'starting') {
+                progressBar.start(progress.totalTools, progress.totalPersonas, progress.totalPrompts ?? 0, progress.totalResources ?? 0);
+            }
+            else if (['interviewing', 'prompts', 'resources'].includes(progress.phase)) {
+                progressBar.update(progress);
+            }
+            else if (progress.phase === 'complete' || progress.phase === 'synthesizing') {
+                progressBar.stop();
+            }
+        };
+        const interview = await interviewer.interview(client, discovery, progressCallback);
+        progressBar.stop();
+        output.newline();
+        output.info(chalk.green('✓ Test complete'));
+        // Display cost summary
+        const costEstimate = costTracker.getCost();
+        const usage = costEstimate.usage;
+        if (usage.totalTokens > 0) {
+            const costStr = costEstimate.costUSD > 0
+                ? `$${costEstimate.costUSD.toFixed(4)}`
+                : 'Free (local model)';
+            output.info(chalk.gray(`  Tokens: ${usage.totalTokens.toLocaleString()} (${usage.inputTokens.toLocaleString()} in, ${usage.outputTokens.toLocaleString()} out)`));
+            output.info(chalk.gray(`  Estimated cost: ${costStr}`));
+        }
+        output.newline();
+        // Generate verification
+        const serverId = options.serverId ?? `${discovery.serverInfo.name}`;
+        const verificationConfig = {
+            serverId,
+            version: options.version ?? discovery.serverInfo.version,
+            targetTier,
+            includeSecurity,
+            outputDir,
+        };
+        const report = generateVerificationReport(interview, verificationConfig);
+        const result = report.result;
+        // Output results
+        if (badgeOnly) {
+            output.info(generateBadgeUrl(result));
+            return;
+        }
+        if (outputJson) {
+            output.json(report);
+            return;
+        }
+        // Display verification result
+        displayVerificationResult(result);
+        // Save report
+        const reportPath = join(outputDir, bellwetherConfig.output.files.verificationReport);
+        await writeFile(reportPath, JSON.stringify(report, null, 2));
+        output.info(chalk.gray(`\nReport saved to: ${reportPath}`));
+        // Display badge
+        output.info('\n' + chalk.bold('Badge:'));
+        output.info(chalk.cyan(generateBadgeUrl(result)));
+        output.newline();
+        output.info(chalk.bold('Markdown:'));
+        output.info(chalk.gray(generateBadgeMarkdown(result)));
+        // Submit to cloud if project ID is specified
+        const projectId = options.project ?? getLinkedProject()?.projectId;
+        if (projectId) {
+            output.newline();
+            // Check if logged in
+            if (!getSessionToken()) {
+                output.warn(chalk.yellow('⚠ Not logged in. Run `bellwether login` to submit verification to the platform.'));
+            }
+            else {
+                output.info(chalk.gray('Submitting verification to platform...'));
+                try {
+                    const cloudClient = createCloudClient();
+                    // Convert local result to cloud format
+                    const cloudResult = {
+                        serverId: result.serverId,
+                        version: result.version,
+                        status: result.status,
+                        tier: result.tier,
+                        verifiedAt: result.verifiedAt,
+                        expiresAt: result.expiresAt,
+                        toolsVerified: result.toolsVerified,
+                        testsPassed: result.testsPassed,
+                        testsTotal: result.testsTotal,
+                        passRate: result.passRate,
+                        reportHash: result.reportHash,
+                        bellwetherVersion: result.bellwetherVersion,
+                    };
+                    const submission = await cloudClient.submitVerification(projectId, cloudResult, report);
+                    output.info(chalk.green(`✓ Verification submitted successfully`));
+                    output.info(chalk.gray(`  View at: ${submission.viewUrl}`));
+                }
+                catch (submitError) {
+                    output.error(chalk.red(`Failed to submit verification: ${submitError instanceof Error ? submitError.message : 'Unknown error'}`));
+                    // Don't exit with error - local verification succeeded
+                }
+            }
+        }
+        // Exit with appropriate code
+        if (result.status !== 'verified') {
+            process.exit(EXIT_CODES.ERROR);
+        }
+    }
+    catch (error) {
+        output.error(chalk.red(`\nError: ${error instanceof Error ? error.message : 'Unknown error'}`));
+        process.exit(EXIT_CODES.ERROR);
+    }
+    finally {
+        await client.disconnect();
+    }
+}
+function selectPersonasForTier(tier, includeSecurity) {
+    switch (tier) {
+        case 'platinum':
+            // All personas including security
+            return ALL_PERSONAS;
+        case 'gold':
+            // Technical Writer, QA Engineer, Security Tester
+            if (includeSecurity) {
+                return ALL_PERSONAS.filter((p) => ['technical_writer', 'qa_engineer', 'security_tester'].includes(p.id));
+            }
+            return ALL_PERSONAS.filter((p) => ['technical_writer', 'qa_engineer', 'novice_user'].includes(p.id));
+        case 'silver':
+            // Technical Writer and QA Engineer
+            return ALL_PERSONAS.filter((p) => ['technical_writer', 'qa_engineer'].includes(p.id));
+        case 'bronze':
+        default:
+            // Just Technical Writer
+            return ALL_PERSONAS.filter((p) => p.id === 'technical_writer');
+    }
+}
+function displayVerificationResult(result) {
+    const statusColor = result.status === 'verified' ? chalk.green : chalk.red;
+    const tierColor = getTierChalk(result.tier);
+    output.info('─'.repeat(60));
+    output.newline();
+    output.info(chalk.bold('Verification Result'));
+    output.newline();
+    output.info(`  Server:     ${result.serverId} v${result.version}`);
+    output.info(`  Status:     ${statusColor(result.status.toUpperCase())}`);
+    if (result.tier) {
+        output.info(`  Tier:       ${tierColor(result.tier.toUpperCase())}`);
+    }
+    output.newline();
+    output.info(`  Pass Rate:  ${result.passRate}% (${result.testsPassed}/${result.testsTotal} tests)`);
+    output.info(`  Tools:      ${result.toolsVerified} verified`);
+    output.newline();
+    output.info(`  Verified:   ${new Date(result.verifiedAt).toLocaleDateString()}`);
+    output.info(`  Expires:    ${new Date(result.expiresAt).toLocaleDateString()}`);
+    output.newline();
+    output.info('─'.repeat(60));
+}
+function getTierChalk(tier) {
+    switch (tier) {
+        case 'platinum':
+            return (text) => chalk.cyan(text);
+        case 'gold':
+            return (text) => chalk.yellow(text);
+        case 'silver':
+            return (text) => chalk.gray(text);
+        case 'bronze':
+            return (text) => chalk.hex('#CD7F32')(text);
+        default:
+            return (text) => text;
+    }
+}
+//# sourceMappingURL=verify.js.map

package/dist/cli/commands/watch.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Watch command - watch for file changes and auto-check.
+ *
+ * Uses bellwether.yaml for configuration.
+ * Only watch-specific options are available as flags.
+ *
+ * Note: Watch mode only runs schema validation (check mode).
+ * For LLM-powered exploration, use 'bellwether explore' directly.
+ */
+import { Command } from 'commander';
+export declare const watchCommand: Command;
+//# sourceMappingURL=watch.d.ts.map