@dotsetlabs/bellwether 0.10.0

package/dist/cli/commands/explore.js

@@ -0,0 +1,437 @@
+/**
+ * Explore command - LLM-powered behavioral exploration for MCP servers.
+ *
+ * Purpose: Deep exploration and documentation of MCP server behavior.
+ * Output: Documentation + JSON report (filenames configurable via output.files)
+ * Baseline: None (use 'bellwether check' for drift detection)
+ * LLM: Required (OpenAI, Anthropic, or Ollama)
+ */
+import { Command } from 'commander';
+import { writeFileSync, existsSync, mkdirSync } from 'fs';
+import { join } from 'path';
+import { MCPClient } from '../../transport/mcp-client.js';
+import { discover } from '../../discovery/discovery.js';
+import { Interviewer } from '../../interview/interviewer.js';
+import { generateAgentsMd, generateJsonReport } from '../../docs/generator.js';
+import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
+import { validateConfigForExplore } from '../../config/validator.js';
+import { CostTracker, estimateInterviewCost, estimateInterviewTime, formatCostAndTimeEstimate, suggestOptimizations, formatOptimizationSuggestions, } from '../../cost/index.js';
+import { getMetricsCollector, resetMetricsCollector } from '../../metrics/collector.js';
+import { EXIT_CODES, WORKFLOW, PATHS } from '../../constants.js';
+import { FallbackLLMClient } from '../../llm/fallback.js';
+import { getGlobalCache, resetGlobalCache } from '../../cache/response-cache.js';
+import { InterviewProgressBar, formatExploreBanner } from '../utils/progress.js';
+import { parsePersonas } from '../../persona/builtins.js';
+import { loadScenariosFromFile, tryLoadDefaultScenarios, DEFAULT_SCENARIOS_FILE } from '../../scenarios/index.js';
+import { loadWorkflowsFromFile, tryLoadDefaultWorkflows, DEFAULT_WORKFLOWS_FILE } from '../../workflow/loader.js';
+import * as output from '../output.js';
+import { StreamingDisplay } from '../output.js';
+import { suppressLogs, restoreLogLevel, configureLogger } from '../../logging/logger.js';
+import { extractServerContextFromArgs } from '../utils/server-context.js';
+import { isCI } from '../utils/env.js';
+/**
+ * Wrapper to parse personas with warning output.
+ */
+function parsePersonasWithWarning(personaList) {
+    return parsePersonas(personaList, (unknownName, validNames) => {
+        output.warn(`Unknown persona: ${unknownName}. Available: ${validNames.join(', ')}`);
+    });
+}
+export const exploreCommand = new Command('explore')
+    .description('Explore MCP server behavior with LLM-powered testing')
+    .argument('[server-command]', 'Server command (overrides config)')
+    .argument('[args...]', 'Server arguments')
+    .option('-c, --config <path>', 'Path to config file', PATHS.DEFAULT_CONFIG_FILENAME)
+    .action(async (serverCommandArg, serverArgs, options) => {
+    // Load configuration
+    let config;
+    try {
+        config = loadConfig(options.config);
+    }
+    catch (error) {
+        if (error instanceof ConfigNotFoundError) {
+            output.error(error.message);
+            process.exit(EXIT_CODES.ERROR);
+        }
+        throw error;
+    }
+    // Determine server command (CLI arg overrides config)
+    const serverCommand = serverCommandArg || config.server.command;
+    const args = serverArgs.length > 0 ? serverArgs : config.server.args;
+    // Validate config for explore
+    try {
+        validateConfigForExplore(config, serverCommand);
+    }
+    catch (error) {
+        output.error(error instanceof Error ? error.message : String(error));
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Extract settings from config (CLI options override config)
+    const timeout = config.server.timeout;
+    const outputDir = config.output.dir;
+    const docsDir = config.output.docsDir;
+    const cacheEnabled = config.cache.enabled;
+    const verbose = config.logging.verbose;
+    const logLevel = config.logging.level;
+    if (!process.env.BELLWETHER_LOG_OVERRIDE) {
+        const effectiveLogLevel = verbose ? logLevel : 'silent';
+        configureLogger({ level: effectiveLogLevel });
+    }
+    // Parse personas from config (using explore section)
+    const selectedPersonas = parsePersonasWithWarning(config.explore.personas);
+    const maxQuestions = config.explore.maxQuestionsPerTool;
+    const parallelPersonas = config.explore.parallelPersonas;
+    const personaConcurrency = config.explore.personaConcurrency;
+    // Get LLM settings from config
+    const provider = config.llm.provider;
+    const model = config.llm.model || undefined;
+    // Display startup banner
+    const banner = formatExploreBanner({
+        serverCommand: `${serverCommand} ${args.join(' ')}`,
+        provider,
+        model: model || 'default',
+        personas: selectedPersonas.map((p) => p.name),
+        questionsPerTool: maxQuestions,
+    });
+    output.info(banner);
+    output.newline();
+    output.info(`Explore: LLM-powered behavioral exploration (using ${provider})`);
+    output.newline();
+    // Initialize cost tracker
+    const costTracker = new CostTracker(model || 'default');
+    // Initialize metrics collector
+    resetMetricsCollector();
+    const metricsCollector = getMetricsCollector();
+    metricsCollector.startInterview();
+    // Initialize cache
+    resetGlobalCache();
+    const cache = getGlobalCache({ enabled: cacheEnabled });
+    if (cacheEnabled && verbose) {
+        output.info('Response caching enabled');
+    }
+    // Initialize MCP client
+    const mcpClient = new MCPClient({
+        timeout,
+        debug: logLevel === 'debug',
+        transport: 'stdio',
+    });
+    // Initialize LLM client
+    let llmClient;
+    const onUsageCallback = (inputTokens, outputTokens) => {
+        costTracker.addUsage(inputTokens, outputTokens);
+        metricsCollector.recordTokenUsage(provider, model || 'default', inputTokens, outputTokens, 'llm_call');
+    };
+    try {
+        llmClient = new FallbackLLMClient({
+            providers: [{ provider, model, baseUrl: provider === 'ollama' ? config.llm.ollama.baseUrl : undefined }],
+            useOllamaFallback: true,
+            onUsage: onUsageCallback,
+        });
+    }
+    catch (error) {
+        output.error('Failed to initialize LLM client: ' + (error instanceof Error ? error.message : String(error)));
+        output.error(`\nProvider: ${provider}`);
+        output.error('Make sure the appropriate API key environment variable is set:');
+        output.error('  - OpenAI: OPENAI_API_KEY');
+        output.error('  - Anthropic: ANTHROPIC_API_KEY');
+        output.error('  - Ollama: No API key needed (ensure Ollama is running)');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    try {
+        // Connect to MCP server
+        output.info('Connecting to MCP server...');
+        await mcpClient.connect(serverCommand, args, config.server.env);
+        // Discovery phase
+        output.info('Discovering capabilities...');
+        const discovery = await discover(mcpClient, serverCommand, args);
+        const resourceCount = discovery.resources?.length ?? 0;
+        const discoveryParts = [`${discovery.tools.length} tools`, `${discovery.prompts.length} prompts`];
+        if (resourceCount > 0) {
+            discoveryParts.push(`${resourceCount} resources`);
+        }
+        output.info(`Found ${discoveryParts.join(', ')}\n`);
+        // Update metrics
+        metricsCollector.updateInterviewCounters({
+            toolsDiscovered: discovery.tools.length,
+            personasUsed: selectedPersonas.length,
+        });
+        if (discovery.tools.length === 0) {
+            output.info('No tools found. Nothing to explore.');
+            metricsCollector.endInterview();
+            await mcpClient.disconnect();
+            return;
+        }
+        // Show cost/time estimate (unless in CI)
+        if (!isCI()) {
+            const costEstimate = estimateInterviewCost(model || 'default', discovery.tools.length, maxQuestions, selectedPersonas.length);
+            const timeEstimate = estimateInterviewTime(discovery.tools.length, maxQuestions, selectedPersonas.length, parallelPersonas, provider, discovery.prompts.length, resourceCount, false);
+            output.info(formatCostAndTimeEstimate(costEstimate, timeEstimate));
+            // Show optimization suggestions
+            const hasScenariosFile = !!(config.scenarios.path || existsSync(join(outputDir, DEFAULT_SCENARIOS_FILE)));
+            const suggestions = suggestOptimizations({
+                estimatedCost: costEstimate.costUSD,
+                toolCount: discovery.tools.length,
+                personaCount: selectedPersonas.length,
+                isParallelPersonas: parallelPersonas,
+                isPremiumModel: false,
+                isUsingCiPreset: false,
+                hasScenariosFile,
+            });
+            if (suggestions.length > 0) {
+                output.newline();
+                output.info(formatOptimizationSuggestions(suggestions));
+            }
+            output.newline();
+        }
+        // Load custom scenarios (work in explore mode too)
+        let customScenarios;
+        if (config.scenarios.path) {
+            try {
+                customScenarios = loadScenariosFromFile(config.scenarios.path);
+                output.info(`Loaded ${customScenarios.toolScenarios.length} tool scenarios from ${config.scenarios.path}`);
+            }
+            catch (error) {
+                output.error(`Failed to load scenarios: ${error instanceof Error ? error.message : error}`);
+                process.exit(EXIT_CODES.ERROR);
+            }
+        }
+        else {
+            const defaultScenarios = tryLoadDefaultScenarios(outputDir);
+            if (defaultScenarios) {
+                customScenarios = defaultScenarios;
+                output.info(`Auto-loaded ${customScenarios.toolScenarios.length} scenarios from ${DEFAULT_SCENARIOS_FILE}`);
+            }
+        }
+        // Build workflow configuration
+        let workflowConfig;
+        if (config.workflows.path || config.workflows.discover) {
+            workflowConfig = {
+                discoverWorkflows: config.workflows.discover,
+                maxDiscoveredWorkflows: WORKFLOW.MAX_DISCOVERED_WORKFLOWS,
+                enableStateTracking: config.workflows.trackState,
+                stepTimeout: config.workflows.stepTimeout,
+                timeouts: config.workflows.timeouts,
+            };
+            if (config.workflows.path) {
+                try {
+                    const workflows = loadWorkflowsFromFile(config.workflows.path);
+                    workflowConfig.workflows = workflows;
+                    workflowConfig.workflowsFile = config.workflows.path;
+                    output.info(`Loaded ${workflows.length} workflow(s) from ${config.workflows.path}`);
+                }
+                catch (error) {
+                    output.error(`Failed to load workflows: ${error instanceof Error ? error.message : error}`);
+                    process.exit(EXIT_CODES.ERROR);
+                }
+            }
+        }
+        else {
+            const defaultWorkflows = tryLoadDefaultWorkflows(outputDir);
+            if (defaultWorkflows && defaultWorkflows.length > 0) {
+                workflowConfig = {
+                    discoverWorkflows: false,
+                    maxDiscoveredWorkflows: WORKFLOW.MAX_DISCOVERED_WORKFLOWS,
+                    enableStateTracking: config.workflows.trackState,
+                    stepTimeout: config.workflows.stepTimeout,
+                    timeouts: config.workflows.timeouts,
+                    workflows: defaultWorkflows,
+                    workflowsFile: `${outputDir}/${DEFAULT_WORKFLOWS_FILE}`,
+                };
+                output.info(`Auto-loaded ${defaultWorkflows.length} workflow(s) from ${DEFAULT_WORKFLOWS_FILE}`);
+            }
+        }
+        // Set up streaming display
+        let streamingDisplay = null;
+        let streamingCallbacks;
+        if (!isCI() && logLevel !== 'silent') {
+            suppressLogs();
+            streamingDisplay = new StreamingDisplay({ style: 'dim', maxWidth: 100 });
+            streamingCallbacks = {
+                onStart: (operation) => {
+                    const parts = operation.split(':');
+                    const opType = parts[0];
+                    const context = parts[1];
+                    let prefix = '';
+                    switch (opType) {
+                        case 'generate-questions':
+                            prefix = context ? `\n  Generating questions for ${context}... ` : '\n  Generating questions... ';
+                            break;
+                        case 'analyze':
+                            prefix = context ? `\n  Analyzing ${context}... ` : '\n  Analyzing... ';
+                            break;
+                        case 'synthesize-tool':
+                            prefix = context ? `\n  Synthesizing profile for ${context}... ` : '\n  Synthesizing profile... ';
+                            break;
+                        case 'synthesize-overall':
+                            prefix = '\n  Synthesizing overall findings... ';
+                            break;
+                        default:
+                            prefix = '\n  Processing... ';
+                    }
+                    streamingDisplay?.start(prefix);
+                },
+                onChunk: (chunk) => {
+                    streamingDisplay?.write(chunk);
+                },
+                onComplete: () => {
+                    streamingDisplay?.finish(' [done]');
+                },
+                onError: (error) => {
+                    streamingDisplay?.abort(`[error: ${error.message}]`);
+                },
+            };
+        }
+        // Create interviewer for explore mode
+        const fullServerCommand = `${serverCommand} ${args.join(' ')}`.trim();
+        const interviewer = new Interviewer(llmClient, {
+            maxQuestionsPerTool: maxQuestions,
+            timeout,
+            skipErrorTests: config.explore.skipErrorTests,
+            model: model || 'default',
+            personas: selectedPersonas,
+            customScenarios,
+            customScenariosOnly: config.scenarios.only,
+            enableStreaming: !!streamingCallbacks,
+            streamingCallbacks,
+            parallelPersonas,
+            personaConcurrency,
+            cache,
+            workflowConfig,
+            checkMode: false, // Full exploration mode with LLM
+            serverCommand: fullServerCommand,
+        });
+        // Extract server context
+        const serverContext = extractServerContextFromArgs(serverCommand, args);
+        if (serverContext.allowedDirectories && serverContext.allowedDirectories.length > 0) {
+            output.info(`Detected allowed directories: ${serverContext.allowedDirectories.join(', ')}`);
+        }
+        interviewer.setServerContext(serverContext);
+        // Set up progress display
+        const progressBar = new InterviewProgressBar({ enabled: !verbose && !streamingCallbacks });
+        const progressCallback = (progress) => {
+            if (verbose) {
+                switch (progress.phase) {
+                    case 'starting':
+                        output.info('Starting exploration...');
+                        progressBar.start(progress.totalTools, progress.totalPersonas, progress.totalPrompts ?? 0, progress.totalResources ?? 0);
+                        break;
+                    case 'interviewing':
+                        output.info(`[${progress.currentPersona}] Exploring: ${progress.currentTool} (${progress.toolsCompleted + 1}/${progress.totalTools})`);
+                        break;
+                    case 'synthesizing':
+                        output.info('Synthesizing findings...');
+                        break;
+                    case 'complete':
+                        output.info('Exploration complete!');
+                        break;
+                }
+            }
+            else {
+                if (progress.phase === 'starting') {
+                    progressBar.start(progress.totalTools, progress.totalPersonas, progress.totalPrompts ?? 0, progress.totalResources ?? 0);
+                }
+                else if (['interviewing', 'prompts', 'resources', 'workflows'].includes(progress.phase)) {
+                    progressBar.update(progress);
+                }
+                else if (progress.phase === 'complete' || progress.phase === 'synthesizing') {
+                    progressBar.stop();
+                }
+            }
+        };
+        output.info('Starting exploration...\n');
+        const result = await interviewer.interview(mcpClient, discovery, progressCallback);
+        progressBar.stop();
+        if (!verbose) {
+            output.newline();
+        }
+        // Generate documentation
+        output.info('Generating documentation...');
+        mkdirSync(outputDir, { recursive: true });
+        if (docsDir !== outputDir) {
+            mkdirSync(docsDir, { recursive: true });
+        }
+        const agentsMd = generateAgentsMd(result);
+        const agentsMdPath = join(docsDir, config.output.files.agentsDoc);
+        writeFileSync(agentsMdPath, agentsMd);
+        output.info(`Written: ${agentsMdPath}`);
+        // Generate JSON report
+        const jsonReport = generateJsonReport(result);
+        const jsonPath = join(outputDir, config.output.files.exploreReport);
+        writeFileSync(jsonPath, jsonReport);
+        output.info(`Written: ${jsonPath}`);
+        // End metrics
+        metricsCollector.endInterview();
+        output.info('\nExploration complete!');
+        output.info(`Duration: ${(result.metadata.durationMs / 1000).toFixed(1)}s`);
+        output.info(`Tools explored: ${result.toolProfiles.length}`);
+        // Display cost summary
+        const costEstimate = costTracker.getCost();
+        if (costEstimate.costUSD > 0) {
+            output.info(`Estimated cost: $${costEstimate.costUSD.toFixed(4)}`);
+        }
+        // Display scenario results
+        if (result.scenarioResults && result.scenarioResults.length > 0) {
+            const passed = result.scenarioResults.filter((r) => r.passed).length;
+            const failed = result.scenarioResults.length - passed;
+            const statusIcon = failed === 0 ? '\u2713' : '\u2717';
+            output.info(`\nCustom scenarios: ${passed}/${result.scenarioResults.length} passed ${statusIcon}`);
+            if (failed > 0) {
+                output.info('\nFailed scenarios:');
+                for (const scenarioResult of result.scenarioResults.filter((r) => !r.passed)) {
+                    const scenario = scenarioResult.scenario;
+                    const toolOrPrompt = 'tool' in scenario ? scenario.tool : scenario.prompt;
+                    output.info(`  - ${toolOrPrompt}: ${scenario.description}`);
+                    if (scenarioResult.error) {
+                        output.info(`    Error: ${scenarioResult.error}`);
+                    }
+                }
+            }
+        }
+        // Display workflow results
+        if (result.workflowResults && result.workflowResults.length > 0) {
+            const successful = result.workflowResults.filter((wr) => wr.success).length;
+            const failed = result.workflowResults.length - successful;
+            const statusIcon = failed === 0 ? '\u2713' : '\u2717';
+            output.info(`\nWorkflows: ${successful}/${result.workflowResults.length} passed ${statusIcon}`);
+            if (failed > 0) {
+                output.info('\nFailed workflows:');
+                for (const wr of result.workflowResults.filter((w) => !w.success)) {
+                    output.info(`  - ${wr.workflow.name}: ${wr.failureReason ?? 'Unknown error'}`);
+                }
+            }
+        }
+        // Note about baselines
+        output.info('\nTip: For drift detection, use "bellwether check" to create and compare baselines.');
+    }
+    catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        output.error('\n--- Exploration Failed ---');
+        output.error(`Error: ${errorMessage}`);
+        if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('Connection refused')) {
+            output.error('\nPossible causes:');
+            output.error('  - The MCP server is not running');
+            output.error('  - The server address/port is incorrect');
+        }
+        else if (errorMessage.includes('timeout') || errorMessage.includes('Timeout')) {
+            output.error('\nPossible causes:');
+            output.error('  - The MCP server is taking too long to respond');
+            output.error('  - Increase server.timeout in bellwether.yaml');
+        }
+        else if (errorMessage.includes('ENOENT') || errorMessage.includes('not found')) {
+            output.error('\nPossible causes:');
+            output.error('  - The server command was not found');
+            output.error('  - Check that the command is installed and in PATH');
+        }
+        else if (errorMessage.includes('API key') || errorMessage.includes('authentication')) {
+            output.error('\nPossible causes:');
+            output.error('  - Missing or invalid API key');
+            output.error('  - Run "bellwether auth" to configure API keys');
+        }
+        process.exit(EXIT_CODES.ERROR);
+    }
+    finally {
+        restoreLogLevel();
+        await mcpClient.disconnect();
+    }
+});
+//# sourceMappingURL=explore.js.map

package/dist/cli/commands/feedback.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Feedback command - submit feedback on drift detection decisions.
+ *
+ * Allows users to report false positives, false negatives, and
+ * confidence calibration issues for algorithm improvement.
+ */
+import { Command } from 'commander';
+export declare function createFeedbackCommand(): Command;
+//# sourceMappingURL=feedback.d.ts.map

package/dist/cli/commands/feedback.js

@@ -0,0 +1,174 @@
+/**
+ * Feedback command - submit feedback on drift detection decisions.
+ *
+ * Allows users to report false positives, false negatives, and
+ * confidence calibration issues for algorithm improvement.
+ */
+import { Command } from 'commander';
+import chalk from 'chalk';
+import { getDecisionLogger, getFeedbackManager, } from '../../baseline/telemetry.js';
+export function createFeedbackCommand() {
+    const feedbackCmd = new Command('feedback')
+        .description('Submit feedback on drift detection decisions')
+        .argument('[decision-id]', 'ID of the comparison decision to report')
+        .option('-t, --type <type>', 'Feedback type: false_positive, false_negative, confidence_wrong')
+        .option('-m, --message <message>', 'Comment explaining the issue')
+        .option('--correct <answer>', 'What the correct answer should have been (true/false)')
+        .option('--list', 'List recent decisions that can receive feedback')
+        .option('--stats', 'Show feedback statistics')
+        .option('--analyze', 'Analyze all feedback for patterns')
+        .option('--export <path>', 'Export decisions to file for analysis')
+        .option('--clear', 'Clear all logged decisions and feedback')
+        .action(async (decisionId, options) => {
+        await handleFeedback(decisionId, options);
+    });
+    return feedbackCmd;
+}
+async function handleFeedback(decisionId, options) {
+    const logger = getDecisionLogger();
+    const feedbackManager = getFeedbackManager();
+    // List recent decisions
+    if (options.list) {
+        const decisions = logger.loadAllDecisions();
+        if (decisions.length === 0) {
+            console.log(chalk.gray('No logged decisions found.'));
+            console.log(chalk.gray('Decisions are logged when running baseline comparisons.'));
+            return;
+        }
+        console.log('');
+        console.log(chalk.bold('Recent Comparison Decisions'));
+        console.log('─'.repeat(60));
+        // Show last 20 decisions
+        const recent = decisions.slice(-20).reverse();
+        for (const decision of recent) {
+            const time = new Date(decision.timestamp).toLocaleString();
+            const match = decision.matchDecision ? chalk.green('match') : chalk.red('no-match');
+            console.log('');
+            console.log(`  ${chalk.cyan(decision.id.slice(0, 8))}`);
+            console.log(`    Time: ${chalk.gray(time)}`);
+            console.log(`    Type: ${decision.type} | Tool: ${decision.toolName}`);
+            console.log(`    Decision: ${match} (confidence: ${decision.rawConfidence}%)`);
+            console.log(`    Text 1: "${truncate(decision.text1, 50)}"`);
+            console.log(`    Text 2: "${truncate(decision.text2, 50)}"`);
+        }
+        console.log('');
+        console.log(chalk.gray(`Total decisions: ${decisions.length}`));
+        console.log(chalk.gray('Use: bellwether feedback <id> --type false_positive'));
+        return;
+    }
+    // Show statistics
+    if (options.stats) {
+        const stats = logger.getStatistics();
+        const feedbackAnalysis = feedbackManager.analyze();
+        console.log('');
+        console.log(chalk.bold('Telemetry Statistics'));
+        console.log('─'.repeat(40));
+        console.log(`  Total decisions logged: ${stats.totalDecisions}`);
+        console.log(`  Average confidence: ${stats.averageConfidence}%`);
+        console.log(`  Match rate: ${stats.matchRate}%`);
+        console.log('');
+        console.log(chalk.gray('  By type:'));
+        for (const [type, count] of Object.entries(stats.byType)) {
+            console.log(`    ${type}: ${count}`);
+        }
+        console.log('');
+        console.log(chalk.bold('Feedback Statistics'));
+        console.log('─'.repeat(40));
+        console.log(`  Total feedback reports: ${feedbackAnalysis.totalReports}`);
+        console.log(`  False positive rate: ${feedbackAnalysis.falsePositiveRate}%`);
+        console.log(`  False negative rate: ${feedbackAnalysis.falseNegativeRate}%`);
+        console.log(`  Confidence issues: ${feedbackAnalysis.confidenceIssueRate}%`);
+        console.log('');
+        return;
+    }
+    // Analyze feedback patterns
+    if (options.analyze) {
+        const analysis = feedbackManager.analyze();
+        console.log('');
+        console.log(chalk.bold('Feedback Pattern Analysis'));
+        console.log('─'.repeat(40));
+        if (analysis.totalReports === 0) {
+            console.log(chalk.gray('  No feedback submitted yet.'));
+            console.log(chalk.gray('  Use: bellwether feedback <id> --type <type>'));
+            return;
+        }
+        console.log(`  Total reports: ${analysis.totalReports}`);
+        console.log('');
+        console.log('  Issue breakdown:');
+        console.log(`    False positives: ${analysis.falsePositiveRate}%`);
+        console.log(`    False negatives: ${analysis.falseNegativeRate}%`);
+        console.log(`    Confidence issues: ${analysis.confidenceIssueRate}%`);
+        if (analysis.commonPatterns.length > 0) {
+            console.log('');
+            console.log('  Common patterns in feedback:');
+            for (const pattern of analysis.commonPatterns) {
+                console.log(`    "${pattern.pattern}": ${pattern.count} occurrences (${pattern.feedbackType})`);
+            }
+        }
+        console.log('');
+        return;
+    }
+    // Export decisions
+    if (options.export) {
+        logger.exportToFile(options.export);
+        console.log(chalk.green(`Decisions exported to: ${options.export}`));
+        return;
+    }
+    // Clear all data
+    if (options.clear) {
+        logger.clear();
+        feedbackManager.clear();
+        console.log(chalk.green('Cleared all decisions and feedback.'));
+        return;
+    }
+    // Submit feedback
+    if (!decisionId) {
+        console.log(chalk.red('Error: Decision ID required'));
+        console.log(chalk.gray('Use: bellwether feedback --list to see recent decisions'));
+        process.exit(1);
+    }
+    if (!options.type) {
+        console.log(chalk.red('Error: Feedback type required'));
+        console.log(chalk.gray('Options: false_positive, false_negative, confidence_wrong'));
+        process.exit(1);
+    }
+    const validTypes = ['false_positive', 'false_negative', 'confidence_wrong'];
+    if (!validTypes.includes(options.type)) {
+        console.log(chalk.red(`Error: Invalid feedback type "${options.type}"`));
+        console.log(chalk.gray(`Valid types: ${validTypes.join(', ')}`));
+        process.exit(1);
+    }
+    // Find the decision
+    const decision = logger.getDecision(decisionId) ??
+        logger.loadAllDecisions().find(d => d.id.startsWith(decisionId));
+    if (!decision) {
+        console.log(chalk.red(`Error: Decision not found: ${decisionId}`));
+        console.log(chalk.gray('Use: bellwether feedback --list to see recent decisions'));
+        process.exit(1);
+    }
+    // Submit feedback
+    const feedback = {
+        decisionId: decision.id,
+        feedbackType: options.type,
+        userComment: options.message,
+    };
+    if (options.correct !== undefined) {
+        feedback.correctAnswer = options.correct === 'true';
+    }
+    feedbackManager.submit(feedback);
+    console.log('');
+    console.log(chalk.green('✓ Feedback submitted'));
+    console.log(`  Decision: ${decision.id.slice(0, 8)}`);
+    console.log(`  Type: ${options.type}`);
+    if (options.message) {
+        console.log(`  Comment: ${options.message}`);
+    }
+    console.log('');
+    console.log(chalk.gray('Thank you for helping improve drift detection accuracy!'));
+}
+function truncate(str, maxLen) {
+    if (str.length <= maxLen)
+        return str;
+    return str.slice(0, maxLen - 3) + '...';
+}
+//# sourceMappingURL=feedback.js.map

package/dist/cli/commands/golden.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Golden command - Manage golden outputs for tool validation.
+ *
+ * Commands:
+ *   bellwether golden save --tool <name>      Save current output as golden
+ *   bellwether golden compare                  Compare against all golden outputs
+ *   bellwether golden list                     List all saved golden outputs
+ *   bellwether golden delete --tool <name>    Delete a golden output
+ */
+import { Command } from 'commander';
+export declare const goldenCommand: Command;
+//# sourceMappingURL=golden.d.ts.map