@dotsetlabs/bellwether 0.10.0

package/dist/cli/interactive.js

@@ -0,0 +1,216 @@
+/**
+ * Interactive mode utilities for CLI commands.
+ *
+ * Provides user prompts for server command, persona selection,
+ * output format, and test control.
+ */
+import * as readline from 'readline';
+import * as output from './output.js';
+/**
+ * Create a readline interface for prompting.
+ */
+function createPrompt() {
+    return readline.createInterface({
+        input: process.stdin,
+        output: process.stdout,
+    });
+}
+/**
+ * Ask a question and wait for user input.
+ */
+function ask(rl, question) {
+    return new Promise((resolve) => {
+        rl.question(question, (answer) => {
+            resolve(answer.trim());
+        });
+    });
+}
+/**
+ * Present numbered options and get user selection.
+ */
+async function selectOption(rl, prompt, options) {
+    output.info(prompt);
+    output.numberedList(options);
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const input = await ask(rl, `Enter choice (1-${options.length}): `);
+        const num = parseInt(input, 10);
+        if (num >= 1 && num <= options.length) {
+            return num - 1;
+        }
+        output.info(`Invalid choice. Please enter a number between 1 and ${options.length}.`);
+    }
+}
+/**
+ * Present yes/no question.
+ */
+async function confirm(rl, prompt, defaultValue = false) {
+    const suffix = defaultValue ? '[Y/n]' : '[y/N]';
+    const input = await ask(rl, `${prompt} ${suffix}: `);
+    if (!input) {
+        return defaultValue;
+    }
+    return input.toLowerCase() === 'y' || input.toLowerCase() === 'yes';
+}
+/**
+ * Available personas for selection.
+ * These match the built-in personas in src/persona/builtins.ts
+ */
+const AVAILABLE_PERSONAS = [
+    { name: 'technical_writer', description: 'Clear, accurate documentation focused' },
+    { name: 'security_tester', description: 'Security-focused boundary and injection testing' },
+    { name: 'qa_engineer', description: 'Thorough test coverage and edge cases' },
+    { name: 'novice_user', description: 'Beginner-friendly, accessibility focused' },
+];
+/**
+ * Output format options.
+ */
+const OUTPUT_FORMATS = [
+    { value: 'markdown', description: 'AGENTS.md documentation file' },
+    { value: 'json', description: 'JSON report file' },
+    { value: 'both', description: 'Both markdown and JSON' },
+];
+/**
+ * Prompt user for interactive configuration.
+ */
+export async function promptForConfig(existingConfig, providedCommand, providedArgs) {
+    const rl = createPrompt();
+    try {
+        output.info('\n=== Bellwether Interactive Mode ===\n');
+        // Server command
+        let serverCommand = providedCommand || '';
+        let serverArgs = providedArgs || [];
+        if (!serverCommand) {
+            output.info('Enter the command to start your MCP server.');
+            output.info('Examples:');
+            output.info('  npx @modelcontextprotocol/server-filesystem /path/to/dir');
+            output.info('  python mcp_server.py');
+            output.info('  node my-server.js\n');
+            const fullCommand = await ask(rl, 'Server command: ');
+            if (!fullCommand) {
+                throw new Error('Server command is required');
+            }
+            const parts = fullCommand.split(/\s+/);
+            serverCommand = parts[0];
+            serverArgs = parts.slice(1);
+        }
+        // Persona selection
+        output.info('\nSelect personas to use during testing:');
+        const selectedPersonas = [];
+        for (const persona of AVAILABLE_PERSONAS) {
+            const selected = await confirm(rl, `  Use ${persona.name}? (${persona.description})`, existingConfig.interview.personas?.includes(persona.name) ?? persona.name === 'technical_writer');
+            if (selected) {
+                selectedPersonas.push(persona.name);
+            }
+        }
+        if (selectedPersonas.length === 0) {
+            output.info('No personas selected. Using default "technical_writer" persona.');
+            selectedPersonas.push('technical_writer');
+        }
+        // Output format
+        output.info('\nSelect output format:');
+        const formatIndex = await selectOption(rl, '', OUTPUT_FORMATS.map((f) => `${f.value} - ${f.description}`));
+        const outputFormat = OUTPUT_FORMATS[formatIndex].value;
+        // Output directory
+        const outputDir = await ask(rl, '\nOutput directory [.]: ') || '.';
+        // Max questions per tool
+        const maxQuestionsInput = await ask(rl, `\nMax questions per tool [${existingConfig.interview.maxQuestionsPerTool}]: `);
+        const maxQuestions = maxQuestionsInput
+            ? parseInt(maxQuestionsInput, 10)
+            : existingConfig.interview.maxQuestionsPerTool;
+        // Baseline options
+        const saveBaseline = await confirm(rl, '\nSave baseline for future drift detection?', true);
+        let baselinePath;
+        if (saveBaseline) {
+            baselinePath = await ask(rl, 'Baseline path [bellwether-baseline.json]: ') || 'bellwether-baseline.json';
+        }
+        const compareBaseline = await ask(rl, '\nCompare against existing baseline (leave empty to skip): ');
+        return {
+            serverCommand,
+            serverArgs,
+            selectedPersonas,
+            outputFormat,
+            outputDir,
+            saveBaseline,
+            baselinePath,
+            compareBaseline: compareBaseline || undefined,
+            maxQuestions,
+        };
+    }
+    finally {
+        rl.close();
+    }
+}
+/**
+ * Create a pause controller for test control.
+ */
+export function createPauseController() {
+    let paused = false;
+    let resumePromiseResolve = null;
+    return {
+        get isPaused() {
+            return paused;
+        },
+        pause() {
+            paused = true;
+            output.info('\n[Test paused. Press Enter to resume, or Ctrl+C to abort]');
+        },
+        resume() {
+            paused = false;
+            if (resumePromiseResolve) {
+                resumePromiseResolve();
+                resumePromiseResolve = null;
+            }
+            output.info('[Resumed]');
+        },
+        async waitIfPaused() {
+            if (!paused)
+                return;
+            return new Promise((resolve) => {
+                resumePromiseResolve = resolve;
+            });
+        },
+    };
+}
+/**
+ * Setup keyboard listener for pause/resume during test.
+ */
+export function setupInteractiveKeyboard(controller) {
+    const rl = createPrompt();
+    // Don't block the process
+    rl.on('line', () => {
+        if (controller.isPaused) {
+            controller.resume();
+        }
+        else {
+            controller.pause();
+        }
+    });
+    // Handle SIGINT gracefully
+    rl.on('SIGINT', () => {
+        output.info('\nTest aborted by user.');
+        process.exit(130);
+    });
+    return () => {
+        rl.close();
+    };
+}
+/**
+ * Display a summary of the interactive configuration.
+ */
+export function displayConfigSummary(config) {
+    output.info('\n=== Configuration Summary ===');
+    output.info(`Server: ${config.serverCommand} ${config.serverArgs.join(' ')}`);
+    output.info(`Personas: ${config.selectedPersonas.join(', ')}`);
+    output.info(`Output format: ${config.outputFormat}`);
+    output.info(`Output directory: ${config.outputDir}`);
+    output.info(`Max questions: ${config.maxQuestions ?? 'default'}`);
+    if (config.saveBaseline) {
+        output.info(`Save baseline: ${config.baselinePath}`);
+    }
+    if (config.compareBaseline) {
+        output.info(`Compare baseline: ${config.compareBaseline}`);
+    }
+    output.newline();
+}
+//# sourceMappingURL=interactive.js.map

package/dist/cli/output/terminal-reporter.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Terminal formatting helpers for check command output.
+ */
+import type { InterviewResult, ToolProfile } from '../../interview/types.js';
+import type { ToolProgressSummary } from '../../interview/interviewer.js';
+export interface CheckSummary {
+    fullyTested: number;
+    skipped: string[];
+    mocked: string[];
+    issueTools: string[];
+    lines: string[];
+    nextSteps: string[];
+}
+export declare function formatToolResultLine(summary: ToolProgressSummary): string;
+export declare function buildCheckSummary(result: InterviewResult): CheckSummary;
+export declare function formatConfidenceLevel(level?: 'high' | 'medium' | 'low'): string;
+export declare function colorizeConfidence(label: string, level?: 'high' | 'medium' | 'low'): string;
+export declare function profileHasIssues(profile: ToolProfile): boolean;
+//# sourceMappingURL=terminal-reporter.d.ts.map

package/dist/cli/output/terminal-reporter.js

@@ -0,0 +1,104 @@
+/**
+ * Terminal formatting helpers for check command output.
+ */
+import chalk from 'chalk';
+import { RELIABILITY_DISPLAY, CONFIDENCE_INDICATORS, DISPLAY_LIMITS } from '../../constants.js';
+import { getOutputConfig } from '../output.js';
+export function formatToolResultLine(summary) {
+    const statusSymbol = resolveStatusSymbol(summary);
+    const toolLabel = summary.toolName;
+    const base = `${statusSymbol} ${toolLabel}`;
+    if (summary.skipped) {
+        const reason = summary.skipReason ?? 'Skipped: external service not configured';
+        return `${base} skipped (${reason})`;
+    }
+    const passPart = `${summary.passedTests}/${summary.totalTests} passed`;
+    const validationPart = summary.validationTotal > 0
+        ? ` (validation: ${summary.validationPassed}/${summary.validationTotal})`
+        : '';
+    const timingPart = summary.avgMs > 0 ? ` (${summary.avgMs}ms avg)` : '';
+    const mockPart = summary.mocked
+        ? ` (mocked${summary.mockService ? `: ${summary.mockService}` : ''})`
+        : '';
+    return `${base} ${passPart}${validationPart}${mockPart}${timingPart}`;
+}
+export function buildCheckSummary(result) {
+    const toolProfiles = result.toolProfiles;
+    const skipped = toolProfiles.filter(p => p.skipped).map(p => p.name);
+    const mocked = toolProfiles.filter(p => p.mocked).map(p => p.name);
+    const issueTools = toolProfiles.filter(profileHasIssues).map(p => p.name);
+    const fullyTested = toolProfiles.filter(p => !p.skipped && !p.mocked).length;
+    const lines = [];
+    lines.push('Summary:');
+    lines.push(`✓ ${fullyTested} tools fully tested`);
+    if (skipped.length > 0) {
+        lines.push(`⚠ ${skipped.length} tools skipped`);
+    }
+    if (mocked.length > 0) {
+        lines.push(`⚠ ${mocked.length} tools mocked`);
+    }
+    if (issueTools.length > 0) {
+        lines.push(`✗ ${issueTools.length} tools have issues`);
+    }
+    const nextSteps = [];
+    const externalServices = result.metadata.externalServices;
+    if (externalServices?.unconfiguredServices.length) {
+        const services = externalServices.unconfiguredServices.join(', ');
+        nextSteps.push(`Configure ${services} credentials to test skipped tools`);
+    }
+    if (issueTools.length > 0) {
+        const issueList = issueTools.slice(0, DISPLAY_LIMITS.SUMMARY_ISSUE_PREVIEW).join(', ');
+        const extra = issueTools.length > DISPLAY_LIMITS.SUMMARY_ISSUE_PREVIEW ? '...' : '';
+        nextSteps.push(`Review issues in CONTRACT.md (${issueList}${extra})`);
+    }
+    if (mocked.length > 0) {
+        nextSteps.push('Re-run with real credentials to confirm mocked tool behavior');
+    }
+    return { fullyTested, skipped, mocked, issueTools, lines, nextSteps };
+}
+export function formatConfidenceLevel(level) {
+    if (!level) {
+        return '-';
+    }
+    const indicator = CONFIDENCE_INDICATORS[level];
+    return `${indicator} ${level.toUpperCase()}`;
+}
+export function colorizeConfidence(label, level) {
+    if (!level || !supportsColor()) {
+        return label;
+    }
+    switch (level) {
+        case 'high':
+            return chalk.green(label);
+        case 'medium':
+            return chalk.yellow(label);
+        case 'low':
+            return chalk.red(label);
+    }
+}
+export function profileHasIssues(profile) {
+    return profile.interactions.some(i => !i.mocked && i.outcomeAssessment && !i.outcomeAssessment.correct);
+}
+function resolveStatusSymbol(summary) {
+    if (summary.skipped) {
+        return RELIABILITY_DISPLAY.SYMBOLS.WARN;
+    }
+    if (summary.totalTests === 0) {
+        return RELIABILITY_DISPLAY.SYMBOLS.WARN;
+    }
+    if (summary.passedTests === summary.totalTests) {
+        return RELIABILITY_DISPLAY.SYMBOLS.PASS;
+    }
+    if (summary.passedTests === 0) {
+        return RELIABILITY_DISPLAY.SYMBOLS.FAIL;
+    }
+    return RELIABILITY_DISPLAY.SYMBOLS.WARN;
+}
+function supportsColor() {
+    const config = getOutputConfig();
+    if (config.noColor) {
+        return false;
+    }
+    return process.stderr.isTTY ?? false;
+}
+//# sourceMappingURL=terminal-reporter.js.map

package/dist/cli/output.d.ts

@@ -0,0 +1,226 @@
+/**
+ * CLI Output Module
+ *
+ * Provides structured output for CLI commands with support for:
+ * - Different message types (info, success, error, warning)
+ * - Quiet mode to suppress non-essential output
+ * - Consistent formatting across all commands
+ *
+ * This module is for USER-FACING output only. For diagnostic/debug logging,
+ * use the logging module (src/logging/logger.ts).
+ */
+/**
+ * Output configuration options.
+ */
+export interface OutputConfig {
+    /** Suppress non-essential output */
+    quiet?: boolean;
+    /** Disable colored output */
+    noColor?: boolean;
+}
+/**
+ * Configure global output settings.
+ */
+export declare function configureOutput(config: OutputConfig): void;
+/**
+ * Get current output configuration.
+ */
+export declare function getOutputConfig(): OutputConfig;
+/**
+ * Reset output configuration to defaults.
+ */
+export declare function resetOutput(): void;
+/**
+ * Check if output is in quiet mode.
+ */
+export declare function isQuiet(): boolean;
+/**
+ * Standard information output.
+ * Use for progress messages, status updates, and general information.
+ */
+export declare function info(message: string): void;
+/**
+ * Success message output.
+ * Use for completion messages and positive confirmations.
+ */
+export declare function success(message: string): void;
+/**
+ * Warning message output.
+ * Always shown (not suppressed by quiet mode) as warnings are important.
+ */
+export declare function warn(message: string): void;
+/**
+ * Error message output.
+ * Always shown (not suppressed by quiet mode) as errors are critical.
+ */
+export declare function error(message: string): void;
+/**
+ * Debug output (only shown in verbose mode).
+ * For detailed information during development/troubleshooting.
+ */
+export declare function debug(message: string, verbose?: boolean): void;
+/**
+ * Print a blank line for formatting.
+ */
+export declare function newline(): void;
+/**
+ * Print multiple lines.
+ */
+export declare function lines(...messages: string[]): void;
+/**
+ * Print formatted JSON output.
+ * Always shown as this is typically requested data output.
+ */
+export declare function json(data: unknown): void;
+/**
+ * Print a section header.
+ */
+export declare function section(title: string): void;
+/**
+ * Print a key-value pair.
+ */
+export declare function keyValue(key: string, value: string | number | boolean | undefined): void;
+/**
+ * Print a list item.
+ */
+export declare function listItem(item: string, indent?: number): void;
+/**
+ * Print numbered list items.
+ */
+export declare function numberedList(items: string[], startIndex?: number): void;
+/**
+ * Create a scoped output instance for a specific command.
+ * Useful for commands that need to track their own quiet state.
+ */
+export declare function createOutput(config?: OutputConfig): Output;
+/**
+ * Output class for scoped output with its own configuration.
+ */
+export declare class Output {
+    private config;
+    constructor(config?: OutputConfig);
+    info(message: string): void;
+    success(message: string): void;
+    warn(message: string): void;
+    error(message: string): void;
+    debug(message: string, verbose?: boolean): void;
+    newline(): void;
+    lines(...messages: string[]): void;
+    json(data: unknown): void;
+    section(title: string): void;
+    keyValue(key: string, value: string | number | boolean | undefined): void;
+    listItem(item: string, indent?: number): void;
+    numberedList(items: string[], startIndex?: number): void;
+}
+/**
+ * Streaming text display configuration.
+ */
+export interface StreamingDisplayConfig {
+    /** Prefix to show before streaming output */
+    prefix?: string;
+    /** Suffix to show after streaming completes */
+    suffix?: string;
+    /** Whether to show a spinner during streaming */
+    showSpinner?: boolean;
+    /** Maximum line width before wrapping */
+    maxWidth?: number;
+    /** Stream to write to (defaults to stdout) */
+    stream?: NodeJS.WriteStream;
+    /** Color/style for the streaming text */
+    style?: 'dim' | 'normal' | 'bright';
+}
+/**
+ * Streaming display controller for real-time LLM output.
+ * Provides clean, formatted output during streaming operations.
+ */
+export declare class StreamingDisplay {
+    private config;
+    private stream;
+    private buffer;
+    private lineLength;
+    private isActive;
+    private quietMode;
+    constructor(config?: StreamingDisplayConfig);
+    /**
+     * Start streaming display with optional prefix.
+     */
+    start(prefix?: string): void;
+    /**
+     * Write a chunk of streaming text.
+     */
+    write(chunk: string): void;
+    /**
+     * Complete the streaming display.
+     */
+    finish(suffix?: string): string;
+    /**
+     * Abort the streaming display (e.g., on error).
+     */
+    abort(message?: string): void;
+    /**
+     * Get the complete buffer content.
+     */
+    getBuffer(): string;
+    /**
+     * Check if streaming is currently active.
+     */
+    isStreaming(): boolean;
+}
+/**
+ * Create a streaming display for interview operations.
+ * Provides context-appropriate prefixes and styling.
+ */
+export declare function createStreamingDisplay(operation: 'generating' | 'analyzing' | 'synthesizing', context?: string): StreamingDisplay;
+/**
+ * Simple streaming callback that writes to stdout.
+ * Use this for basic streaming output without the full StreamingDisplay.
+ */
+export declare function createStreamingCallback(prefix?: string): {
+    onStart: (operation: string, context?: string) => void;
+    onChunk: (chunk: string, operation: string) => void;
+    onComplete: (text: string, operation: string) => void;
+    onError: (error: Error, operation: string) => void;
+};
+/**
+ * Diff summary data used for displaying comparison results.
+ */
+export interface DiffSummary {
+    severity: string;
+    toolsAdded: number;
+    toolsRemoved: number;
+    toolsModified: number;
+    behaviorChanges: number;
+}
+/**
+ * Get the icon for a severity level.
+ */
+export declare function getSeverityIcon(severity: string): string;
+/**
+ * Default export for convenient importing.
+ */
+declare const _default: {
+    configureOutput: typeof configureOutput;
+    getOutputConfig: typeof getOutputConfig;
+    resetOutput: typeof resetOutput;
+    isQuiet: typeof isQuiet;
+    info: typeof info;
+    success: typeof success;
+    warn: typeof warn;
+    error: typeof error;
+    debug: typeof debug;
+    newline: typeof newline;
+    lines: typeof lines;
+    json: typeof json;
+    section: typeof section;
+    keyValue: typeof keyValue;
+    listItem: typeof listItem;
+    numberedList: typeof numberedList;
+    createOutput: typeof createOutput;
+    Output: typeof Output;
+    StreamingDisplay: typeof StreamingDisplay;
+    createStreamingDisplay: typeof createStreamingDisplay;
+    createStreamingCallback: typeof createStreamingCallback;
+    getSeverityIcon: typeof getSeverityIcon;
+};
+export default _default;
+//# sourceMappingURL=output.d.ts.map