@dotsetlabs/bellwether 0.10.0

package/dist/cli/commands/golden.js

@@ -0,0 +1,407 @@
+/**
+ * Golden command - Manage golden outputs for tool validation.
+ *
+ * Commands:
+ *   bellwether golden save --tool <name>      Save current output as golden
+ *   bellwether golden compare                  Compare against all golden outputs
+ *   bellwether golden list                     List all saved golden outputs
+ *   bellwether golden delete --tool <name>    Delete a golden output
+ */
+import { Command } from 'commander';
+import { existsSync, mkdirSync } from 'fs';
+import { MCPClient } from '../../transport/mcp-client.js';
+import { discover } from '../../discovery/discovery.js';
+import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
+import { validateConfigForCheck } from '../../config/validator.js';
+import { getGoldenStorePath, saveGoldenOutput, createGoldenOutput, listGoldenOutputs, deleteGoldenOutput, compareWithGolden, } from '../../baseline/golden-output.js';
+import * as output from '../output.js';
+import { EXIT_CODES, PATHS } from '../../constants.js';
+import { formatDateISO } from '../../utils/index.js';
+export const goldenCommand = new Command('golden')
+    .description('Manage golden outputs for tool validation');
+// Save command
+goldenCommand
+    .command('save')
+    .description('Capture current tool output as golden reference')
+    .requiredOption('--tool <name>', 'Tool name to capture golden output for')
+    .option('-c, --config <path>', 'Path to config file', PATHS.DEFAULT_CONFIG_FILENAME)
+    .option('--args <json>', 'JSON arguments to pass to the tool')
+    .option('--mode <mode>', 'Comparison mode: exact, structural, semantic')
+    .option('--allowed-drift <paths>', 'Comma-separated JSONPath patterns for allowed changes')
+    .option('--no-normalize-timestamps', 'Disable timestamp normalization')
+    .option('--no-normalize-uuids', 'Disable UUID normalization')
+    .option('--description <text>', 'Description of this golden output')
+    .action(async (options) => {
+    // Load configuration
+    let config;
+    try {
+        config = loadConfig(options.config);
+    }
+    catch (error) {
+        if (error instanceof ConfigNotFoundError) {
+            output.error(error.message);
+            process.exit(EXIT_CODES.ERROR);
+        }
+        throw error;
+    }
+    // Validate config
+    const serverCommand = config.server.command;
+    const args = config.server.args;
+    try {
+        validateConfigForCheck(config, serverCommand);
+    }
+    catch (error) {
+        output.error(error instanceof Error ? error.message : String(error));
+        process.exit(EXIT_CODES.ERROR);
+    }
+    const argsJson = options.args ?? config.golden.defaultArgs;
+    const mode = options.mode ?? config.golden.mode;
+    const normalizeTimestamps = options.normalizeTimestamps === false
+        ? false
+        : config.golden.normalizeTimestamps;
+    const normalizeUuids = options.normalizeUuids === false
+        ? false
+        : config.golden.normalizeUuids;
+    // Parse tool arguments
+    let toolArgs;
+    try {
+        toolArgs = JSON.parse(argsJson);
+    }
+    catch (error) {
+        output.error(`Invalid JSON for --args: ${error instanceof Error ? error.message : error}`);
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Parse allowed drift paths
+    const allowedDrift = options.allowedDrift
+        ? options.allowedDrift.split(',').map((p) => p.trim())
+        : [];
+    // Validate comparison mode
+    const validModes = ['exact', 'structural', 'semantic'];
+    if (!validModes.includes(mode)) {
+        output.error(`Invalid mode "${mode}". Valid modes: ${validModes.join(', ')}`);
+        process.exit(EXIT_CODES.ERROR);
+    }
+    const outputDir = config.output.dir;
+    const storePath = getGoldenStorePath(outputDir);
+    output.info(`Capturing golden output for: ${options.tool}`);
+    output.info(`Mode: ${mode}`);
+    if (allowedDrift.length > 0) {
+        output.info(`Allowed drift paths: ${allowedDrift.join(', ')}`);
+    }
+    output.newline();
+    const mcpClient = new MCPClient({
+        timeout: config.server.timeout,
+        debug: config.logging.level === 'debug',
+        transport: 'stdio',
+    });
+    try {
+        // Connect to server
+        output.info('Connecting to MCP server...');
+        await mcpClient.connect(serverCommand, args, config.server.env);
+        // Discover tools
+        const discovery = await discover(mcpClient, serverCommand, args);
+        const tool = discovery.tools.find(t => t.name === options.tool);
+        if (!tool) {
+            output.error(`Tool not found: ${options.tool}`);
+            output.info(`Available tools: ${discovery.tools.map(t => t.name).join(', ')}`);
+            process.exit(EXIT_CODES.ERROR);
+        }
+        // Call the tool
+        output.info(`Calling tool: ${options.tool}`);
+        const response = await mcpClient.callTool(options.tool, toolArgs);
+        if (response.isError) {
+            output.error('Tool returned an error:');
+            const textContent = response.content.find(c => c.type === 'text');
+            if (textContent && 'text' in textContent) {
+                output.error(String(textContent.text));
+            }
+            process.exit(EXIT_CODES.ERROR);
+        }
+        // Create golden output
+        const golden = createGoldenOutput(options.tool, toolArgs, response, {
+            mode,
+            allowedDrift,
+            normalizeTimestamps,
+            normalizeUuids,
+            description: options.description,
+        });
+        // Ensure output directory exists
+        mkdirSync(outputDir, { recursive: true });
+        // Save golden output
+        saveGoldenOutput(golden, storePath);
+        output.success('\nGolden output saved!');
+        output.info(`Store: ${storePath}`);
+        output.info(`Tool: ${golden.toolName}`);
+        output.info(`Content type: ${golden.output.contentType}`);
+        output.info(`Content hash: ${golden.output.contentHash}`);
+        output.info(`Comparison mode: ${golden.tolerance.mode}`);
+        // Show preview of captured content
+        const preview = golden.output.raw.slice(0, 200);
+        if (preview) {
+            output.newline();
+            output.info('Output preview:');
+            output.info(preview + (golden.output.raw.length > 200 ? '...' : ''));
+        }
+    }
+    catch (error) {
+        output.error(`Failed to capture golden output: ${error instanceof Error ? error.message : error}`);
+        process.exit(EXIT_CODES.ERROR);
+    }
+    finally {
+        await mcpClient.disconnect();
+    }
+});
+// Compare command
+goldenCommand
+    .command('compare')
+    .description('Compare current outputs against saved golden outputs')
+    .option('-c, --config <path>', 'Path to config file', PATHS.DEFAULT_CONFIG_FILENAME)
+    .option('--tool <name>', 'Only compare a specific tool')
+    .option('--fail-on-drift', 'Exit with error if any drift detected')
+    .option('--format <format>', 'Output format: text, json, markdown')
+    .action(async (options) => {
+    // Load configuration
+    let config;
+    try {
+        config = loadConfig(options.config);
+    }
+    catch (error) {
+        if (error instanceof ConfigNotFoundError) {
+            output.error(error.message);
+            process.exit(EXIT_CODES.ERROR);
+        }
+        throw error;
+    }
+    // Validate config
+    const serverCommand = config.server.command;
+    const args = config.server.args;
+    try {
+        validateConfigForCheck(config, serverCommand);
+    }
+    catch (error) {
+        output.error(error instanceof Error ? error.message : String(error));
+        process.exit(EXIT_CODES.ERROR);
+    }
+    const outputDir = config.output.dir;
+    const storePath = getGoldenStorePath(outputDir);
+    const format = options.format ?? config.golden.compareFormat;
+    if (!existsSync(storePath)) {
+        output.warn('No golden outputs found.');
+        output.info('Use "bellwether golden save --tool <name>" to capture golden outputs.');
+        process.exit(EXIT_CODES.CLEAN);
+    }
+    const goldens = listGoldenOutputs(storePath);
+    const filteredGoldens = options.tool
+        ? goldens.filter(g => g.toolName === options.tool)
+        : goldens;
+    if (filteredGoldens.length === 0) {
+        if (options.tool) {
+            output.warn(`No golden output found for tool: ${options.tool}`);
+        }
+        else {
+            output.warn('No golden outputs saved.');
+        }
+        process.exit(EXIT_CODES.CLEAN);
+    }
+    output.info(`Comparing ${filteredGoldens.length} golden output(s)...`);
+    output.newline();
+    const mcpClient = new MCPClient({
+        timeout: config.server.timeout,
+        debug: config.logging.level === 'debug',
+        transport: 'stdio',
+    });
+    try {
+        // Connect to server
+        output.info('Connecting to MCP server...');
+        await mcpClient.connect(serverCommand, args, config.server.env);
+        output.newline();
+        const results = [];
+        for (const golden of filteredGoldens) {
+            output.info(`Comparing: ${golden.toolName}`);
+            try {
+                const response = await mcpClient.callTool(golden.toolName, golden.inputArgs);
+                const result = compareWithGolden(golden, response);
+                results.push(result);
+                const icon = result.passed ? '\u2713' : '\u2717';
+                if (result.passed) {
+                    output.success(`  ${icon} ${result.summary}`);
+                }
+                else {
+                    output.error(`  ${icon} ${result.summary}`);
+                    if (result.differences.filter(d => !d.allowed).length <= 5) {
+                        for (const diff of result.differences.filter(d => !d.allowed)) {
+                            output.warn(`    - ${diff.description} at ${diff.path}`);
+                        }
+                    }
+                }
+            }
+            catch (error) {
+                results.push({
+                    toolName: golden.toolName,
+                    passed: false,
+                    severity: 'breaking',
+                    mode: golden.tolerance.mode,
+                    goldenCapturedAt: golden.capturedAt,
+                    differences: [{
+                            type: 'changed',
+                            path: '$',
+                            expected: 'successful response',
+                            actual: `error: ${error instanceof Error ? error.message : String(error)}`,
+                            allowed: false,
+                            description: 'Tool call failed',
+                        }],
+                    summary: `Tool call failed: ${error instanceof Error ? error.message : String(error)}`,
+                });
+                output.error(`  \u2717 Tool call failed: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        output.newline();
+        // Summary
+        const passed = results.filter(r => r.passed).length;
+        const failed = results.length - passed;
+        if (format === 'json') {
+            output.info(JSON.stringify(results, null, 2));
+        }
+        else if (format === 'markdown') {
+            output.info(formatResultsMarkdown(results));
+        }
+        else {
+            output.info('--- Summary ---');
+            output.info(`Total: ${results.length}`);
+            output.success(`Passed: ${passed}`);
+            if (failed > 0) {
+                output.error(`Failed: ${failed}`);
+            }
+        }
+        if (options.failOnDrift && failed > 0) {
+            process.exit(EXIT_CODES.BREAKING);
+        }
+    }
+    finally {
+        await mcpClient.disconnect();
+    }
+});
+// List command
+goldenCommand
+    .command('list')
+    .description('List all saved golden outputs')
+    .option('-c, --config <path>', 'Path to config file', PATHS.DEFAULT_CONFIG_FILENAME)
+    .option('--format <format>', 'Output format: text, json')
+    .action(async (options) => {
+    // Load configuration
+    let config;
+    try {
+        config = loadConfig(options.config);
+    }
+    catch (error) {
+        if (error instanceof ConfigNotFoundError) {
+            output.error(error.message);
+            process.exit(EXIT_CODES.ERROR);
+        }
+        throw error;
+    }
+    const outputDir = config.output.dir;
+    const storePath = getGoldenStorePath(outputDir);
+    const format = options.format ?? config.golden.listFormat;
+    if (!existsSync(storePath)) {
+        output.info('No golden outputs found.');
+        output.info('Use "bellwether golden save --tool <name>" to capture golden outputs.');
+        return;
+    }
+    const goldens = listGoldenOutputs(storePath);
+    if (goldens.length === 0) {
+        output.info('No golden outputs saved.');
+        return;
+    }
+    if (format === 'json') {
+        output.info(JSON.stringify(goldens, null, 2));
+        return;
+    }
+    output.info(`Golden Outputs (${goldens.length}):`);
+    output.newline();
+    output.info('| Tool | Captured | Mode | Content Type |');
+    output.info('|------|----------|------|--------------|');
+    for (const golden of goldens) {
+        const captured = formatDateISO(new Date(golden.capturedAt));
+        output.info(`| \`${golden.toolName}\` | ${captured} | ${golden.tolerance.mode} | ${golden.output.contentType} |`);
+    }
+    output.newline();
+    output.info(`Store: ${storePath}`);
+});
+// Delete command
+goldenCommand
+    .command('delete')
+    .description('Delete a saved golden output')
+    .requiredOption('--tool <name>', 'Tool name to delete golden output for')
+    .option('-c, --config <path>', 'Path to config file', PATHS.DEFAULT_CONFIG_FILENAME)
+    .option('--all', 'Delete all golden outputs for this tool (if multiple with different args)')
+    .action(async (options) => {
+    // Load configuration
+    let config;
+    try {
+        config = loadConfig(options.config);
+    }
+    catch (error) {
+        if (error instanceof ConfigNotFoundError) {
+            output.error(error.message);
+            process.exit(EXIT_CODES.ERROR);
+        }
+        throw error;
+    }
+    const outputDir = config.output.dir;
+    const storePath = getGoldenStorePath(outputDir);
+    if (!existsSync(storePath)) {
+        output.warn('No golden outputs found.');
+        return;
+    }
+    const deleted = deleteGoldenOutput(options.tool, storePath);
+    if (deleted) {
+        output.success(`Deleted golden output for: ${options.tool}`);
+    }
+    else {
+        output.warn(`No golden output found for: ${options.tool}`);
+    }
+});
+/**
+ * Format comparison results as Markdown.
+ */
+function formatResultsMarkdown(results) {
+    const lines = [];
+    lines.push('## Golden Output Validation');
+    lines.push('');
+    lines.push('| Tool | Status | Mode | Differences |');
+    lines.push('|------|--------|------|-------------|');
+    for (const result of results) {
+        const status = result.passed ? '✓ Match' : `✗ ${result.severity}`;
+        const diffCount = result.differences.filter(d => !d.allowed).length;
+        lines.push(`| \`${result.toolName}\` | ${status} | ${result.mode} | ${diffCount} |`);
+    }
+    lines.push('');
+    // Details for failed comparisons
+    const failed = results.filter(r => !r.passed);
+    if (failed.length > 0) {
+        lines.push('### Drift Details');
+        lines.push('');
+        for (const result of failed) {
+            lines.push(`#### ${result.toolName}`);
+            lines.push('');
+            lines.push(`**Golden captured:** ${formatDateISO(new Date(result.goldenCapturedAt))}`);
+            lines.push(`**Mode:** ${result.mode}`);
+            lines.push(`**Severity:** ${result.severity}`);
+            lines.push('');
+            lines.push('**Changes:**');
+            for (const diff of result.differences.filter(d => !d.allowed)) {
+                lines.push(`- ${diff.description}`);
+                if (diff.expected !== undefined) {
+                    lines.push(`  - Expected: \`${String(diff.expected)}\``);
+                }
+                if (diff.actual !== undefined) {
+                    lines.push(`  - Actual: \`${String(diff.actual)}\``);
+                }
+            }
+            lines.push('');
+        }
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=golden.js.map

package/dist/cli/commands/history.d.ts

@@ -0,0 +1,10 @@
+/**
+ * History command for viewing baseline history in Bellwether Cloud.
+ */
+import { Command } from 'commander';
+export declare const historyCommand: Command;
+/**
+ * Diff command for comparing specific versions.
+ */
+export declare const diffCommand: Command;
+//# sourceMappingURL=history.d.ts.map

package/dist/cli/commands/history.js

@@ -0,0 +1,202 @@
+/**
+ * History command for viewing baseline history in Bellwether Cloud.
+ */
+import { Command } from 'commander';
+import { getSessionToken, getLinkedProject } from '../../cloud/auth.js';
+import { createCloudClient } from '../../cloud/client.js';
+import { formatDateLocale } from '../../utils/index.js';
+import { EXIT_CODES } from '../../constants.js';
+import * as output from '../output.js';
+import { getSeverityIcon } from '../output.js';
+export const historyCommand = new Command('history')
+    .description('View baseline history for a project')
+    .argument('[project-id]', 'Project ID (uses linked project if not specified)')
+    .option('-n, --limit <n>', 'Number of versions to show', '10')
+    .option('--json', 'Output as JSON')
+    .option('--session <session>', 'Session token (overrides stored/env session)')
+    .action(async (projectIdArg, options) => {
+    // Get session
+    const sessionToken = options.session ?? getSessionToken();
+    if (!sessionToken) {
+        output.error('Not authenticated. Run `bellwether login` first.');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Determine project ID
+    let projectId = projectIdArg;
+    if (!projectId) {
+        const link = getLinkedProject();
+        if (link) {
+            projectId = link.projectId;
+        }
+    }
+    if (!projectId) {
+        output.error('No project specified.');
+        output.error('\nEither:');
+        output.error('  - Provide a project ID as argument');
+        output.error('  - Run `bellwether link` to link this directory to a project');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Create client and fetch history
+    const client = createCloudClient({ sessionToken });
+    if (!client.isAuthenticated()) {
+        output.error('Authentication failed. Run `bellwether login` to re-authenticate.');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    const limit = parseInt(options.limit, 10);
+    try {
+        const history = await client.getHistory(projectId, limit);
+        if (options.json) {
+            output.info(JSON.stringify(history, null, 2));
+            return;
+        }
+        if (history.length === 0) {
+            output.info('No baselines uploaded yet.');
+            output.info('\nRun `bellwether check <server> --save-baseline` then `bellwether upload`.');
+            return;
+        }
+        // Get project info for display
+        const project = await client.getProject(projectId);
+        const projectName = project?.name ?? projectId;
+        output.info(`Baseline History: ${projectName}`);
+        output.info(`Showing ${history.length} version(s)\n`);
+        output.info('Ver  Uploaded                 CLI Version  Hash');
+        output.info('───  ───────────────────────  ───────────  ────────────────');
+        for (const baseline of history) {
+            const date = formatDateLocale(baseline.uploadedAt);
+            const cliVersion = baseline.cliVersion.padEnd(11);
+            const hash = baseline.hash.slice(0, 16);
+            output.info(`${baseline.version.toString().padStart(3)}  ` +
+                `${date.padEnd(23)}  ` +
+                `${cliVersion}  ` +
+                hash);
+        }
+        // Show diff summary if multiple versions
+        if (history.length >= 2) {
+            output.info('\nLatest changes:');
+            try {
+                const diff = await client.getLatestDiff(projectId);
+                if (diff) {
+                    printDiffSummary(diff);
+                }
+            }
+            catch {
+                // Diff failed, just skip
+            }
+        }
+    }
+    catch (error) {
+        output.error('Failed to fetch history: ' + (error instanceof Error ? error.message : String(error)));
+        process.exit(EXIT_CODES.ERROR);
+    }
+});
+/**
+ * Print a diff summary (compact format).
+ */
+function printDiffSummary(diff) {
+    const parts = [];
+    if (diff.toolsAdded > 0) {
+        parts.push(`+${diff.toolsAdded} tools`);
+    }
+    if (diff.toolsRemoved > 0) {
+        parts.push(`-${diff.toolsRemoved} tools`);
+    }
+    if (diff.toolsModified > 0) {
+        parts.push(`~${diff.toolsModified} modified`);
+    }
+    if (diff.behaviorChanges > 0) {
+        parts.push(`${diff.behaviorChanges} behavior changes`);
+    }
+    if (parts.length === 0) {
+        output.info('  No changes from previous version');
+    }
+    else {
+        const icon = getSeverityIcon(diff.severity);
+        output.info(`  ${icon} ${diff.severity}: ${parts.join(', ')}`);
+    }
+}
+/**
+ * Diff command for comparing specific versions.
+ */
+export const diffCommand = new Command('diff')
+    .description('Compare two baseline versions')
+    .argument('<from>', 'From version number')
+    .argument('<to>', 'To version number')
+    .option('-p, --project <id>', 'Project ID (uses linked project if not specified)')
+    .option('--json', 'Output as JSON')
+    .option('--session <session>', 'Session token (overrides stored/env session)')
+    .action(async (fromArg, toArg, options) => {
+    // Get session
+    const sessionToken = options.session ?? getSessionToken();
+    if (!sessionToken) {
+        output.error('Not authenticated. Run `bellwether login` first.');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Determine project ID
+    let projectId = options.project;
+    if (!projectId) {
+        const link = getLinkedProject();
+        if (link) {
+            projectId = link.projectId;
+        }
+    }
+    if (!projectId) {
+        output.error('No project specified. Use --project <id> or run `bellwether link`.');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Parse versions
+    const fromVersion = parseInt(fromArg, 10);
+    const toVersion = parseInt(toArg, 10);
+    if (isNaN(fromVersion) || isNaN(toVersion)) {
+        output.error('Invalid version numbers. Provide integers (e.g., `bellwether diff 1 2`).');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    // Create client and fetch diff
+    const client = createCloudClient({ sessionToken });
+    if (!client.isAuthenticated()) {
+        output.error('Authentication failed. Run `bellwether login` to re-authenticate.');
+        process.exit(EXIT_CODES.ERROR);
+    }
+    try {
+        const diff = await client.getDiff(projectId, fromVersion, toVersion);
+        if (options.json) {
+            output.info(JSON.stringify(diff, null, 2));
+            return;
+        }
+        output.info(`Comparing v${fromVersion} → v${toVersion}\n`);
+        const severityIcon = {
+            none: '✓',
+            info: 'ℹ',
+            warning: '⚠',
+            breaking: '✗',
+        };
+        output.info(`Severity: ${severityIcon[diff.severity] ?? '?'} ${diff.severity.toUpperCase()}`);
+        output.info('');
+        if (diff.toolsAdded > 0) {
+            output.info(`Tools added:     +${diff.toolsAdded}`);
+        }
+        if (diff.toolsRemoved > 0) {
+            output.info(`Tools removed:   -${diff.toolsRemoved}`);
+        }
+        if (diff.toolsModified > 0) {
+            output.info(`Tools modified:  ~${diff.toolsModified}`);
+        }
+        if (diff.behaviorChanges > 0) {
+            output.info(`Behavior changes: ${diff.behaviorChanges}`);
+        }
+        if (diff.toolsAdded === 0 &&
+            diff.toolsRemoved === 0 &&
+            diff.toolsModified === 0 &&
+            diff.behaviorChanges === 0) {
+            output.info('No changes detected between these versions.');
+        }
+        if (diff.severity === 'breaking') {
+            output.info('\n⚠️  Breaking changes detected!');
+            output.info('   Tools were removed or modified in incompatible ways.');
+        }
+    }
+    catch (error) {
+        output.error('Failed to compute diff: ' + (error instanceof Error ? error.message : String(error)));
+        process.exit(EXIT_CODES.ERROR);
+    }
+});
+//# sourceMappingURL=history.js.map

package/dist/cli/commands/init.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Init command - creates a bellwether.yaml configuration file.
+ *
+ * The generated config includes ALL possible options with comments,
+ * making it self-documenting. Users can customize by editing the file.
+ */
+import { Command } from 'commander';
+export declare const initCommand: Command;
+//# sourceMappingURL=init.d.ts.map