@dotsetlabs/bellwether 2.1.2 → 2.1.3

package/CHANGELOG.md

@@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.1.3] - 2026-02-18
+
+### Added
+
+- **Check reporting utilities for CI**: Added dedicated check output formatting plus new GitHub Action scripts for metrics extraction and JSON-to-JUnit/SARIF conversion.
+- **Shared CLI/runtime helper modules**: Added reusable helpers for server runtime setup, config/report loading, path resolution, and improved contextual error hints.
+- **Docs generation bootstrap script**: Added `scripts/build-docs.mjs` so docs generation can self-bootstrap website dependencies.
+
+### Changed
+
+- **Modular check and baseline command flows**: Refactored `check`/`baseline` internals to reduce coupling and make command behavior easier to maintain.
+- **Transport/interview utility refactors**: Consolidated header/content-type/auth handling and split interview/question fixture types into clearer shared modules.
+- **Documentation updates for current CLI behavior**: Refreshed docs and sidebar navigation for revised `check`/`baseline` behavior and the `validate-config` command.
+
+### Removed
+
+- **Unused core constants module**: Removed `src/constants/core.ts` after internal CLI/runtime simplification.
+
 ## [2.1.2] - 2026-02-16
 
 ### Added

package/README.md

@@ -132,9 +132,9 @@ Comparisons are **protocol-version-aware** — version-specific fields (annotati
 ## GitHub Action
 
 ```yaml
-- uses: dotsetlabs/bellwether@v2.1.2
+- uses: dotsetlabs/bellwether@v2.1.3
   with:
-    version: '2.1.2'
+    version: '2.1.3'
     server-command: 'npx @mcp/your-server'
     baseline-path: './bellwether-baseline.json'
     fail-on-severity: 'warning'

package/dist/baseline/golden-output.d.ts

@@ -155,8 +155,4 @@ export declare function deleteGoldenOutput(toolName: string, storePath: string,
  * Compare current output against a golden output.
  */
 export declare function compareWithGolden(golden: GoldenOutput, currentResponse: MCPToolCallResult): GoldenComparisonResult;
-/**
- * Compare all golden outputs against current tool responses.
- */
-export declare function compareAllGoldens(storePath: string, getToolResponse: (toolName: string, args: Record<string, unknown>) => Promise<MCPToolCallResult>): Promise<GoldenComparisonResult[]>;
 //# sourceMappingURL=golden-output.d.ts.map

package/dist/baseline/golden-output.js

@@ -9,6 +9,7 @@ import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
 import { join, dirname } from 'path';
 import { createHash } from 'crypto';
 import { PATHS } from '../constants.js';
+import { detectContentType as detectCommonContentType } from '../utils/content-type.js';
 // Constants
 const GOLDEN_STORE_VERSION = 1;
 const DEFAULT_GOLDEN_DIR = '.bellwether/golden';
@@ -236,58 +237,12 @@ export function compareWithGolden(golden, currentResponse) {
         summary: generateComparisonSummary(disallowedDiffs, mode),
     };
 }
-/**
- * Compare all golden outputs against current tool responses.
- */
-export function compareAllGoldens(storePath, getToolResponse) {
-    const store = loadGoldenStore(storePath);
-    return Promise.all(store.outputs.map(async (golden) => {
-        try {
-            const response = await getToolResponse(golden.toolName, golden.inputArgs);
-            return compareWithGolden(golden, response);
-        }
-        catch (error) {
-            return {
-                toolName: golden.toolName,
-                passed: false,
-                severity: 'breaking',
-                mode: golden.tolerance.mode,
-                goldenCapturedAt: golden.capturedAt,
-                differences: [{
-                        type: 'changed',
-                        path: '$',
-                        expected: 'successful response',
-                        actual: `error: ${error instanceof Error ? error.message : String(error)}`,
-                        allowed: false,
-                        description: 'Tool call failed',
-                    }],
-                summary: `Tool call failed: ${error instanceof Error ? error.message : String(error)}`,
-            };
-        }
-    }));
-}
 // Helper functions
 /**
  * Detect content type from raw output.
  */
 function detectContentType(raw) {
-    const trimmed = raw.trim();
-    // Check for JSON
-    if ((trimmed.startsWith('{') && trimmed.endsWith('}')) ||
-        (trimmed.startsWith('[') && trimmed.endsWith(']'))) {
-        try {
-            JSON.parse(trimmed);
-            return 'json';
-        }
-        catch {
-            // Not valid JSON
-        }
-    }
-    // Check for Markdown patterns
-    if (/^#|^\*{1,3}[^*]|\[.*\]\(.*\)|^```/.test(trimmed)) {
-        return 'markdown';
-    }
-    return 'text';
+    return detectCommonContentType(raw);
 }
 /**
  * Compute a hash of content for quick comparison.

package/dist/cli/commands/baseline-accept.js

@@ -11,51 +11,14 @@
  *   bellwether baseline accept --dry-run    # Show what would be accepted
  */
 import { Command } from 'commander';
-import { existsSync, readFileSync } from 'fs';
+import { existsSync } from 'fs';
 import { join } from 'path';
 import { createBaseline, saveBaseline, loadBaseline, compareBaselines, acceptDrift, formatDiffText, } from '../../baseline/index.js';
-import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
 import { EXIT_CODES } from '../../constants.js';
 import * as output from '../output.js';
-function loadConfigOrExit(configPath) {
-    try {
-        return loadConfig(configPath);
-    }
-    catch (error) {
-        if (error instanceof ConfigNotFoundError) {
-            output.error(error.message);
-            process.exit(EXIT_CODES.ERROR);
-        }
-        throw error;
-    }
-}
-/**
- * Load interview result from JSON report.
- */
-function loadInterviewResult(reportPath) {
-    if (!existsSync(reportPath)) {
-        throw new Error(`Test report not found: ${reportPath}\n\n` +
-            'Run `bellwether check` first to generate a report.\n' +
-            'Configure in bellwether.yaml:\n' +
-            '  output:\n' +
-            '    format: json  # or "both" for JSON + markdown');
-    }
-    const content = readFileSync(reportPath, 'utf-8');
-    let result;
-    try {
-        result = JSON.parse(content);
-    }
-    catch (error) {
-        throw new Error(`Invalid JSON in report file ${reportPath}: ${error instanceof Error ? error.message : 'Unknown error'}`);
-    }
-    // Validate that this is a check mode result
-    if (result.metadata.model && result.metadata.model !== 'check') {
-        throw new Error(`Baseline operations only work with check mode results.\n\n` +
-            `The report at ${reportPath} was created with explore mode.\n` +
-            'Run `bellwether check` to generate a check mode report first.');
-    }
-    return result;
-}
+import { loadConfigOrExit } from '../utils/config-loader.js';
+import { loadCheckInterviewResult } from '../utils/report-loader.js';
+import { resolvePathFromOutputDirOrCwd } from '../utils/path-resolution.js';
 export const acceptCommand = new Command('accept')
     .description('Accept detected drift as intentional and update the baseline')
     .option('-c, --config <path>', 'Path to config file')
@@ -74,9 +37,7 @@ export const acceptCommand = new Command('accept')
         process.exit(EXIT_CODES.ERROR);
     }
     // Determine paths
-    const baselinePath = resolvedBaselinePath.startsWith('/')
-        ? resolvedBaselinePath
-        : join(outputDir, resolvedBaselinePath);
+    const baselinePath = resolvePathFromOutputDirOrCwd(resolvedBaselinePath, outputDir);
     const reportPath = options.report || join(outputDir, config.output.files.checkReport);
     // Load the existing baseline
     if (!existsSync(baselinePath)) {
@@ -96,7 +57,15 @@ export const acceptCommand = new Command('accept')
     // Load the current test results
     let result;
     try {
-        result = loadInterviewResult(reportPath);
+        result = loadCheckInterviewResult(reportPath, {
+            missingReportMessage: 'Run `bellwether check` first to generate a report.\n' +
+                'Configure in bellwether.yaml:\n' +
+                '  output:\n' +
+                '    format: json  # or "both" for JSON + markdown',
+            invalidModeMessage: () => `Baseline operations only work with check mode results.\n\n` +
+                `The report at ${reportPath} was created with explore mode.\n` +
+                'Run `bellwether check` to generate a check mode report first.',
+        });
     }
     catch (error) {
         output.error(error instanceof Error ? error.message : String(error));

package/dist/cli/commands/baseline.js

@@ -9,61 +9,17 @@
  *   - accept               Accept detected drift as intentional
  */
 import { Command } from 'commander';
-import { existsSync, readFileSync } from 'fs';
+import { existsSync } from 'fs';
 import { join, basename } from 'path';
 import { createBaseline, saveBaseline, loadBaseline, compareBaselines, formatDiffText, formatDiffJson, formatDiffMarkdown, formatDiffCompact, verifyBaselineHash, getBaselineGeneratedAt, getBaselineMode, getBaselineServerCommand, getToolFingerprints, } from '../../baseline/index.js';
 import { BaselineVersionError } from '../../baseline/version.js';
 import { EXIT_CODES, MCP } from '../../constants.js';
 import { getExcludedFeatureNames } from '../../protocol/index.js';
 import { acceptCommand } from './baseline-accept.js';
-import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
 import * as output from '../output.js';
-/**
- * Load interview result from JSON report.
- * Only accepts check mode results - explore results are for documentation only.
- */
-function loadInterviewResult(reportPath) {
-    if (!existsSync(reportPath)) {
-        throw new Error(`Test report not found: ${reportPath}\n\n` +
-            'Run `bellwether check` first with JSON output enabled.\n' +
-            'Configure in bellwether.yaml:\n' +
-            '  output:\n' +
-            '    format: json  # or "both" for JSON + markdown');
-    }
-    const content = readFileSync(reportPath, 'utf-8');
-    let result;
-    try {
-        result = JSON.parse(content);
-    }
-    catch (error) {
-        throw new Error(`Invalid JSON in report file ${reportPath}: ${error instanceof Error ? error.message : 'Unknown error'}`);
-    }
-    // Validate that this is a check mode result, not explore
-    if (result.metadata.model && result.metadata.model !== 'check') {
-        throw new Error(`Baseline operations only work with check mode results.\n\n` +
-            `The report at ${reportPath} was created with explore mode (model: ${result.metadata.model}).\n` +
-            `Explore results are for documentation only and cannot be used for baselines.\n\n` +
-            'To create a baseline:\n' +
-            '  1. Run `bellwether check` to generate a check mode report\n' +
-            '  2. Run `bellwether baseline save` to create the baseline');
-    }
-    return result;
-}
-/**
- * Get the output directory from config or use current directory.
- */
-function loadConfigOrExit(configPath) {
-    try {
-        return loadConfig(configPath);
-    }
-    catch (error) {
-        if (error instanceof ConfigNotFoundError) {
-            output.error(error.message);
-            process.exit(EXIT_CODES.ERROR);
-        }
-        throw error;
-    }
-}
+import { loadConfigOrExit } from '../utils/config-loader.js';
+import { loadCheckInterviewResult } from '../utils/report-loader.js';
+import { resolvePathFromOutputDir, resolvePathFromOutputDirOrCwd, } from '../utils/path-resolution.js';
 export const baselineCommand = new Command('baseline')
     .description('Manage baselines for drift detection')
     .addHelpText('after', `
@@ -99,16 +55,21 @@ baselineCommand
     // Load interview result
     let result;
     try {
-        result = loadInterviewResult(reportPath);
+        result = loadCheckInterviewResult(reportPath, {
+            invalidModeMessage: (model) => `Baseline operations only work with check mode results.\n\n` +
+                `The report at ${reportPath} was created with explore mode (model: ${model}).\n` +
+                `Explore results are for documentation only and cannot be used for baselines.\n\n` +
+                'To create a baseline:\n' +
+                '  1. Run `bellwether check` to generate a check mode report\n' +
+                '  2. Run `bellwether baseline save` to create the baseline',
+        });
     }
     catch (error) {
         output.error(error instanceof Error ? error.message : String(error));
         process.exit(EXIT_CODES.ERROR);
     }
     // Determine baseline path (relative to output dir if not absolute)
-    const finalPath = resolvedBaselinePath.startsWith('/')
-        ? resolvedBaselinePath
-        : join(outputDir, resolvedBaselinePath);
+    const finalPath = resolvePathFromOutputDir(resolvedBaselinePath, outputDir);
     // Check for existing baseline
     if (existsSync(finalPath) && !options.force) {
         output.error(`Baseline already exists: ${finalPath}`);
@@ -147,28 +108,7 @@ baselineCommand
         output.error('No baseline path provided. Set baseline.path or baseline.comparePath in config, or pass a path argument.');
         process.exit(EXIT_CODES.ERROR);
     }
-    // Resolve baseline path consistently with 'show' command:
-    // 1. If absolute path, use as-is
-    // 2. First try relative to outputDir (e.g., .bellwether/)
-    // 3. Fall back to relative to cwd
-    let fullBaselinePath;
-    if (resolvedBaselinePath.startsWith('/')) {
-        fullBaselinePath = resolvedBaselinePath;
-    }
-    else {
-        const outputDirPath = join(outputDir, resolvedBaselinePath);
-        const cwdPath = join(process.cwd(), resolvedBaselinePath);
-        if (existsSync(outputDirPath)) {
-            fullBaselinePath = outputDirPath;
-        }
-        else if (existsSync(cwdPath)) {
-            fullBaselinePath = cwdPath;
-        }
-        else {
-            // Default to outputDir path for error message consistency
-            fullBaselinePath = outputDirPath;
-        }
-    }
+    const fullBaselinePath = resolvePathFromOutputDirOrCwd(resolvedBaselinePath, outputDir);
     if (!existsSync(fullBaselinePath)) {
         output.error(`Baseline not found: ${fullBaselinePath}`);
         output.error('\nRun `bellwether baseline save` to create a baseline.');
@@ -186,7 +126,14 @@ baselineCommand
     const reportPath = options.report || join(outputDir, config.output.files.checkReport);
     let result;
     try {
-        result = loadInterviewResult(reportPath);
+        result = loadCheckInterviewResult(reportPath, {
+            invalidModeMessage: (model) => `Baseline operations only work with check mode results.\n\n` +
+                `The report at ${reportPath} was created with explore mode (model: ${model}).\n` +
+                `Explore results are for documentation only and cannot be used for baselines.\n\n` +
+                'To create a baseline:\n' +
+                '  1. Run `bellwether check` to generate a check mode report\n' +
+                '  2. Run `bellwether baseline save` to create the baseline',
+        });
     }
     catch (error) {
         output.error(error instanceof Error ? error.message : String(error));
@@ -285,9 +232,7 @@ baselineCommand
         process.exit(EXIT_CODES.ERROR);
     }
     // Determine full path
-    const fullPath = resolvedBaselinePath.startsWith('/')
-        ? resolvedBaselinePath
-        : join(outputDir, resolvedBaselinePath);
+    const fullPath = resolvePathFromOutputDirOrCwd(resolvedBaselinePath, outputDir);
     if (!existsSync(fullPath)) {
         output.error(`Baseline not found: ${fullPath}`);
         output.error('\nRun `bellwether baseline save` to create a baseline.');

package/dist/cli/commands/check-formatters.d.ts

@@ -0,0 +1,10 @@
+import { type BehavioralDiff, type BehavioralBaseline } from '../../baseline/index.js';
+/**
+ * Format a diff using the requested output format.
+ */
+export declare function formatDiffOutput(diff: BehavioralDiff, format: string, baselinePath: string): string;
+/**
+ * Format check-only output when no baseline is provided.
+ */
+export declare function formatCheckResults(baseline: BehavioralBaseline, format: string): string | null;
+//# sourceMappingURL=check-formatters.d.ts.map

package/dist/cli/commands/check-formatters.js

@@ -0,0 +1,160 @@
+import { formatDiffText, formatDiffJson, formatDiffCompact, formatDiffGitHubActions, formatDiffMarkdown, formatDiffJUnit, formatDiffSarif, getToolFingerprints, } from '../../baseline/index.js';
+/**
+ * Format a diff using the requested output format.
+ */
+export function formatDiffOutput(diff, format, baselinePath) {
+    switch (format.toLowerCase()) {
+        case 'json':
+            return formatDiffJson(diff);
+        case 'compact':
+            return formatDiffCompact(diff);
+        case 'github':
+            return formatDiffGitHubActions(diff);
+        case 'markdown':
+        case 'md':
+            return formatDiffMarkdown(diff);
+        case 'junit':
+        case 'junit-xml':
+        case 'xml':
+            return formatDiffJUnit(diff, 'bellwether-check');
+        case 'sarif':
+            return formatDiffSarif(diff, baselinePath);
+        case 'text':
+        default:
+            return formatDiffText(diff);
+    }
+}
+/**
+ * Format check-only output when no baseline is provided.
+ */
+export function formatCheckResults(baseline, format) {
+    switch (format.toLowerCase()) {
+        case 'junit':
+        case 'junit-xml':
+        case 'xml':
+            return formatCheckResultsJUnit(baseline);
+        case 'sarif':
+            return formatCheckResultsSarif(baseline);
+        default:
+            return null;
+    }
+}
+function formatCheckResultsJUnit(baseline) {
+    const tools = getToolFingerprints(baseline);
+    const lines = [];
+    const securityFailures = tools.filter((t) => t.securityFingerprint?.findings?.some((f) => f.riskLevel === 'critical' || f.riskLevel === 'high')).length;
+    lines.push('<?xml version="1.0" encoding="UTF-8"?>');
+    lines.push('<testsuites>');
+    lines.push(`  <testsuite name="bellwether-check" tests="${tools.length}" failures="${securityFailures}" errors="0">`);
+    for (const tool of tools) {
+        const successRate = tool.baselineSuccessRate ?? 1;
+        const status = successRate >= 0.9 ? 'passed' : 'warning';
+        lines.push(`    <testcase name="${tool.name}" classname="mcp-tools" time="0">`);
+        lines.push(`      <system-out>Success rate: ${(successRate * 100).toFixed(0)}%</system-out>`);
+        if (status === 'warning') {
+            lines.push('      <system-err>Tool has success rate below 90%</system-err>');
+        }
+        lines.push('    </testcase>');
+    }
+    const securityTools = tools.filter((t) => t.securityFingerprint?.findings?.length);
+    if (securityTools.length > 0) {
+        lines.push('    <!-- Security findings -->');
+        for (const tool of securityTools) {
+            const findings = tool.securityFingerprint?.findings ?? [];
+            const criticalHigh = findings.filter((f) => f.riskLevel === 'critical' || f.riskLevel === 'high').length;
+            if (criticalHigh > 0) {
+                lines.push(`    <testcase name="${tool.name}-security" classname="security">`);
+                lines.push(`      <failure message="${criticalHigh} critical/high security findings">`);
+                for (const finding of findings.filter((f) => f.riskLevel === 'critical' || f.riskLevel === 'high')) {
+                    lines.push(`        ${finding.riskLevel.toUpperCase()}: ${finding.title} (${finding.cweId})`);
+                }
+                lines.push('      </failure>');
+                lines.push('    </testcase>');
+            }
+        }
+    }
+    lines.push('  </testsuite>');
+    lines.push('</testsuites>');
+    return lines.join('\n');
+}
+function formatCheckResultsSarif(baseline) {
+    const tools = getToolFingerprints(baseline);
+    const serverUri = baseline.metadata?.serverCommand || baseline.server.name || 'mcp-server';
+    const results = [];
+    const securityTools = tools.filter((t) => t.securityFingerprint?.findings?.length);
+    for (const tool of securityTools) {
+        const findings = tool.securityFingerprint?.findings ?? [];
+        for (const finding of findings) {
+            const level = finding.riskLevel === 'critical' || finding.riskLevel === 'high'
+                ? 'error'
+                : finding.riskLevel === 'medium'
+                    ? 'warning'
+                    : 'note';
+            results.push({
+                ruleId: finding.cweId || 'BWH-SEC',
+                level,
+                message: { text: `[${tool.name}] ${finding.title}: ${finding.description}` },
+                locations: [
+                    {
+                        physicalLocation: {
+                            artifactLocation: { uri: serverUri },
+                            region: { startLine: 1 },
+                        },
+                    },
+                ],
+            });
+        }
+    }
+    for (const tool of tools) {
+        const successRate = tool.baselineSuccessRate ?? 1;
+        if (successRate < 0.9) {
+            results.push({
+                ruleId: 'BWH-REL',
+                level: 'warning',
+                message: {
+                    text: `Tool "${tool.name}" has ${(successRate * 100).toFixed(0)}% success rate`,
+                },
+                locations: [
+                    {
+                        physicalLocation: {
+                            artifactLocation: { uri: serverUri },
+                            region: { startLine: 1 },
+                        },
+                    },
+                ],
+            });
+        }
+    }
+    const sarif = {
+        $schema: 'https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json',
+        version: '2.1.0',
+        runs: [
+            {
+                tool: {
+                    driver: {
+                        name: 'bellwether',
+                        version: '1.0.0',
+                        informationUri: 'https://github.com/dotsetlabs/bellwether',
+                        rules: [
+                            {
+                                id: 'BWH-SEC',
+                                name: 'SecurityFinding',
+                                shortDescription: { text: 'Security vulnerability detected' },
+                                defaultConfiguration: { level: 'warning' },
+                            },
+                            {
+                                id: 'BWH-REL',
+                                name: 'LowReliability',
+                                shortDescription: { text: 'Tool reliability below threshold' },
+                                defaultConfiguration: { level: 'warning' },
+                            },
+                        ],
+                    },
+                },
+                results,
+            },
+        ],
+    };
+    return JSON.stringify(sarif, null, 2);
+}
+//# sourceMappingURL=check-formatters.js.map