@dotsetlabs/bellwether 1.0.2 → 1.0.3

package/CHANGELOG.md

@@ -7,6 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.0.3] - 2026-02-02
+
+### Added
+
+- Added `version` input to GitHub Action for explicit npm version selection
+  - Action now derives version from ref (e.g., `v1.0.3`) or accepts explicit `inputs.version`
+  - Provides clear error message when version cannot be determined
+- Added `signal` option to LLM completion requests for request cancellation via AbortSignal
+- Added AbortController integration to timeout utilities for proper request cancellation
+- Added JSON extraction from mixed LLM responses (handles prose around JSON blocks)
+
+### Changed
+
+- Improved timeout handling with AbortController propagation across LLM and transport layers
+- Improved error handling and resource cleanup in interview, orchestrator, and transport modules
+- Refactored response cache, workflow executor, and state tracker for better reliability
+- Updated CI/CD and GitHub/GitLab integration documentation
+
+### Fixed
+
+- Fixed GitHub Action stderr handling in check command output capture
+- Fixed various code formatting and linting issues across LLM clients and transport modules
+
 ## [1.0.2] - 2026-01-30
 
 ### Added

package/README.md

@@ -124,8 +124,9 @@ Requires LLM (Ollama for free local, or OpenAI/Anthropic). Generates `AGENTS.md`
 ## GitHub Action
 
 ```yaml
-- uses: dotsetlabs/bellwether@v1
+- uses: dotsetlabs/bellwether@v1.0.2
   with:
+    version: '1.0.2'
     server-command: 'npx @mcp/your-server'
     baseline-path: './bellwether-baseline.json'
     fail-on-severity: 'warning'
@@ -167,7 +168,7 @@ bellwether init --preset local npx @mcp/server # Local Ollama (free)
 
 ```bash
 git clone https://github.com/dotsetlabs/bellwether
-cd bellwether/cli
+cd bellwether
 npm install
 npm run build
 npm test

package/dist/cache/response-cache.d.ts

@@ -10,6 +10,8 @@ export interface CacheEntry<T> {
     value: T;
     /** When the entry was created */
     createdAt: Date;
+    /** When the entry was last accessed */
+    lastAccessedAt: Date;
     /** When the entry expires */
     expiresAt: Date;
     /** Cache key (hash) */
@@ -99,9 +101,9 @@ export declare class ResponseCache {
      */
     private evictIfNeeded;
     /**
-     * Evict the oldest entry (LRU based on creation time).
+     * Evict the least recently used entry (LRU based on last access time).
      */
-    private evictOldest;
+    private evictLeastRecentlyUsed;
     /**
      * Estimate the size of a value in bytes.
      */

package/dist/cache/response-cache.js

@@ -30,21 +30,9 @@ export class ResponseCache {
      * Generate a cache key from input data.
      */
     generateKey(...parts) {
-        const serialized = parts.map((p) => {
-            if (typeof p === 'string')
-                return p;
-            if (typeof p === 'undefined')
-                return 'undefined';
-            if (p === null)
-                return 'null';
-            try {
-                return JSON.stringify(p, Object.keys(p).sort());
-            }
-            catch {
-                return String(p);
-            }
-        }).join('|');
-        return createHash('sha256').update(serialized).digest('hex').slice(0, 16);
+        const serialized = parts.map((p) => stableStringify(p)).join('|');
+        // Use 128-bit hash (32 hex chars) to reduce collision risk.
+        return createHash('sha256').update(serialized).digest('hex').slice(0, 32);
     }
     /**
      * Get an entry from cache.
@@ -66,6 +54,7 @@ export class ResponseCache {
             return undefined;
         }
         entry.hitCount++;
+        entry.lastAccessedAt = new Date();
         this.stats.hits++;
         logger.debug({ key, hitCount: entry.hitCount }, 'Cache hit');
         return entry.value;
@@ -86,6 +75,7 @@ export class ResponseCache {
         const entry = {
             value,
             createdAt: now,
+            lastAccessedAt: now,
             expiresAt: new Date(now.getTime() + ttl),
             key,
             description: options?.description,
@@ -168,31 +158,30 @@ export class ResponseCache {
     evictIfNeeded(newEntrySize) {
         // Check entry count
         while (this.cache.size >= this.config.maxEntries) {
-            this.evictOldest();
+            this.evictLeastRecentlyUsed();
         }
         // Check size
-        while (this.totalSizeBytes + newEntrySize > this.config.maxSizeBytes &&
-            this.cache.size > 0) {
-            this.evictOldest();
+        while (this.totalSizeBytes + newEntrySize > this.config.maxSizeBytes && this.cache.size > 0) {
+            this.evictLeastRecentlyUsed();
         }
     }
     /**
-     * Evict the oldest entry (LRU based on creation time).
+     * Evict the least recently used entry (LRU based on last access time).
      */
-    evictOldest() {
-        let oldestKey;
-        let oldestTime = Infinity;
+    evictLeastRecentlyUsed() {
+        let lruKey;
+        let oldestAccessTime = Infinity;
         for (const [key, entry] of this.cache) {
-            const time = entry.createdAt.getTime();
-            if (time < oldestTime) {
-                oldestTime = time;
-                oldestKey = key;
+            const time = entry.lastAccessedAt.getTime();
+            if (time < oldestAccessTime) {
+                oldestAccessTime = time;
+                lruKey = key;
             }
         }
-        if (oldestKey) {
-            this.delete(oldestKey);
+        if (lruKey) {
+            this.delete(lruKey);
             this.stats.evictions++;
-            logger.debug({ key: oldestKey }, 'Evicted cache entry');
+            logger.debug({ key: lruKey }, 'Evicted cache entry');
         }
     }
     /**
@@ -207,6 +196,55 @@ export class ResponseCache {
         }
     }
 }
+/**
+ * Stable, deterministic JSON stringify with deep key sorting.
+ * Falls back to string conversion for unsupported types.
+ */
+function stableStringify(value) {
+    const seen = new WeakSet();
+    const normalize = (input) => {
+        if (input === null || input === undefined)
+            return input;
+        const type = typeof input;
+        if (type === 'string' || type === 'number' || type === 'boolean') {
+            return input;
+        }
+        if (type === 'bigint') {
+            return input.toString();
+        }
+        if (type === 'symbol' || type === 'function') {
+            return String(input);
+        }
+        if (input instanceof Date) {
+            return input.toISOString();
+        }
+        if (Array.isArray(input)) {
+            return input.map((item) => normalize(item));
+        }
+        if (typeof input === 'object') {
+            const obj = input;
+            if (seen.has(obj)) {
+                return '[Circular]';
+            }
+            seen.add(obj);
+            const keys = Object.keys(obj).sort();
+            const normalized = {};
+            for (const key of keys) {
+                normalized[key] = normalize(obj[key]);
+            }
+            return normalized;
+        }
+        try {
+            return JSON.parse(JSON.stringify(input));
+        }
+        catch {
+            return String(input);
+        }
+    };
+    const normalized = normalize(value);
+    const json = JSON.stringify(normalized);
+    return json === undefined ? 'undefined' : json;
+}
 /**
  * Specialized cache for tool responses.
  */

package/dist/cli/commands/check.js

@@ -13,7 +13,7 @@ import { MCPClient } from '../../transport/mcp-client.js';
 import { discover } from '../../discovery/discovery.js';
 import { Interviewer } from '../../interview/interviewer.js';
 import { generateContractMd, generateJsonReport } from '../../docs/generator.js';
-import { loadConfig, ConfigNotFoundError, parseCommandString } from '../../config/loader.js';
+import { loadConfig, ConfigNotFoundError, parseCommandString, } from '../../config/loader.js';
 import { validateConfigForCheck, getConfigWarnings } from '../../config/validator.js';
 import { createBaseline, loadBaseline, saveBaseline, getToolFingerprints, toToolCapability, compareBaselines, acceptDrift, formatDiffText, formatDiffJson, formatDiffCompact, formatDiffGitHubActions, formatDiffMarkdown, formatDiffJUnit, formatDiffSarif, applySeverityConfig, shouldFailOnDiff, analyzeForIncremental, formatIncrementalSummary, runSecurityTests, parseSecurityCategories, getAllSecurityCategories, } from '../../baseline/index.js';
 import { convertAssertions } from '../../baseline/converter.js';
@@ -21,7 +21,7 @@ import { getMetricsCollector, resetMetricsCollector } from '../../metrics/collec
 import { getGlobalCache, resetGlobalCache } from '../../cache/response-cache.js';
 import { InterviewProgressBar, formatCheckBanner } from '../utils/progress.js';
 import { buildCheckSummary, colorizeConfidence, formatConfidenceLevel, formatToolResultLine, } from '../output/terminal-reporter.js';
-import { loadScenariosFromFile, tryLoadDefaultScenarios, DEFAULT_SCENARIOS_FILE } from '../../scenarios/index.js';
+import { loadScenariosFromFile, tryLoadDefaultScenarios, DEFAULT_SCENARIOS_FILE, } from '../../scenarios/index.js';
 import { loadWorkflowsFromFile, tryLoadDefaultWorkflows, DEFAULT_WORKFLOWS_FILE, WorkflowExecutor, generateWorkflowsFromTools, generateWorkflowYamlContent, } from '../../workflow/index.js';
 import * as output from '../output.js';
 import { extractServerContextFromArgs } from '../utils/server-context.js';
@@ -73,14 +73,6 @@ export const checkCommand = new Command('check')
         output.error(error instanceof Error ? error.message : String(error));
         process.exit(EXIT_CODES.ERROR);
     }
-    const warnings = getConfigWarnings(config);
-    if (warnings.length > 0) {
-        output.warn('Configuration warnings:');
-        for (const warning of warnings) {
-            output.warn(`  - ${warning}`);
-        }
-        output.newline();
-    }
     // Extract settings from config
     const timeout = config.server.timeout;
     const outputDir = config.output.dir;
@@ -105,7 +97,8 @@ export const checkCommand = new Command('check')
         minimumSeverity: options.minSeverity ?? config.baseline.severity.minimumSeverity,
         failOnSeverity: options.failOnSeverity ?? config.baseline.severity.failOnSeverity,
         suppressWarnings: config.baseline.severity.suppressWarnings,
-        aspectOverrides: config.baseline.severity.aspectOverrides,
+        aspectOverrides: config.baseline.severity
+            .aspectOverrides,
     };
     // Resolve check options from config (no CLI overrides for these)
     const incrementalEnabled = config.check.incremental;
@@ -114,9 +107,26 @@ export const checkCommand = new Command('check')
     const parallelWorkers = config.check.parallelWorkers;
     const performanceThreshold = config.check.performanceThreshold / PERCENTAGE_CONVERSION.DIVISOR;
     const diffFormat = options.format ?? config.check.diffFormat;
+    const machineReadableFormats = new Set(['json', 'junit', 'sarif']);
+    const machineReadable = machineReadableFormats.has(String(diffFormat).toLowerCase());
+    if (machineReadable) {
+        // Suppress standard CLI output to keep stdout clean for machine-readable formats.
+        output.configureOutput({ quiet: true });
+    }
+    const warnings = getConfigWarnings(config);
+    if (warnings.length > 0) {
+        output.warn('Configuration warnings:');
+        for (const warning of warnings) {
+            output.warn(`  - ${warning}`);
+        }
+        if (!machineReadable) {
+            output.newline();
+        }
+    }
     // Resolve security options from config
     const securityEnabled = config.check.security.enabled;
-    let securityCategories = config.check.security.categories;
+    let securityCategories = config.check.security
+        .categories;
     // Validate security categories
     try {
         securityCategories = parseSecurityCategories(securityCategories.join(','));
@@ -141,13 +151,15 @@ export const checkCommand = new Command('check')
         ? `${serverCommand} ${args.join(' ')}`.trim()
         : (remoteUrl ?? 'unknown');
     // Display startup banner
-    const banner = formatCheckBanner({
-        serverCommand: serverIdentifier,
-    });
-    output.info(banner);
-    output.newline();
-    output.info('Check: Schema validation and drift detection (free, deterministic)');
-    output.newline();
+    if (!machineReadable) {
+        const banner = formatCheckBanner({
+            serverCommand: serverIdentifier,
+        });
+        output.info(banner);
+        output.newline();
+        output.info('Check: Schema validation and drift detection (free, deterministic)');
+        output.newline();
+    }
     // Initialize metrics collector
     resetMetricsCollector();
     const metricsCollector = getMetricsCollector();
@@ -182,9 +194,12 @@ export const checkCommand = new Command('check')
         }
         // Discovery phase
         output.info('Discovering capabilities...');
-        const discovery = await discover(mcpClient, transport === 'stdio' ? serverCommand : remoteUrl ?? serverCommand, transport === 'stdio' ? args : []);
+        const discovery = await discover(mcpClient, transport === 'stdio' ? serverCommand : (remoteUrl ?? serverCommand), transport === 'stdio' ? args : []);
         const resourceCount = discovery.resources?.length ?? 0;
-        const discoveryParts = [`${discovery.tools.length} tools`, `${discovery.prompts.length} prompts`];
+        const discoveryParts = [
+            `${discovery.tools.length} tools`,
+            `${discovery.prompts.length} prompts`,
+        ];
         if (resourceCount > 0) {
             discoveryParts.push(`${resourceCount} resources`);
         }
@@ -228,7 +243,9 @@ export const checkCommand = new Command('check')
             }
             else {
                 incrementalBaseline = loadBaseline(baselinePath);
-                const result = analyzeForIncremental(discovery.tools, incrementalBaseline, { maxCacheAgeHours: incrementalCacheHours });
+                const result = analyzeForIncremental(discovery.tools, incrementalBaseline, {
+                    maxCacheAgeHours: incrementalCacheHours,
+                });
                 incrementalResult = result;
                 const summary = formatIncrementalSummary(result.changeSummary);
                 output.info(`Incremental analysis: ${summary}`);
@@ -240,7 +257,7 @@ export const checkCommand = new Command('check')
                 else {
                     output.info(`Testing ${result.toolsToTest.length} tools (${result.toolsToSkip.length} cached)\n`);
                     // Filter discovery to only include tools that need testing
-                    discovery.tools = discovery.tools.filter(t => result.toolsToTest.includes(t.name));
+                    discovery.tools = discovery.tools.filter((t) => result.toolsToTest.includes(t.name));
                 }
             }
         }
@@ -323,7 +340,7 @@ export const checkCommand = new Command('check')
             interviewer.setServerContext(serverContext);
         }
         // Set up progress display
-        const progressBar = new InterviewProgressBar({ enabled: !verbose });
+        const progressBar = new InterviewProgressBar({ enabled: !verbose && !machineReadable });
         const reportedTools = new Set();
         const progressCallback = (progress) => {
             if (verbose) {
@@ -462,7 +479,7 @@ export const checkCommand = new Command('check')
                         try {
                             const response = await mcpClient.callTool(tool.name, args);
                             const content = response.content
-                                .map((c) => c.type === 'text' ? c.text : '')
+                                .map((c) => (c.type === 'text' ? c.text : ''))
                                 .join('\n');
                             return {
                                 isError: response.isError ?? false,
@@ -580,7 +597,7 @@ export const checkCommand = new Command('check')
                     const workflowResult = await workflowExecutor.execute(workflow);
                     workflowResults.push(workflowResult);
                     const statusIcon = workflowResult.success ? '\u2713' : '\u2717';
-                    const stepsInfo = `${workflowResult.steps.filter(s => s.success).length}/${workflow.steps.length} steps`;
+                    const stepsInfo = `${workflowResult.steps.filter((s) => s.success).length}/${workflow.steps.length} steps`;
                     if (workflowResult.success) {
                         output.success(`  ${statusIcon} ${workflow.name} (${stepsInfo}) - ${workflowResult.durationMs}ms`);
                     }
@@ -599,7 +616,7 @@ export const checkCommand = new Command('check')
                 }
             }
             // Workflow summary
-            const passed = workflowResults.filter(r => r.success).length;
+            const passed = workflowResults.filter((r) => r.success).length;
             const failed = workflowResults.length - passed;
             output.newline();
             if (failed === 0) {
@@ -631,9 +648,7 @@ export const checkCommand = new Command('check')
         }
         if (writeJson) {
             // Add workflow results to the result object for the JSON report
-            const resultWithWorkflows = workflowResults.length > 0
-                ? { ...result, workflowResults }
-                : result;
+            const resultWithWorkflows = workflowResults.length > 0 ? { ...result, workflowResults } : result;
             let jsonReport;
             try {
                 jsonReport = generateJsonReport(resultWithWorkflows, {
@@ -671,10 +686,7 @@ export const checkCommand = new Command('check')
         if (incrementalResult && incrementalResult.cachedFingerprints.length > 0) {
             // Merge new fingerprints with cached ones
             const cachedTools = incrementalResult.cachedFingerprints.map(toToolCapability);
-            const mergedTools = [
-                ...currentBaseline.capabilities.tools,
-                ...cachedTools,
-            ].sort((a, b) => a.name.localeCompare(b.name));
+            const mergedTools = [...currentBaseline.capabilities.tools, ...cachedTools].sort((a, b) => a.name.localeCompare(b.name));
             currentBaseline = {
                 ...currentBaseline,
                 capabilities: {
@@ -773,7 +785,9 @@ export const checkCommand = new Command('check')
         if (!baselinePath) {
             const formattedCheckResults = formatCheckResults(currentBaseline, diffFormat);
             if (formattedCheckResults) {
-                output.info('\n--- Check Results ---');
+                if (!machineReadable) {
+                    output.info('\n--- Check Results ---');
+                }
                 // Output directly to stdout for machine-readable formats
                 console.log(formattedCheckResults);
             }
@@ -790,10 +804,17 @@ export const checkCommand = new Command('check')
             });
             // Apply severity configuration (filtering, overrides)
             const diff = applySeverityConfig(rawDiff, severityConfig);
-            output.info('\n--- Drift Report ---');
+            if (!machineReadable) {
+                output.info('\n--- Drift Report ---');
+            }
             // Select formatter based on --format option
             const formattedDiff = formatDiff(diff, diffFormat, baselinePath);
-            output.info(formattedDiff);
+            if (machineReadable) {
+                console.log(formattedDiff);
+            }
+            else {
+                output.info(formattedDiff);
+            }
             // Report performance regressions if detected
             if (diff.performanceReport?.hasRegressions) {
                 output.warn('\n--- Performance Regressions ---');
@@ -936,7 +957,7 @@ function formatDiff(diff, format, baselinePath) {
 function formatCheckResultsJUnit(baseline) {
     const tools = getToolFingerprints(baseline);
     const lines = [];
-    const securityFailures = tools.filter(t => t.securityFingerprint?.findings?.some(f => f.riskLevel === 'critical' || f.riskLevel === 'high')).length;
+    const securityFailures = tools.filter((t) => t.securityFingerprint?.findings?.some((f) => f.riskLevel === 'critical' || f.riskLevel === 'high')).length;
     lines.push('<?xml version="1.0" encoding="UTF-8"?>');
     lines.push('<testsuites>');
     lines.push(`  <testsuite name="bellwether-check" tests="${tools.length}" failures="${securityFailures}" errors="0">`);
@@ -951,16 +972,16 @@ function formatCheckResultsJUnit(baseline) {
         lines.push('    </testcase>');
     }
     // Add security findings as test cases if present
-    const securityTools = tools.filter(t => t.securityFingerprint?.findings?.length);
+    const securityTools = tools.filter((t) => t.securityFingerprint?.findings?.length);
     if (securityTools.length > 0) {
         lines.push(`    <!-- Security findings -->`);
         for (const tool of securityTools) {
             const findings = tool.securityFingerprint?.findings ?? [];
-            const criticalHigh = findings.filter(f => f.riskLevel === 'critical' || f.riskLevel === 'high').length;
+            const criticalHigh = findings.filter((f) => f.riskLevel === 'critical' || f.riskLevel === 'high').length;
             if (criticalHigh > 0) {
                 lines.push(`    <testcase name="${tool.name}-security" classname="security">`);
                 lines.push(`      <failure message="${criticalHigh} critical/high security findings">`);
-                for (const finding of findings.filter(f => f.riskLevel === 'critical' || f.riskLevel === 'high')) {
+                for (const finding of findings.filter((f) => f.riskLevel === 'critical' || f.riskLevel === 'high')) {
                     lines.push(`        ${finding.riskLevel.toUpperCase()}: ${finding.title} (${finding.cweId})`);
                 }
                 lines.push(`      </failure>`);
@@ -981,7 +1002,7 @@ function formatCheckResultsSarif(baseline) {
     const serverUri = baseline.metadata?.serverCommand || baseline.server.name || 'mcp-server';
     const results = [];
     // Add results for tools with security findings
-    const securityTools = tools.filter(t => t.securityFingerprint?.findings?.length);
+    const securityTools = tools.filter((t) => t.securityFingerprint?.findings?.length);
     for (const tool of securityTools) {
         const findings = tool.securityFingerprint?.findings ?? [];
         for (const finding of findings) {
@@ -994,12 +1015,14 @@ function formatCheckResultsSarif(baseline) {
                 ruleId: finding.cweId || 'BWH-SEC',
                 level,
                 message: { text: `[${tool.name}] ${finding.title}: ${finding.description}` },
-                locations: [{
+                locations: [
+                    {
                         physicalLocation: {
                             artifactLocation: { uri: serverUri },
                             region: { startLine: 1 },
                         },
-                    }],
+                    },
+                ],
             });
         }
     }
@@ -1010,20 +1033,25 @@ function formatCheckResultsSarif(baseline) {
             results.push({
                 ruleId: 'BWH-REL',
                 level: 'warning',
-                message: { text: `Tool "${tool.name}" has ${(successRate * 100).toFixed(0)}% success rate` },
-                locations: [{
+                message: {
+                    text: `Tool "${tool.name}" has ${(successRate * 100).toFixed(0)}% success rate`,
+                },
+                locations: [
+                    {
                         physicalLocation: {
                             artifactLocation: { uri: serverUri },
                             region: { startLine: 1 },
                         },
-                    }],
+                    },
+                ],
             });
         }
     }
     const sarif = {
         $schema: 'https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json',
         version: '2.1.0',
-        runs: [{
+        runs: [
+            {
                 tool: {
                     driver: {
                         name: 'bellwether',
@@ -1046,7 +1074,8 @@ function formatCheckResultsSarif(baseline) {
                     },
                 },
                 results,
-            }],
+            },
+        ],
     };
     return JSON.stringify(sarif, null, 2);
 }

package/dist/cli/index.js

@@ -16,7 +16,7 @@ if (existsSync(globalEnvPath)) {
     config({ path: globalEnvPath, quiet: true });
 }
 // Then load project .env (overrides global settings)
-config({ quiet: true });
+config({ quiet: true, override: true });
 function normalizeEncryptedEnvVar(key) {
     const value = process.env[key];
     if (!value || !isEncryptedEnvValue(value)) {
@@ -167,9 +167,11 @@ program.configureHelp({
     subcommandTerm: (cmd) => `${cmd.name()} ${cmd.usage()}`,
 });
 // Load keychain credentials, then parse commands
-loadKeychainCredentials().then(() => {
+loadKeychainCredentials()
+    .then(() => {
     program.parse();
-}).catch(() => {
+})
+    .catch(() => {
     // If keychain loading fails, still parse commands
     program.parse();
 });