@dotsetlabs/bellwether 1.0.2 → 2.0.0

package/dist/workflow/executor.js

@@ -63,7 +63,7 @@ export class WorkflowExecutor {
     emitProgress(workflow, phase, currentStep, startTime, currentStepInfo) {
         if (!this.onProgress)
             return;
-        const stepsFailed = this.stepResults.filter(r => !r.success).length;
+        const stepsFailed = this.stepResults.filter((r) => !r.success).length;
         this.onProgress({
             phase,
             workflow,
@@ -118,7 +118,9 @@ export class WorkflowExecutor {
             if (requireSuccessfulDeps) {
                 const failedDependencies = this.getFailedDependencies(step, i);
                 if (failedDependencies.length > 0) {
-                    const failedStepNames = failedDependencies.map((idx) => `step ${idx + 1} (${workflow.steps[idx]?.tool ?? 'unknown'})`).join(', ');
+                    const failedStepNames = failedDependencies
+                        .map((idx) => `step ${idx + 1} (${workflow.steps[idx]?.tool ?? 'unknown'})`)
+                        .join(', ');
                     this.logger.debug({
                         stepIndex: i,
                         tool: step.tool,
@@ -229,7 +231,7 @@ export class WorkflowExecutor {
             workflowId: workflow.id,
             success,
             stepsCompleted: this.stepResults.length,
-            stepsFailed: this.stepResults.filter(r => !r.success).length,
+            stepsFailed: this.stepResults.filter((r) => !r.success).length,
             durationMs,
         }, 'Workflow execution complete');
         done();
@@ -251,7 +253,7 @@ export class WorkflowExecutor {
     async executeStep(step, stepIndex, workflow) {
         const startTime = Date.now();
         // Verify tool exists
-        const tool = this.tools.find(t => t.name === step.tool);
+        const tool = this.tools.find((t) => t.name === step.tool);
         if (!tool) {
             return {
                 step,
@@ -284,7 +286,8 @@ export class WorkflowExecutor {
         let error;
         const stepTimeout = this.options.stepTimeout ?? DEFAULT_OPTIONS.stepTimeout;
         try {
-            response = await withTimeout(this.client.callTool(step.tool, resolvedArgs), stepTimeout, `Tool call '${step.tool}'`);
+            const abortController = new AbortController();
+            response = await withTimeout(this.client.callTool(step.tool, resolvedArgs, { signal: abortController.signal }), stepTimeout, `Tool call '${step.tool}'`, { abortController });
             if (response.isError) {
                 error = this.extractErrorMessage(response);
             }
@@ -296,7 +299,7 @@ export class WorkflowExecutor {
         const assertionResults = step.assertions
             ? this.runAssertions(step.assertions, response)
             : undefined;
-        const assertionsFailed = assertionResults?.some(r => !r.passed) ?? false;
+        const assertionsFailed = assertionResults?.some((r) => !r.passed) ?? false;
         const success = !error && !assertionsFailed;
         // Generate analysis if requested
         let analysis;
@@ -359,7 +362,7 @@ export class WorkflowExecutor {
         if (propertyPath.startsWith('result.') || propertyPath === 'result') {
             // Extract text content from the response
             const content = stepResult.response.content;
-            const textContent = content.find(c => c.type === 'text' && c.text !== undefined);
+            const textContent = content.find((c) => c.type === 'text' && c.text !== undefined);
             if (!textContent || textContent.text === undefined) {
                 throw new Error(`Step ${stepIndex} response has no text content`);
             }
@@ -418,7 +421,7 @@ export class WorkflowExecutor {
      * Run assertions against a step response.
      */
     runAssertions(assertions, response) {
-        return assertions.map(assertion => this.runAssertion(assertion, response));
+        return assertions.map((assertion) => this.runAssertion(assertion, response));
     }
     /**
      * Run a single assertion.
@@ -435,7 +438,7 @@ export class WorkflowExecutor {
         let actualValue;
         try {
             // Parse the response content as JSON
-            const textContent = response.content.find(c => c.type === 'text' && c.text !== undefined);
+            const textContent = response.content.find((c) => c.type === 'text' && c.text !== undefined);
             if (!textContent || textContent.text === undefined) {
                 throw new Error('No text content in response');
             }
@@ -484,14 +487,16 @@ export class WorkflowExecutor {
             assertion,
             passed,
             actualValue,
-            message: passed ? undefined : (assertion.message ?? `Assertion failed: ${assertion.condition}`),
+            message: passed
+                ? undefined
+                : (assertion.message ?? `Assertion failed: ${assertion.condition}`),
         };
     }
     /**
      * Extract error message from a tool response.
      */
     extractErrorMessage(response) {
-        const textContent = response.content.find(c => c.type === 'text');
+        const textContent = response.content.find((c) => c.type === 'text');
         if (textContent && 'text' in textContent) {
             return String(textContent.text);
         }
@@ -595,7 +600,7 @@ export class WorkflowExecutor {
         if (!this.llm) {
             return success
                 ? `Workflow "${workflow.name}" completed successfully with ${stepResults.length} steps.`
-                : `Workflow "${workflow.name}" failed at step ${stepResults.findIndex(r => !r.success) + 1}.`;
+                : `Workflow "${workflow.name}" failed at step ${stepResults.findIndex((r) => !r.success) + 1}.`;
         }
         const prompt = buildWorkflowSummaryPrompt({ workflow, stepResults, success });
         try {
@@ -604,7 +609,7 @@ export class WorkflowExecutor {
         catch {
             return success
                 ? `Workflow "${workflow.name}" completed successfully with ${stepResults.length} steps.`
-                : `Workflow "${workflow.name}" failed at step ${stepResults.findIndex(r => !r.success) + 1}.`;
+                : `Workflow "${workflow.name}" failed at step ${stepResults.findIndex((r) => !r.success) + 1}.`;
         }
     }
 }

package/dist/workflow/loader.js

@@ -6,8 +6,10 @@ import { join } from 'path';
 import { parseAllDocuments } from 'yaml';
 import { parseYamlSecure, YAML_SECURITY_LIMITS } from '../utils/yaml-parser.js';
 import { PATHS } from '../constants.js';
+import { getLogger } from '../logging/logger.js';
 /** Default file name for workflow definitions */
 export const DEFAULT_WORKFLOWS_FILE = PATHS.DEFAULT_WORKFLOWS_FILE;
+const logger = getLogger('workflow');
 /**
  * Load workflows from a YAML file.
  * Supports both single-document and multi-document YAML (separated by ---).
@@ -56,9 +58,10 @@ export function tryLoadDefaultWorkflows(directory) {
     try {
         return loadWorkflowsFromFile(path);
     }
-    catch {
+    catch (error) {
         // If the file exists but is invalid, return null rather than throwing
         // This allows the interview to proceed without workflows
+        logger.warn({ path, error: error instanceof Error ? error.message : String(error) }, 'Failed to load default workflow file');
         return null;
     }
 }

package/dist/workflow/state-tracker.js

@@ -74,7 +74,7 @@ export class StateTracker {
         }
         // Use specified probe tools if provided
         if (this.options.probeTools?.length) {
-            this.probeTools = this.options.probeTools.filter(name => this.tools.some(t => t.name === name));
+            this.probeTools = this.options.probeTools.filter((name) => this.tools.some((t) => t.name === name));
         }
         this.logger.debug({
             toolCount: this.tools.length,
@@ -89,9 +89,9 @@ export class StateTracker {
         const name = tool.name;
         const description = tool.description ?? '';
         const combined = `${name} ${description}`;
-        const isReader = READER_PATTERNS.some(p => p.test(combined));
-        const isWriter = WRITER_PATTERNS.some(p => p.test(combined));
-        const isProbe = PROBE_PATTERNS.some(p => p.test(combined));
+        const isReader = READER_PATTERNS.some((p) => p.test(combined));
+        const isWriter = WRITER_PATTERNS.some((p) => p.test(combined));
+        const isProbe = PROBE_PATTERNS.some((p) => p.test(combined));
         let role;
         let confidence;
         if (isReader && isWriter) {
@@ -194,8 +194,9 @@ export class StateTracker {
                 break;
             }
             try {
+                const abortController = new AbortController();
                 // Apply timeout to individual probe tool call
-                const result = await withTimeout(this.client.callTool(probeName, {}), this.probeTimeout, `Probe tool '${probeName}'`);
+                const result = await withTimeout(this.client.callTool(probeName, {}, { signal: abortController.signal }), this.probeTimeout, `Probe tool '${probeName}'`, { abortController });
                 const content = this.extractContent(result);
                 stateData[probeName] = content;
                 successCount++;
@@ -240,7 +241,7 @@ export class StateTracker {
      * Extract content from a tool call result.
      */
     extractContent(result) {
-        const textContent = result.content.find(c => c.type === 'text' && c.text !== undefined);
+        const textContent = result.content.find((c) => c.type === 'text' && c.text !== undefined);
         if (!textContent || textContent.text === undefined) {
             return null;
         }
@@ -256,7 +257,10 @@ export class StateTracker {
      */
     hashState(data) {
         const json = JSON.stringify(data, null, 0);
-        return createHash('sha256').update(json).digest('hex').slice(0, DISPLAY_LIMITS.HASH_DISPLAY_LENGTH);
+        return createHash('sha256')
+            .update(json)
+            .digest('hex')
+            .slice(0, DISPLAY_LIMITS.HASH_DISPLAY_LENGTH);
     }
     /**
      * Compare two snapshots and identify changes.
@@ -351,7 +355,7 @@ export class StateTracker {
                 for (const stateType of stateTypes) {
                     const writers = writerSteps.get(stateType) ?? [];
                     // Find most recent writer for this state type
-                    const recentWriters = writers.filter(w => w < i);
+                    const recentWriters = writers.filter((w) => w < i);
                     if (recentWriters.length > 0) {
                         const producerStep = recentWriters[recentWriters.length - 1];
                         const producerTool = stepResults[producerStep].step.tool;
@@ -372,9 +376,9 @@ export class StateTracker {
      * Verify dependencies using state snapshots.
      */
     verifyDependencies(dependencies, _snapshots, changes) {
-        return dependencies.map(dep => {
+        return dependencies.map((dep) => {
             // Check if the producer step caused any changes
-            const producerChanges = changes.filter(c => c.causedByStep === dep.producerStep);
+            const producerChanges = changes.filter((c) => c.causedByStep === dep.producerStep);
             const verified = producerChanges.length > 0;
             return {
                 ...dep,
@@ -388,19 +392,19 @@ export class StateTracker {
     async generateSummary(tracking) {
         const parts = [];
         // Summarize tool roles
-        const writers = tracking.toolRoles.filter(t => t.role === 'writer' || t.role === 'both');
-        const readers = tracking.toolRoles.filter(t => t.role === 'reader' || t.role === 'both');
+        const writers = tracking.toolRoles.filter((t) => t.role === 'writer' || t.role === 'both');
+        const readers = tracking.toolRoles.filter((t) => t.role === 'reader' || t.role === 'both');
         if (writers.length > 0) {
-            parts.push(`State writers: ${writers.map(t => t.tool).join(', ')}`);
+            parts.push(`State writers: ${writers.map((t) => t.tool).join(', ')}`);
         }
         if (readers.length > 0) {
-            parts.push(`State readers: ${readers.map(t => t.tool).join(', ')}`);
+            parts.push(`State readers: ${readers.map((t) => t.tool).join(', ')}`);
         }
         // Summarize changes
         if (tracking.changes.length > 0) {
-            const created = tracking.changes.filter(c => c.type === 'created').length;
-            const modified = tracking.changes.filter(c => c.type === 'modified').length;
-            const deleted = tracking.changes.filter(c => c.type === 'deleted').length;
+            const created = tracking.changes.filter((c) => c.type === 'created').length;
+            const modified = tracking.changes.filter((c) => c.type === 'modified').length;
+            const deleted = tracking.changes.filter((c) => c.type === 'deleted').length;
             const changeParts = [];
             if (created > 0)
                 changeParts.push(`${created} created`);
@@ -415,7 +419,7 @@ export class StateTracker {
         }
         // Summarize dependencies
         if (tracking.dependencies.length > 0) {
-            const verified = tracking.dependencies.filter(d => d.verified).length;
+            const verified = tracking.dependencies.filter((d) => d.verified).length;
             parts.push(`Dependencies: ${tracking.dependencies.length} inferred (${verified} verified)`);
         }
         return `${parts.join('. ')}.`;

package/man/bellwether.1

@@ -0,0 +1,204 @@
+.TH "BELLWETHER" "1" "2026\-02\-04" "Bellwether 2.0.0" "User Commands"
+.SH NAME
+.PP
+bellwether \[em] MCP server testing and validation tool
+.SH SYNOPSIS
+.PP
+\f[B]bellwether\f[] [OPTIONS] COMMAND [ARGS...]
+.PP
+\f[B]bellwether\f[] \f[B]\-\-version\f[]
+.PP
+\f[B]bellwether\f[] \f[B]\-\-help\f[]
+.SH DESCRIPTION
+.PP
+Bellwether is an open\-source MCP (Model Context Protocol) testing tool
+that provides structural drift detection and behavioral documentation
+for MCP servers.
+.SH COMMANDS
+.TP
+.B \f[B]check\f[] [\f[I]options\f[]] [server\-command]
+Schema validation and drift detection (free, fast, deterministic)
+.RS
+.RE
+.TP
+.B \f[B]explore\f[] [\f[I]options\f[]] [server\-command]
+LLM\-powered behavioral exploration and documentation
+.RS
+.RE
+.TP
+.B \f[B]discover\f[] [\f[I]options\f[]] [server\-command]
+Discover MCP server capabilities (tools, prompts, resources)
+.RS
+.RE
+.TP
+.B \f[B]watch\f[] [\f[I]options\f[]]
+Watch for MCP server changes and auto\-check
+.RS
+.RE
+.TP
+.B \f[B]init\f[] [\f[I]options\f[]] [server\-command]
+Initialize a bellwether.yaml configuration file
+.RS
+.RE
+.TP
+.B \f[B]auth\f[] \f[I]subcommand\f[] [\f[I]options\f[]]
+Manage LLM provider API keys
+.RS
+.RE
+.TP
+.B \f[B]baseline\f[] \f[I]subcommand\f[] [\f[I]options\f[]]
+Manage baselines for drift detection
+.RS
+.RE
+.TP
+.B \f[B]golden\f[] \f[I]subcommand\f[] [\f[I]options\f[]]
+Manage golden outputs for validation
+.RS
+.RE
+.TP
+.B \f[B]registry\f[] [\f[I]options\f[]] \f[I]search\f[]
+Search the MCP Registry for servers
+.RS
+.RE
+.TP
+.B \f[B]contract\f[] \f[I]subcommand\f[] [\f[I]options\f[]]
+Validate MCP servers against contracts
+.RS
+.RE
+.TP
+.B \f[B]validate\-config\f[] [\f[I]options\f[]]
+Validate bellwether.yaml configuration
+.RS
+.RE
+.SH GLOBAL OPTIONS
+.TP
+.B \f[B]\-h\f[], \f[B]\-\-help\f[]
+Show help message and exit
+.RS
+.RE
+.TP
+.B \f[B]\-\-version\f[]
+Show version information and exit
+.RS
+.RE
+.TP
+.B \f[B]\-\-log\-level\f[] \f[I]LEVEL\f[]
+Set log level: debug, info, warn, error, silent
+.RS
+.RE
+.TP
+.B \f[B]\-\-log\-file\f[] \f[I]PATH\f[]
+Write logs to file instead of stderr
+.RS
+.RE
+.SH EXAMPLES
+.PP
+Initialize configuration:
+.IP
+.nf
+\f[C]
+bellwether\ init\ npx\ \@modelcontextprotocol/server\-filesystem
+\f[]
+.fi
+.PP
+Run drift detection:
+.IP
+.nf
+\f[C]
+bellwether\ check
+\f[]
+.fi
+.PP
+Save baseline:
+.IP
+.nf
+\f[C]
+bellwether\ baseline\ save
+\f[]
+.fi
+.PP
+Explore with LLM:
+.IP
+.nf
+\f[C]
+bellwether\ explore
+\f[]
+.fi
+.SH FILES
+.TP
+.B \f[I]bellwether.yaml\f[]
+Configuration file for the project
+.RS
+.RE
+.TP
+.B \f[I]bellwether\-baseline.json\f[]
+Saved baseline for drift detection
+.RS
+.RE
+.TP
+.B \f[I]CONTRACT.md\f[]
+Generated contract documentation
+.RS
+.RE
+.TP
+.B \f[I]AGENTS.md\f[]
+Generated behavioral documentation
+.RS
+.RE
+.SH ENVIRONMENT
+.TP
+.B \f[I]OPENAI_API_KEY\f[]
+API key for OpenAI (explore mode only)
+.RS
+.RE
+.TP
+.B \f[I]ANTHROPIC_API_KEY\f[]
+API key for Anthropic (explore mode only)
+.RS
+.RE
+.TP
+.B \f[I]OLLAMA_BASE_URL\f[]
+Ollama URL (default: http://localhost:11434)
+.RS
+.RE
+.SH EXIT STATUS
+.TP
+.B \f[B]0\f[]
+Success, no changes detected
+.RS
+.RE
+.TP
+.B \f[B]1\f[]
+Info\-level changes only
+.RS
+.RE
+.TP
+.B \f[B]2\f[]
+Warning\-level changes
+.RS
+.RE
+.TP
+.B \f[B]3\f[]
+Breaking changes detected
+.RS
+.RE
+.TP
+.B \f[B]4\f[]
+Runtime error
+.RS
+.RE
+.TP
+.B \f[B]5\f[]
+Low confidence metrics
+.RS
+.RE
+.SH SEE ALSO
+.PP
+Project homepage: <https://github.com/dotsetlabs/bellwether>
+.PP
+Documentation: <https://docs.bellwether.sh>
+.PP
+MCP Specification: <https://modelcontextprotocol.io>
+.SH AUTHORS
+.PP
+Dotset Labs LLC <hello@dotsetlabs.com>

package/man/bellwether.1.md

@@ -0,0 +1,148 @@
+---
+title: BELLWETHER
+section: 1
+header: User Commands
+footer: Bellwether 2.0.0
+date: 2026-02-04
+---
+
+# NAME
+
+bellwether — MCP server testing and validation tool
+
+# SYNOPSIS
+
+**bellwether** [OPTIONS] COMMAND [ARGS...]
+
+**bellwether** **--version**
+
+**bellwether** **--help**
+
+# DESCRIPTION
+
+Bellwether is an open-source MCP (Model Context Protocol) testing tool that provides
+structural drift detection and behavioral documentation for MCP servers.
+
+# COMMANDS
+
+**check** [*options*] [server-command]
+:   Schema validation and drift detection (free, fast, deterministic)
+
+**explore** [*options*] [server-command]
+:   LLM-powered behavioral exploration and documentation
+
+**discover** [*options*] [server-command]
+:   Discover MCP server capabilities (tools, prompts, resources)
+
+**watch** [*options*]
+:   Watch for MCP server changes and auto-check
+
+**init** [*options*] [server-command]
+:   Initialize a bellwether.yaml configuration file
+
+**auth** *subcommand* [*options*]
+:   Manage LLM provider API keys
+
+**baseline** *subcommand* [*options*]
+:   Manage baselines for drift detection
+
+**golden** *subcommand* [*options*]
+:   Manage golden outputs for validation
+
+**registry** [*options*] *search*
+:   Search the MCP Registry for servers
+
+**contract** *subcommand* [*options*]
+:   Validate MCP servers against contracts
+
+**validate-config** [*options*]
+:   Validate bellwether.yaml configuration
+
+# GLOBAL OPTIONS
+
+**-h**, **--help**
+:   Show help message and exit
+
+**--version**
+:   Show version information and exit
+
+**--log-level** *LEVEL*
+:   Set log level: debug, info, warn, error, silent
+
+**--log-file** *PATH*
+:   Write logs to file instead of stderr
+
+# EXAMPLES
+
+Initialize configuration:
+
+    bellwether init npx @modelcontextprotocol/server-filesystem
+
+Run drift detection:
+
+    bellwether check
+
+Save baseline:
+
+    bellwether baseline save
+
+Explore with LLM:
+
+    bellwether explore
+
+# FILES
+
+*bellwether.yaml*
+:   Configuration file for the project
+
+*bellwether-baseline.json*
+:   Saved baseline for drift detection
+
+*CONTRACT.md*
+:   Generated contract documentation
+
+*AGENTS.md*
+:   Generated behavioral documentation
+
+# ENVIRONMENT
+
+*OPENAI_API_KEY*
+:   API key for OpenAI (explore mode only)
+
+*ANTHROPIC_API_KEY*
+:   API key for Anthropic (explore mode only)
+
+*OLLAMA_BASE_URL*
+:   Ollama URL (default: http://localhost:11434)
+
+# EXIT STATUS
+
+**0**
+:   Success, no changes detected
+
+**1**
+:   Info-level changes only
+
+**2**
+:   Warning-level changes
+
+**3**
+:   Breaking changes detected
+
+**4**
+:   Runtime error
+
+**5**
+:   Low confidence metrics
+
+# SEE ALSO
+
+Project homepage: <https://github.com/dotsetlabs/bellwether>
+
+Documentation: <https://docs.bellwether.sh>
+
+MCP Specification: <https://modelcontextprotocol.io>
+
+# AUTHORS
+
+Dotset Labs LLC <hello@dotsetlabs.com>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dotsetlabs/bellwether",
-  "version": "1.0.2",
+  "version": "2.0.0",
   "description": "The open-source MCP testing tool. Structural drift detection and behavioral documentation for Model Context Protocol servers.",
   "type": "module",
   "main": "dist/index.js",
@@ -33,10 +33,11 @@
     "format": "prettier --write \"src/**/*.ts\"",
     "format:check": "prettier --check \"src/**/*.ts\"",
     "clean": "rm -rf dist",
-    "docs:generate": "typedoc",
+    "docs:generate": "npm --prefix website run build",
+    "docs:dev": "npm --prefix website run start",
     "man:generate": "./scripts/generate-manpage.sh",
     "prepare": "husky install || true",
-    "prepublishOnly": "npm run build"
+    "prepublishOnly": "npm run build && npm run man:generate"
   },
   "keywords": [
     "mcp",
@@ -59,8 +60,7 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/dotsetlabs/bellwether",
-    "directory": "cli"
+    "url": "https://github.com/dotsetlabs/bellwether"
   },
   "funding": {
     "type": "github",
@@ -97,10 +97,9 @@
     "@typescript-eslint/parser": "^6.21.0",
     "eslint": "^8.57.1",
     "husky": "^9.1.0",
-    "lint-staged": "^15.2.0",
+    "lint-staged": "^16.2.7",
     "prettier": "^3.3.0",
     "tsx": "^4.21.0",
-    "typedoc": "^0.28.16",
     "typescript": "^5.3.0",
     "vitest": "^4.0.17"
   },