@dotsetlabs/bellwether 0.11.0 → 0.12.0

package/CHANGELOG.md

@@ -2,6 +2,47 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.12.0] - 2026-01-26
+
+### Features
+
+- **Streamable HTTP transport improvements**: Full compliance with [MCP Streamable HTTP specification](https://modelcontextprotocol.io/specification/2025-03-26/basic/transports)
+  - Fixed Accept header to include both `application/json` and `text/event-stream` as required by spec
+  - Added automatic session ID capture from `Mcp-Session-Id` response header
+  - Session ID is automatically included in all subsequent requests after initialization
+  - Changed header name from `X-Session-Id` to `Mcp-Session-Id` per MCP specification
+- **False positive reduction**: Intelligent pattern detection to reduce false positives in automated testing
+  - **Operation-based tool detection**: Tools with `operation` enum + `args` object patterns now use flexible `either` outcome
+  - **Self-stateful tool detection**: Tools requiring prior state (session/chain/context) are handled appropriately
+  - **Complex array schema detection**: Arrays with nested objects containing required properties use flexible validation
+  - **Flexible semantic validation**: Semantic type tests now use `either` outcome by default, allowing tools to accept varied formats (e.g., dayjs, date-fns)
+- **Pattern detection metadata**: Test metadata now includes detection flags for transparency
+  - `operationBased`, `operationParam`, `argsParam` for operation-based tools
+  - `selfStateful`, `selfStatefulReason` for stateful tools
+  - `hasComplexArrays`, `complexArrayParams` for complex schema tools
+
+### Configuration
+
+- **New semantic validation option**: `check.flexibleSemanticTests` (default: `true`)
+  - When `true`, semantic validation tests use `either` outcome
+  - Set to `false` for strict format enforcement
+
+### Documentation
+
+- Updated remote-servers guide with correct streamable-http protocol details
+- Added MCP specification link for transport documentation
+- Clarified session ID behavior and Accept header requirements
+
+### Fixes
+
+- **Streamable HTTP session management**: Fixed session ID header to use MCP-compliant `Mcp-Session-Id`
+- **False positive tests**: Tests for operation-based, self-stateful, and complex array patterns no longer fail incorrectly
+
+### Tests
+
+- Added 17 HTTP transport tests including session ID capture verification
+- Added 11 new pattern detection tests for false positive reduction
+
 ## [0.11.0] - 2026-01-26
 
 ### Breaking Changes

package/README.md

@@ -245,6 +245,26 @@ bellwether golden save --tool my_tool --args '{"id":"123"}'
 bellwether golden compare
 ```
 
+### Server Command Options
+
+Server commands can be specified in several ways:
+
+```bash
+# As separate command and args
+bellwether check npx @mcp/server
+
+# With flags for the server command (flags pass through automatically)
+bellwether check npx -y @mcp/server
+
+# In config as a command string (auto-parsed)
+# bellwether.yaml:
+#   server:
+#     command: "npx -y @mcp/server"
+```
+
+For complex commands with flags, the CLI automatically parses command strings with spaces
+and handles server flags like `-y` correctly without requiring `--` separators.
+
 ### Baseline Commands
 
 ```bash
@@ -674,14 +694,14 @@ Use with: `bellwether init --preset <name> npx @mcp/server`
 
 ```yaml
 - name: Detect Behavioral Drift
-  uses: dotsetlabs/bellwether/action@v1
+  uses: dotsetlabs/bellwether@v1
   with:
     server-command: 'npx @mcp/your-server'
     baseline-path: './bellwether-baseline.json'
     fail-on-severity: 'warning'
 ```
 
-See [action/README.md](./action/README.md) for full documentation.
+See the [CI/CD Integration guide](https://bellwether.sh/guides/ci-cd) for full documentation.
 
 ## Environment Variables
 

package/dist/cli/commands/check.js

@@ -13,7 +13,7 @@ import { MCPClient } from '../../transport/mcp-client.js';
 import { discover } from '../../discovery/discovery.js';
 import { Interviewer } from '../../interview/interviewer.js';
 import { generateContractMd, generateJsonReport } from '../../docs/generator.js';
-import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
+import { loadConfig, ConfigNotFoundError, parseCommandString } from '../../config/loader.js';
 import { validateConfigForCheck, getConfigWarnings } from '../../config/validator.js';
 import { createBaseline, loadBaseline, saveBaseline, getToolFingerprints, toToolCapability, compareBaselines, acceptDrift, formatDiffText, formatDiffJson, formatDiffCompact, formatDiffGitHubActions, formatDiffMarkdown, formatDiffJUnit, formatDiffSarif, applySeverityConfig, shouldFailOnDiff, analyzeForIncremental, formatIncrementalSummary, runSecurityTests, parseSecurityCategories, getAllSecurityCategories, } from '../../baseline/index.js';
 import { convertAssertions } from '../../baseline/converter.js';
@@ -29,6 +29,7 @@ import { configureLogger } from '../../logging/logger.js';
 import { EXIT_CODES, SEVERITY_TO_EXIT_CODE, PATHS, SECURITY_TESTING, CHECK_SAMPLING, WORKFLOW, REPORT_SCHEMAS, } from '../../constants.js';
 export const checkCommand = new Command('check')
     .description('Check MCP server schema and detect drift (free, fast, deterministic)')
+    .allowUnknownOption() // Allow server flags like -y for npx to pass through
     .argument('[server-command]', 'Server command (overrides config)')
     .argument('[args...]', 'Server arguments')
     .option('-c, --config <path>', 'Path to config file', PATHS.DEFAULT_CONFIG_FILENAME)
@@ -52,8 +53,15 @@ export const checkCommand = new Command('check')
         throw error;
     }
     // Determine server command (CLI arg overrides config)
-    const serverCommand = serverCommandArg || config.server.command;
-    const args = serverArgs.length > 0 ? serverArgs : config.server.args;
+    // If command string contains spaces and no separate args, parse it
+    let serverCommand = serverCommandArg || config.server.command;
+    let args = serverArgs.length > 0 ? serverArgs : config.server.args;
+    // Handle command strings like "npx @package" in config when args is empty
+    if (!serverCommandArg && args.length === 0 && serverCommand.includes(' ')) {
+        const parsed = parseCommandString(serverCommand);
+        serverCommand = parsed.command;
+        args = parsed.args;
+    }
     const transport = config.server.transport ?? 'stdio';
     const remoteUrl = config.server.url?.trim();
     const remoteSessionId = config.server.sessionId?.trim();

package/dist/cli/commands/discover.js

@@ -92,6 +92,7 @@ async function discoverAction(command, args, options) {
 }
 export const discoverCommand = new Command('discover')
     .description('Discover MCP server capabilities (tools, prompts, resources)')
+    .allowUnknownOption() // Allow server flags like -y for npx to pass through
     .argument('[command]', 'Command to start the MCP server (not required for remote)')
     .argument('[args...]', 'Arguments to pass to the server')
     .option('-c, --config <path>', 'Path to config file')

package/dist/cli/commands/explore.js

@@ -13,7 +13,7 @@ import { MCPClient } from '../../transport/mcp-client.js';
 import { discover } from '../../discovery/discovery.js';
 import { Interviewer } from '../../interview/interviewer.js';
 import { generateAgentsMd, generateJsonReport } from '../../docs/generator.js';
-import { loadConfig, ConfigNotFoundError } from '../../config/loader.js';
+import { loadConfig, ConfigNotFoundError, parseCommandString } from '../../config/loader.js';
 import { validateConfigForExplore } from '../../config/validator.js';
 import { CostTracker, estimateInterviewCost, estimateInterviewTime, formatCostAndTimeEstimate, suggestOptimizations, formatOptimizationSuggestions, } from '../../cost/index.js';
 import { getMetricsCollector, resetMetricsCollector } from '../../metrics/collector.js';
@@ -39,6 +39,7 @@ function parsePersonasWithWarning(personaList) {
 }
 export const exploreCommand = new Command('explore')
     .description('Explore MCP server behavior with LLM-powered testing')
+    .allowUnknownOption() // Allow server flags like -y for npx to pass through
     .argument('[server-command]', 'Server command (overrides config)')
     .argument('[args...]', 'Server arguments')
     .option('-c, --config <path>', 'Path to config file', PATHS.DEFAULT_CONFIG_FILENAME)
@@ -56,8 +57,15 @@ export const exploreCommand = new Command('explore')
         throw error;
     }
     // Determine server command (CLI arg overrides config)
-    const serverCommand = serverCommandArg || config.server.command;
-    const args = serverArgs.length > 0 ? serverArgs : config.server.args;
+    // If command string contains spaces and no separate args, parse it
+    let serverCommand = serverCommandArg || config.server.command;
+    let args = serverArgs.length > 0 ? serverArgs : config.server.args;
+    // Handle command strings like "npx @package" in config when args is empty
+    if (!serverCommandArg && args.length === 0 && serverCommand.includes(' ')) {
+        const parsed = parseCommandString(serverCommand);
+        serverCommand = parsed.command;
+        args = parsed.args;
+    }
     const transport = config.server.transport ?? 'stdio';
     const remoteUrl = config.server.url?.trim();
     const remoteSessionId = config.server.sessionId?.trim();

package/dist/config/loader.d.ts

@@ -6,6 +6,20 @@
  */
 import { type BellwetherConfig } from './validator.js';
 export type { BellwetherConfig };
+/**
+ * Parse a command string into command and arguments.
+ * Handles quoted strings properly for cases like:
+ *   "npx @gitkraken/gk@latest" -> { command: "npx", args: ["@gitkraken/gk@latest"] }
+ *   "node ./server.js --port 3000" -> { command: "node", args: ["./server.js", "--port", "3000"] }
+ *   'my-cmd "path with spaces"' -> { command: "my-cmd", args: ["path with spaces"] }
+ *
+ * @param commandString - Full command string that may include arguments
+ * @returns Parsed command and arguments
+ */
+export declare function parseCommandString(commandString: string): {
+    command: string;
+    args: string[];
+};
 /**
  * Error thrown when no config file is found.
  */

package/dist/config/loader.js

@@ -58,6 +58,65 @@ function interpolateConfig(obj) {
     }
     return obj;
 }
+/**
+ * Parse a command string into command and arguments.
+ * Handles quoted strings properly for cases like:
+ *   "npx @gitkraken/gk@latest" -> { command: "npx", args: ["@gitkraken/gk@latest"] }
+ *   "node ./server.js --port 3000" -> { command: "node", args: ["./server.js", "--port", "3000"] }
+ *   'my-cmd "path with spaces"' -> { command: "my-cmd", args: ["path with spaces"] }
+ *
+ * @param commandString - Full command string that may include arguments
+ * @returns Parsed command and arguments
+ */
+export function parseCommandString(commandString) {
+    const tokens = [];
+    let current = '';
+    let inQuotes = false;
+    let quoteChar = '';
+    for (let i = 0; i < commandString.length; i++) {
+        const char = commandString[i];
+        const prevChar = i > 0 ? commandString[i - 1] : '';
+        // Handle escape sequences (\" or \')
+        if (char === '\\' && i + 1 < commandString.length) {
+            const nextChar = commandString[i + 1];
+            if (nextChar === '"' || nextChar === "'" || nextChar === '\\') {
+                current += nextChar;
+                i++; // Skip next char
+                continue;
+            }
+        }
+        // Handle quote start
+        if ((char === '"' || char === "'") && !inQuotes) {
+            inQuotes = true;
+            quoteChar = char;
+            continue;
+        }
+        // Handle quote end
+        if (char === quoteChar && inQuotes && prevChar !== '\\') {
+            inQuotes = false;
+            quoteChar = '';
+            continue;
+        }
+        // Handle space outside quotes
+        if (char === ' ' && !inQuotes) {
+            if (current.length > 0) {
+                tokens.push(current);
+                current = '';
+            }
+            continue;
+        }
+        // Regular character
+        current += char;
+    }
+    // Push final token
+    if (current.length > 0) {
+        tokens.push(current);
+    }
+    return {
+        command: tokens[0] ?? '',
+        args: tokens.slice(1),
+    };
+}
 /**
  * Error thrown when no config file is found.
  */

package/dist/config/validator.d.ts

@@ -24,20 +24,20 @@ export declare const serverConfigSchema: z.ZodDefault<z.ZodObject<{
     /** Additional environment variables */
     env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
 }, "strip", z.ZodTypeAny, {
+    command: string;
+    args: string[];
     transport: "stdio" | "sse" | "streamable-http";
     timeout: number;
     sessionId: string;
     url: string;
-    args: string[];
-    command: string;
     env?: Record<string, string> | undefined;
 }, {
+    command?: string | undefined;
+    args?: string[] | undefined;
     transport?: "stdio" | "sse" | "streamable-http" | undefined;
     timeout?: number | undefined;
     sessionId?: string | undefined;
     url?: string | undefined;
-    args?: string[] | undefined;
-    command?: string | undefined;
     env?: Record<string, string> | undefined;
 }>>;
 /**
@@ -1101,16 +1101,16 @@ export declare const contractConfigSchema: z.ZodDefault<z.ZodObject<{
     /** Exit with error when violations are found */
     failOnViolation: z.ZodDefault<z.ZodBoolean>;
 }, "strip", z.ZodTypeAny, {
+    mode: "strict" | "lenient" | "report";
     format: "text" | "json" | "markdown";
     timeout: number;
-    mode: "strict" | "lenient" | "report";
     failOnViolation: boolean;
     path?: string | undefined;
 }, {
+    mode?: "strict" | "lenient" | "report" | undefined;
     format?: "text" | "json" | "markdown" | undefined;
     timeout?: number | undefined;
     path?: string | undefined;
-    mode?: "strict" | "lenient" | "report" | undefined;
     failOnViolation?: boolean | undefined;
 }>>;
 /**
@@ -1137,20 +1137,20 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
         /** Additional environment variables */
         env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
     }, "strip", z.ZodTypeAny, {
+        command: string;
+        args: string[];
         transport: "stdio" | "sse" | "streamable-http";
         timeout: number;
         sessionId: string;
         url: string;
-        args: string[];
-        command: string;
         env?: Record<string, string> | undefined;
     }, {
+        command?: string | undefined;
+        args?: string[] | undefined;
         transport?: "stdio" | "sse" | "streamable-http" | undefined;
         timeout?: number | undefined;
         sessionId?: string | undefined;
         url?: string | undefined;
-        args?: string[] | undefined;
-        command?: string | undefined;
         env?: Record<string, string> | undefined;
     }>>;
     /** LLM configuration (used by explore command) */
@@ -1888,16 +1888,16 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
         /** Exit with error when violations are found */
         failOnViolation: z.ZodDefault<z.ZodBoolean>;
     }, "strip", z.ZodTypeAny, {
+        mode: "strict" | "lenient" | "report";
         format: "text" | "json" | "markdown";
         timeout: number;
-        mode: "strict" | "lenient" | "report";
         failOnViolation: boolean;
         path?: string | undefined;
     }, {
+        mode?: "strict" | "lenient" | "report" | undefined;
         format?: "text" | "json" | "markdown" | undefined;
         timeout?: number | undefined;
         path?: string | undefined;
-        mode?: "strict" | "lenient" | "report" | undefined;
         failOnViolation?: boolean | undefined;
     }>>;
 }, "strip", z.ZodTypeAny, {
@@ -1922,12 +1922,12 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
         defaultServerCommand: string;
     };
     server: {
+        command: string;
+        args: string[];
         transport: "stdio" | "sse" | "streamable-http";
         timeout: number;
         sessionId: string;
         url: string;
-        args: string[];
-        command: string;
         env?: Record<string, string> | undefined;
     };
     discovery: {
@@ -2076,9 +2076,9 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
         badgeOnly: boolean;
     };
     contract: {
+        mode: "strict" | "lenient" | "report";
         format: "text" | "json" | "markdown";
         timeout: number;
-        mode: "strict" | "lenient" | "report";
         failOnViolation: boolean;
         path?: string | undefined;
     };
@@ -2104,12 +2104,12 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
         defaultServerCommand?: string | undefined;
     } | undefined;
     server?: {
+        command?: string | undefined;
+        args?: string[] | undefined;
         transport?: "stdio" | "sse" | "streamable-http" | undefined;
         timeout?: number | undefined;
         sessionId?: string | undefined;
         url?: string | undefined;
-        args?: string[] | undefined;
-        command?: string | undefined;
         env?: Record<string, string> | undefined;
     } | undefined;
     discovery?: {
@@ -2258,10 +2258,10 @@ export declare const bellwetherConfigSchema: z.ZodObject<{
         badgeOnly?: boolean | undefined;
     } | undefined;
     contract?: {
+        mode?: "strict" | "lenient" | "report" | undefined;
         format?: "text" | "json" | "markdown" | undefined;
         timeout?: number | undefined;
         path?: string | undefined;
-        mode?: "strict" | "lenient" | "report" | undefined;
         failOnViolation?: boolean | undefined;
     } | undefined;
 }>;

package/dist/constants/testing.d.ts

@@ -474,6 +474,55 @@ export declare const STATEFUL_TESTING: {
     /** Maximum number of stored values across tool calls */
     readonly MAX_STORED_VALUES: 50;
 };
+/**
+ * Configuration for detecting tool patterns that commonly cause false positives.
+ * Used by schema-test-generator.ts to adjust test expectations.
+ *
+ * These patterns help Bellwether distinguish between:
+ * - Actual tool bugs (should be flagged)
+ * - Expected behavior for specialized tool patterns (should not be flagged)
+ */
+/**
+ * Operation-based tool detection patterns.
+ * Tools with operation enum + args object pattern dispatch to different handlers
+ * where each operation has different required arguments.
+ */
+export declare const OPERATION_BASED_DETECTION: {
+    /** Parameter names that indicate an operation/action discriminator */
+    readonly OPERATION_PARAM_NAMES: readonly string[];
+    /** Parameter names that indicate a dynamic arguments object */
+    readonly ARGS_PARAM_NAMES: readonly string[];
+    /** Minimum enum values to consider a dispatch pattern (single value is not dispatch) */
+    readonly MIN_ENUM_VALUES: 2;
+};
+/**
+ * Self-stateful tool detection patterns.
+ * Tools that require prior invocation to establish state (sessions, chains, contexts).
+ * These tools need an active session/chain before they can be used.
+ */
+export declare const SELF_STATEFUL_DETECTION: {
+    /** Description patterns indicating the tool requires prior state */
+    readonly DESCRIPTION_PATTERNS: readonly RegExp[];
+    /** Parameter names that suggest session/state dependency */
+    readonly STATE_PARAM_PATTERNS: readonly RegExp[];
+    /** Tool name patterns that suggest stateful behavior */
+    readonly STATEFUL_TOOL_NAME_PATTERNS: readonly RegExp[];
+};
+/**
+ * Complex array schema detection patterns.
+ * Tools with arrays whose items have complex nested structures with required properties.
+ * These require properly structured input data that simple test generation can't provide.
+ */
+export declare const COMPLEX_SCHEMA_DETECTION: {
+    /** Maximum nesting depth before considering schema "complex" */
+    readonly MAX_SIMPLE_DEPTH: 2;
+    /** Minimum required properties in array items to consider "complex" */
+    readonly MIN_REQUIRED_PROPERTIES: 1;
+    /** Property names that typically require structured data (chart/data visualization) */
+    readonly STRUCTURED_DATA_PATTERNS: readonly RegExp[];
+    /** Minimum array items in schema examples/defaults to use instead of generating */
+    readonly MIN_EXAMPLE_ITEMS: 2;
+};
 /**
  * Security testing configuration for check mode.
  * Used by security-tester.ts for deterministic vulnerability detection.
@@ -550,6 +599,14 @@ export declare const SEMANTIC_VALIDATION: {
     readonly MAX_INVALID_VALUES_PER_PARAM: 2;
     /** Maximum semantic tests per tool */
     readonly MAX_SEMANTIC_TESTS_PER_TOOL: 6;
+    /**
+     * Enable flexible semantic validation by default.
+     * When true, semantic tests use 'either' outcome, allowing tools to
+     * accept flexible formats (e.g., dayjs accepting various date strings).
+     * This reduces false positives for tools with lenient parsing libraries.
+     * Set to false for strict semantic validation enforcement.
+     */
+    readonly FLEXIBLE_BY_DEFAULT: true;
     /** Confidence scores for different inference sources */
     readonly CONFIDENCE: {
         /** Confidence when schema format explicitly specifies type */
@@ -719,7 +776,9 @@ export declare const EXTERNAL_DEPENDENCIES: {
         };
         readonly stripe: {
             readonly name: "Stripe";
+            /** Patterns in tool names/descriptions that indicate Stripe usage (high confidence = has 'stripe' in name) */
             readonly toolPatterns: readonly RegExp[];
+            /** Low confidence patterns - may match non-Stripe tools (removed: payment, charge, subscription) */
             readonly errorPatterns: readonly RegExp[];
             readonly statusCodes: readonly number[];
             readonly remediation: "Configure Stripe API keys (STRIPE_SECRET_KEY)";
@@ -747,6 +806,7 @@ export declare const EXTERNAL_DEPENDENCIES: {
         };
         readonly openai: {
             readonly name: "OpenAI";
+            /** Patterns in tool names that indicate OpenAI usage (removed generic: completion, embedding) */
             readonly toolPatterns: readonly RegExp[];
             readonly errorPatterns: readonly RegExp[];
             readonly statusCodes: readonly number[];
@@ -789,6 +849,7 @@ export declare const EXTERNAL_DEPENDENCIES: {
         };
         readonly twilio: {
             readonly name: "Twilio";
+            /** Patterns in tool names that indicate Twilio usage (removed generic: sms) */
             readonly toolPatterns: readonly RegExp[];
             readonly errorPatterns: readonly RegExp[];
             readonly statusCodes: readonly number[];
@@ -803,6 +864,7 @@ export declare const EXTERNAL_DEPENDENCIES: {
         };
         readonly sendgrid: {
             readonly name: "SendGrid";
+            /** Patterns in tool names that indicate SendGrid usage (removed generic: email.*send) */
             readonly toolPatterns: readonly RegExp[];
             readonly errorPatterns: readonly RegExp[];
             readonly statusCodes: readonly number[];
@@ -817,6 +879,7 @@ export declare const EXTERNAL_DEPENDENCIES: {
         };
         readonly github: {
             readonly name: "GitHub";
+            /** Patterns in tool names that indicate GitHub usage (removed generic: repository, pull.*request) */
             readonly toolPatterns: readonly RegExp[];
             readonly errorPatterns: readonly RegExp[];
             readonly statusCodes: readonly number[];
@@ -831,6 +894,7 @@ export declare const EXTERNAL_DEPENDENCIES: {
         };
         readonly database: {
             readonly name: "Database";
+            /** Patterns in tool names that indicate database usage (removed generic: sql) */
             readonly toolPatterns: readonly RegExp[];
             readonly errorPatterns: readonly RegExp[];
             readonly statusCodes: readonly number[];