@dotsetlabs/bellwether 2.0.0 → 2.0.1

package/CHANGELOG.md

@@ -7,6 +7,50 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.0.1] - 2026-02-07
+
+### Added
+
+- **MCP protocol version gating**: New `src/protocol/` module with version-to-feature-flag mapping
+  - Supports MCP protocol versions: `2024-11-05`, `2025-03-26`, `2025-06-18`, `2025-11-25`
+  - `MCPFeatureFlags` interface with 9 feature flags (`toolAnnotations`, `entityTitles`, `completions`, `resourceAnnotations`, `structuredOutput`, `serverInstructions`, `httpVersionHeader`, `tasks`, `icons`)
+  - `getSharedFeatureFlags(v1, v2)` computes AND-intersection for cross-version baseline comparison
+  - All version-specific fields in baselines are now gated by protocol version during conversion and comparison
+- **Version-gated drift detection**: Comparator now detects changes in version-specific fields
+  - Tool annotations (readOnlyHint, destructiveHint, idempotentHint, openWorldHint)
+  - Entity titles (tool, prompt, resource, and resource template titles)
+  - Output schema and structured output changes
+  - Execution/task support changes
+  - Server instructions changes
+- **MCPClient protocol version tracking**: Client stores negotiated protocol version after `initialize()`, exposes via getters
+- **Mock server protocol version support**: Mock MCP server now supports `MOCK_PROTOCOL_VERSION` env var for testing
+
+### Fixed
+
+- **20 production-blocking bugs across all layers** (`4717ca1`):
+  - Transport: HTTP transport URL construction, SSE error event handling, MCP client error propagation
+  - Discovery: ResourceTemplate type handling, discovery error handling
+  - Baseline: Converter version-gated field handling, saver hash calculation, comparator severity logic
+  - CLI: Check command exit code handling, explore command cleanup, baseline command error paths
+  - Config: Environment variable expansion edge cases
+  - Docs: Contract and agents generator error handling
+- **Protocol version gating gaps causing false negatives and data loss** (`dce73ed`):
+  - Fixed tool title comparison using wrong feature flag (`toolAnnotations` instead of `entityTitles`)
+  - Fixed tool title comparison condition (AND → OR) to detect added/removed titles
+  - Added missing `execution` and `baselineP99Ms` fields to `ToolFingerprint` type
+  - Added missing fields (`title`, `outputSchema`, `outputSchemaHash`, `annotations`, `execution`, `baselineP99Ms`) to `toToolCapability()` accessor — prevents data loss during incremental check merges
+  - Added `execution` and `baselineP99Ms` mapping to `getToolFingerprints()` accessor
+  - Added prompt title comparison gated by `entityTitles` flag
+  - Added resource title comparison gated by `entityTitles` flag
+  - Added resource template title comparison gated by `entityTitles` flag
+  - Added execution/task support comparison gated by `tasks` flag
+  - Added server instructions comparison gated by `serverInstructions` flag
+  - Gated resource template `title` in converter by `entityTitles` flag
+- **Clean JSON output from baseline commands** (`7aab450`):
+  - `baseline compare --format json` no longer appends summary text after JSON object
+  - `baseline diff --format json` no longer prepends header or appends summary text around JSON object
+  - JSON output is now machine-parseable without text contamination
+
 ## [2.0.0] - 2026-02-04
 
 ### Breaking Changes

package/README.md

@@ -75,6 +75,15 @@ jobs:
 | Parameter renamed | `path` to `file_path` | Breaking |
 | Description changed | Tool help text updated | Warning |
 | Performance regression | Latency increased >10% | Warning |
+| Tool annotations changed | `readOnlyHint` flipped to `false` | Warning |
+| Output schema changed | Return type structure modified | Warning |
+| Entity title changed | Tool/prompt/resource title updated | Info |
+| Task support changed | Execution mode switched to `async` | Warning |
+| Server instructions changed | Server-level instructions updated | Info |
+| Prompt added/removed | Prompt template appears or disappears | Breaking |
+| Resource changed | Resource URI or MIME type modified | Warning |
+
+Comparisons are **protocol-version-aware** — version-specific fields (annotations, titles, output schemas, etc.) are only compared when both baselines support the relevant MCP protocol version.
 
 ## Commands
 

package/dist/auth/credentials.js

@@ -58,6 +58,8 @@ function readEnvFile(filePath, envVar, options) {
                 if (decrypted) {
                     return decrypted;
                 }
+                // Warn about decryption failure so users know their credential exists but can't be decrypted
+                console.warn(`[bellwether] Encrypted credential found for ${envVar} but decryption failed. Check your encryption key.`);
                 return undefined;
             }
             if (value) {

package/dist/baseline/accessors.js

@@ -57,9 +57,15 @@ export function toToolCapability(tool) {
         errorPatterns: tool.errorPatterns,
         baselineP50Ms: tool.baselineP50Ms,
         baselineP95Ms: tool.baselineP95Ms,
+        baselineP99Ms: tool.baselineP99Ms,
         baselineSuccessRate: tool.baselineSuccessRate,
         performanceConfidence: tool.performanceConfidence,
         securityFingerprint: tool.securityFingerprint,
+        title: tool.title,
+        outputSchema: tool.outputSchema,
+        outputSchemaHash: tool.outputSchemaHash,
+        annotations: tool.annotations,
+        execution: tool.execution,
     };
 }
 export function getToolFingerprints(baseline) {
@@ -93,6 +99,12 @@ export function getToolFingerprints(baseline) {
             baselineSuccessRate: tool.baselineSuccessRate,
             performanceConfidence: tool.performanceConfidence,
             securityFingerprint: tool.securityFingerprint,
+            title: tool.title,
+            outputSchema: tool.outputSchema,
+            outputSchemaHash: tool.outputSchemaHash,
+            annotations: tool.annotations,
+            execution: tool.execution,
+            baselineP99Ms: tool.baselineP99Ms,
         };
     });
     if (fingerprints.length > 0) {

package/dist/baseline/baseline-format.d.ts

@@ -72,6 +72,8 @@ export interface BaselineServerFingerprint {
     protocolVersion: string;
     /** Available capabilities */
     capabilities: string[];
+    /** Server-provided instructions (MCP 2025-11-25) */
+    instructions?: string;
 }
 /**
  * Tool capability from discovery.
@@ -85,6 +87,24 @@ export interface ToolCapability {
     inputSchema: Record<string, unknown>;
     /** Hash of the schema for change detection */
     schemaHash: string;
+    /** Human-readable title for the tool (MCP 2025-11-25) */
+    title?: string;
+    /** JSON Schema for the tool's output (MCP 2025-11-25 structured content) */
+    outputSchema?: Record<string, unknown>;
+    /** Hash of the output schema for drift detection */
+    outputSchemaHash?: string;
+    /** Behavioral annotations/hints (MCP 2025-11-25) */
+    annotations?: {
+        title?: string;
+        readOnlyHint?: boolean;
+        destructiveHint?: boolean;
+        idempotentHint?: boolean;
+        openWorldHint?: boolean;
+    };
+    /** Task execution configuration (MCP 2025-11-25) */
+    execution?: {
+        taskSupport?: string;
+    };
     /** Hash of observed arguments schema (from actual calls) */
     observedArgsSchemaHash?: string;
     /** Consistency of observed argument schemas (0-1) */
@@ -137,6 +157,31 @@ export interface ResourceCapability {
     description?: string;
     /** MIME type */
     mimeType?: string;
+    /** Human-readable title (MCP 2025-11-25) */
+    title?: string;
+    /** Resource annotations (MCP 2025-11-25) */
+    annotations?: {
+        audience?: string[];
+        priority?: number;
+        lastModified?: string;
+    };
+    /** Resource size in bytes (MCP 2025-11-25) */
+    size?: number;
+}
+/**
+ * Resource template capability from discovery (MCP 2025-11-25).
+ */
+export interface ResourceTemplateCapability {
+    /** URI template (RFC 6570) */
+    uriTemplate: string;
+    /** Template name */
+    name: string;
+    /** Human-readable title */
+    title?: string;
+    /** Template description */
+    description?: string;
+    /** Expected MIME type */
+    mimeType?: string;
 }
 /**
  * Prompt capability from discovery.
@@ -146,6 +191,8 @@ export interface PromptCapability {
     name: string;
     /** Prompt description */
     description?: string;
+    /** Human-readable title (MCP 2025-11-25) */
+    title?: string;
     /** Arguments the prompt accepts */
     arguments?: Array<{
         name: string;
@@ -265,6 +312,7 @@ export interface BellwetherBaseline {
     capabilities: {
         tools: ToolCapability[];
         resources?: ResourceCapability[];
+        resourceTemplates?: ResourceTemplateCapability[];
         prompts?: PromptCapability[];
     };
     /** Interview results by persona */

package/dist/baseline/comparator.js

@@ -18,6 +18,7 @@ import { compareSchemaEvolution } from './response-schema-tracker.js';
 import { checkVersionCompatibility, BaselineVersionError, parseVersion, areVersionsCompatible, getCompatibilityWarning, } from './version.js';
 import { compareSchemas, computeSchemaHash } from './schema-compare.js';
 import { PERFORMANCE_TRACKING } from '../constants.js';
+import { getSharedFeatureFlags } from '../protocol/index.js';
 import { hasReliableConfidence } from './performance-tracker.js';
 import { compareDocumentationScores, scoreDocumentation } from './documentation-scorer.js';
 /**
@@ -53,6 +54,8 @@ export function compareBaselines(previous, current, options = {}) {
             `Recreate the older baseline with the current CLI version, ` +
             `or use --ignore-version-mismatch to force comparison (results may be incorrect).`, v1.raw, v2.raw);
     }
+    // Compute shared feature flags from both baselines' protocol versions
+    const sharedFeatures = getSharedFeatureFlags(previous.server.protocolVersion, current.server.protocolVersion);
     const previousTools = getToolFingerprints(previous);
     const currentTools = getToolFingerprints(current);
     const previousToolMap = new Map(previousTools.map((t) => [t.name, t]));
@@ -79,7 +82,7 @@ export function compareBaselines(previous, current, options = {}) {
             toolsAdded.push(name);
             continue;
         }
-        const toolDiff = compareTool(previousTool, currentTool, options);
+        const toolDiff = compareTool(previousTool, currentTool, options, sharedFeatures);
         if (toolDiff.changes.length > 0 ||
             toolDiff.schemaChanged ||
             toolDiff.descriptionChanged ||
@@ -89,11 +92,12 @@ export function compareBaselines(previous, current, options = {}) {
             behaviorChanges.push(...toolDiff.changes);
         }
     }
-    // Compare prompts and resources
-    behaviorChanges.push(...comparePrompts(previous.capabilities.prompts, current.capabilities.prompts));
-    behaviorChanges.push(...compareResources(previous.capabilities.resources, current.capabilities.resources));
+    // Compare prompts, resources, and resource templates
+    behaviorChanges.push(...comparePrompts(previous.capabilities.prompts, current.capabilities.prompts, sharedFeatures));
+    behaviorChanges.push(...compareResources(previous.capabilities.resources, current.capabilities.resources, sharedFeatures));
+    behaviorChanges.push(...compareResourceTemplates(previous.capabilities.resourceTemplates, current.capabilities.resourceTemplates, sharedFeatures));
     // Compare server metadata and capabilities
-    behaviorChanges.push(...compareServerInfo(previous.server, current.server));
+    behaviorChanges.push(...compareServerInfo(previous.server, current.server, sharedFeatures));
     // Compare workflows
     const workflowChanges = compareWorkflows(previous.workflows || [], current.workflows || []);
     behaviorChanges.push(...workflowChanges);
@@ -127,7 +131,7 @@ export function compareBaselines(previous, current, options = {}) {
         documentationScoreReport,
     };
 }
-function compareTool(previous, current, options) {
+function compareTool(previous, current, options, features) {
     const changes = [];
     let schemaChanged = false;
     let descriptionChanged = false;
@@ -309,6 +313,117 @@ function compareTool(previous, current, options) {
             }
         }
     }
+    // Compare tool title — only when both versions support entity titles
+    if (features.entityTitles) {
+        if (previous.title !== current.title &&
+            (previous.title !== undefined || current.title !== undefined)) {
+            changes.push({
+                tool: current.name,
+                aspect: 'tool_annotations',
+                before: previous.title ?? 'none',
+                after: current.title ?? 'none',
+                severity: 'info',
+                description: `Tool "${current.name}" title changed`,
+            });
+        }
+    }
+    // Compare tool annotations — only when both versions support them
+    if (features.toolAnnotations) {
+        // Compare annotations
+        const prevAnno = previous.annotations;
+        const currAnno = current.annotations;
+        if (prevAnno || currAnno) {
+            if (prevAnno?.readOnlyHint !== currAnno?.readOnlyHint) {
+                // readOnlyHint changing (e.g., tool becoming non-read-only) is breaking
+                changes.push({
+                    tool: current.name,
+                    aspect: 'tool_annotations',
+                    before: String(prevAnno?.readOnlyHint ?? 'unset'),
+                    after: String(currAnno?.readOnlyHint ?? 'unset'),
+                    severity: 'breaking',
+                    description: `Tool "${current.name}" readOnlyHint changed`,
+                });
+            }
+            if (prevAnno?.destructiveHint !== currAnno?.destructiveHint) {
+                changes.push({
+                    tool: current.name,
+                    aspect: 'tool_annotations',
+                    before: String(prevAnno?.destructiveHint ?? 'unset'),
+                    after: String(currAnno?.destructiveHint ?? 'unset'),
+                    severity: 'warning',
+                    description: `Tool "${current.name}" destructiveHint changed`,
+                });
+            }
+            if (prevAnno?.idempotentHint !== currAnno?.idempotentHint) {
+                changes.push({
+                    tool: current.name,
+                    aspect: 'tool_annotations',
+                    before: String(prevAnno?.idempotentHint ?? 'unset'),
+                    after: String(currAnno?.idempotentHint ?? 'unset'),
+                    severity: 'warning',
+                    description: `Tool "${current.name}" idempotentHint changed`,
+                });
+            }
+            if (prevAnno?.openWorldHint !== currAnno?.openWorldHint) {
+                changes.push({
+                    tool: current.name,
+                    aspect: 'tool_annotations',
+                    before: String(prevAnno?.openWorldHint ?? 'unset'),
+                    after: String(currAnno?.openWorldHint ?? 'unset'),
+                    severity: 'info',
+                    description: `Tool "${current.name}" openWorldHint changed`,
+                });
+            }
+        }
+    }
+    // Compare output schema — only when both versions support structured output
+    if (features.structuredOutput && previous.outputSchemaHash !== current.outputSchemaHash) {
+        if (!previous.outputSchemaHash && current.outputSchemaHash) {
+            changes.push({
+                tool: current.name,
+                aspect: 'output_schema',
+                before: 'none',
+                after: `outputSchema: ${current.outputSchemaHash}`,
+                severity: 'warning',
+                description: `Tool "${current.name}" outputSchema added`,
+            });
+        }
+        else if (previous.outputSchemaHash && !current.outputSchemaHash) {
+            changes.push({
+                tool: current.name,
+                aspect: 'output_schema',
+                before: `outputSchema: ${previous.outputSchemaHash}`,
+                after: 'none',
+                severity: 'warning',
+                description: `Tool "${current.name}" outputSchema removed`,
+            });
+        }
+        else {
+            changes.push({
+                tool: current.name,
+                aspect: 'output_schema',
+                before: `outputSchema: ${previous.outputSchemaHash}`,
+                after: `outputSchema: ${current.outputSchemaHash}`,
+                severity: 'breaking',
+                description: `Tool "${current.name}" outputSchema changed`,
+            });
+        }
+    }
+    // Compare execution/task support — only when both versions support tasks
+    if (features.tasks) {
+        const prevExec = previous.execution?.taskSupport;
+        const currExec = current.execution?.taskSupport;
+        if (prevExec !== currExec && (prevExec !== undefined || currExec !== undefined)) {
+            changes.push({
+                tool: current.name,
+                aspect: 'tool_annotations',
+                before: prevExec ?? 'none',
+                after: currExec ?? 'none',
+                severity: 'warning',
+                description: `Tool "${current.name}" task support changed`,
+            });
+        }
+    }
     return {
         tool: current.name,
         changes,
@@ -321,7 +436,7 @@ function compareTool(previous, current, options) {
         schemaEvolutionDiff,
     };
 }
-function comparePrompts(previous, current) {
+function comparePrompts(previous, current, features) {
     const changes = [];
     const prevMap = new Map((previous ?? []).map((p) => [p.name, p]));
     const currMap = new Map((current ?? []).map((p) => [p.name, p]));
@@ -348,6 +463,18 @@ function comparePrompts(previous, current) {
                 description: `Prompt "${name}" description changed`,
             });
         }
+        if (features?.entityTitles &&
+            prevPrompt.title !== currPrompt.title &&
+            (prevPrompt.title !== undefined || currPrompt.title !== undefined)) {
+            changes.push({
+                tool: `prompt:${name}`,
+                aspect: 'prompt',
+                before: prevPrompt.title ?? 'none',
+                after: currPrompt.title ?? 'none',
+                severity: 'info',
+                description: `Prompt "${name}" title changed`,
+            });
+        }
         const prevArgs = prevPrompt.arguments ?? [];
         const currArgs = currPrompt.arguments ?? [];
         const prevArgMap = new Map(prevArgs.map((a) => [a.name, a]));
@@ -413,7 +540,7 @@ function comparePrompts(previous, current) {
     }
     return changes;
 }
-function compareResources(previous, current) {
+function compareResources(previous, current, features) {
     const changes = [];
     const prevMap = new Map((previous ?? []).map((r) => [r.uri, r]));
     const currMap = new Map((current ?? []).map((r) => [r.uri, r]));
@@ -460,6 +587,45 @@ function compareResources(previous, current) {
                 description: `Resource "${uri}" mime type changed`,
             });
         }
+        // Compare resource title — only when both versions support entity titles
+        if (features?.entityTitles &&
+            prevResource.title !== currResource.title &&
+            (prevResource.title !== undefined || currResource.title !== undefined)) {
+            changes.push({
+                tool: `resource:${currResource.name ?? uri}`,
+                aspect: 'resource',
+                before: prevResource.title ?? 'none',
+                after: currResource.title ?? 'none',
+                severity: 'info',
+                description: `Resource "${uri}" title changed`,
+            });
+        }
+        // Compare resource annotations — only when both versions support them
+        if (features?.resourceAnnotations) {
+            const prevAudience = prevResource.annotations?.audience?.join(',');
+            const currAudience = currResource.annotations?.audience?.join(',');
+            if (prevAudience !== currAudience && (prevAudience || currAudience)) {
+                changes.push({
+                    tool: `resource:${currResource.name ?? uri}`,
+                    aspect: 'resource_annotations',
+                    before: prevAudience ?? 'none',
+                    after: currAudience ?? 'none',
+                    severity: 'warning',
+                    description: `Resource "${uri}" audience annotation changed`,
+                });
+            }
+            if (prevResource.size !== currResource.size &&
+                (prevResource.size !== undefined || currResource.size !== undefined)) {
+                changes.push({
+                    tool: `resource:${currResource.name ?? uri}`,
+                    aspect: 'resource_annotations',
+                    before: prevResource.size !== undefined ? String(prevResource.size) : 'unknown',
+                    after: currResource.size !== undefined ? String(currResource.size) : 'unknown',
+                    severity: 'info',
+                    description: `Resource "${uri}" size changed`,
+                });
+            }
+        }
     }
     for (const [uri, prevResource] of prevMap) {
         if (!currMap.has(uri)) {
@@ -475,7 +641,71 @@ function compareResources(previous, current) {
     }
     return changes;
 }
-function compareServerInfo(previous, current) {
+function compareResourceTemplates(previous, current, features) {
+    const changes = [];
+    const prevMap = new Map((previous ?? []).map((t) => [t.uriTemplate, t]));
+    const currMap = new Map((current ?? []).map((t) => [t.uriTemplate, t]));
+    for (const [uriTemplate, currTemplate] of currMap) {
+        const prevTemplate = prevMap.get(uriTemplate);
+        if (!prevTemplate) {
+            changes.push({
+                tool: `resource_template:${currTemplate.name ?? uriTemplate}`,
+                aspect: 'resource_template',
+                before: 'absent',
+                after: 'present',
+                severity: 'info',
+                description: `Resource template "${uriTemplate}" added`,
+            });
+            continue;
+        }
+        if (prevTemplate.description !== currTemplate.description) {
+            changes.push({
+                tool: `resource_template:${currTemplate.name ?? uriTemplate}`,
+                aspect: 'resource_template',
+                before: prevTemplate.description ?? 'none',
+                after: currTemplate.description ?? 'none',
+                severity: 'info',
+                description: `Resource template "${uriTemplate}" description changed`,
+            });
+        }
+        if (prevTemplate.mimeType !== currTemplate.mimeType) {
+            changes.push({
+                tool: `resource_template:${currTemplate.name ?? uriTemplate}`,
+                aspect: 'resource_template',
+                before: prevTemplate.mimeType ?? 'none',
+                after: currTemplate.mimeType ?? 'none',
+                severity: 'info',
+                description: `Resource template "${uriTemplate}" mime type changed`,
+            });
+        }
+        if (features?.entityTitles &&
+            prevTemplate.title !== currTemplate.title &&
+            (prevTemplate.title !== undefined || currTemplate.title !== undefined)) {
+            changes.push({
+                tool: `resource_template:${currTemplate.name ?? uriTemplate}`,
+                aspect: 'resource_template',
+                before: prevTemplate.title ?? 'none',
+                after: currTemplate.title ?? 'none',
+                severity: 'info',
+                description: `Resource template "${uriTemplate}" title changed`,
+            });
+        }
+    }
+    for (const [uriTemplate, prevTemplate] of prevMap) {
+        if (!currMap.has(uriTemplate)) {
+            changes.push({
+                tool: `resource_template:${prevTemplate.name ?? uriTemplate}`,
+                aspect: 'resource_template',
+                before: 'present',
+                after: 'absent',
+                severity: 'breaking',
+                description: `Resource template "${uriTemplate}" removed`,
+            });
+        }
+    }
+    return changes;
+}
+function compareServerInfo(previous, current, features) {
     const changes = [];
     if (previous.name !== current.name) {
         changes.push({
@@ -498,20 +728,41 @@ function compareServerInfo(previous, current) {
         });
     }
     if (previous.protocolVersion !== current.protocolVersion) {
-        const breaking = isMajorVersionChange(previous.protocolVersion, current.protocolVersion);
+        // Protocol version change is always warning severity.
+        // The version registry handles feature gating — the version change itself
+        // is informational drift, not a breaking change.
         changes.push({
             tool: 'server',
             aspect: 'server',
             before: previous.protocolVersion,
             after: current.protocolVersion,
-            severity: breaking ? 'breaking' : 'warning',
-            description: 'Protocol version changed',
+            severity: 'warning',
+            description: `Protocol version changed from ${previous.protocolVersion} to ${current.protocolVersion}`,
         });
     }
+    // Compare server instructions — only when both versions support them
+    if (features?.serverInstructions) {
+        if (previous.instructions !== current.instructions &&
+            (previous.instructions !== undefined || current.instructions !== undefined)) {
+            changes.push({
+                tool: 'server',
+                aspect: 'server',
+                before: previous.instructions ? `"${previous.instructions.slice(0, 50)}..."` : 'none',
+                after: current.instructions ? `"${current.instructions.slice(0, 50)}..."` : 'none',
+                severity: 'info',
+                description: 'Server instructions changed',
+            });
+        }
+    }
     const prevCaps = new Set(previous.capabilities);
     const currCaps = new Set(current.capabilities);
     for (const cap of prevCaps) {
         if (!currCaps.has(cap)) {
+            // Skip capabilities that are version-gated and not in the shared feature set
+            if (cap === 'completions' && !features?.completions)
+                continue;
+            if (cap === 'tasks' && !features?.tasks)
+                continue;
             changes.push({
                 tool: 'server',
                 aspect: 'capability',
@@ -536,14 +787,6 @@ function compareServerInfo(previous, current) {
     }
     return changes;
 }
-function isMajorVersionChange(previous, current) {
-    const prevMajor = parseInt(previous.split('.')[0] ?? '0', 10);
-    const currMajor = parseInt(current.split('.')[0] ?? '0', 10);
-    if (Number.isNaN(prevMajor) || Number.isNaN(currMajor)) {
-        return previous !== current;
-    }
-    return prevMajor !== currMajor;
-}
 function getDeclaredSchemaHash(tool) {
     if (tool.inputSchema && Object.keys(tool.inputSchema).length > 0) {
         return computeSchemaHash(tool.inputSchema);