@dotsetlabs/bellwether 0.10.0

package/dist/baseline/pr-comment-generator.js

@@ -0,0 +1,546 @@
+/**
+ * Enhanced GitHub PR comment generation for schema changes.
+ *
+ * This module generates detailed, actionable PR comments that help reviewers
+ * understand the impact of schema changes on downstream consumers.
+ */
+import { PR_COMMENTS } from '../constants.js';
+/**
+ * Get badge color for severity level.
+ */
+export function getBadgeColor(severity) {
+    return PR_COMMENTS.BADGE_COLORS[severity];
+}
+/**
+ * Generate a shields.io badge URL.
+ */
+export function generateBadgeUrl(label, message, color) {
+    const encodedLabel = encodeURIComponent(label);
+    const encodedMessage = encodeURIComponent(message);
+    return `https://img.shields.io/badge/${encodedLabel}-${encodedMessage}-${color}`;
+}
+/**
+ * Generate a markdown badge.
+ */
+export function generateBadgeMarkdown(label, message, color) {
+    const url = generateBadgeUrl(label, message, color);
+    return `![${label}](${url})`;
+}
+/**
+ * Generate the summary section.
+ */
+function generateSummarySection(diff) {
+    const lines = [];
+    // High-level stats
+    const stats = [
+        diff.toolsAdded.length > 0 ? `**${diff.toolsAdded.length}** tools added` : null,
+        diff.toolsRemoved.length > 0 ? `**${diff.toolsRemoved.length}** tools removed` : null,
+        diff.toolsModified.length > 0 ? `**${diff.toolsModified.length}** tools modified` : null,
+    ].filter(Boolean);
+    if (stats.length > 0) {
+        lines.push(stats.join(' | '));
+        lines.push('');
+    }
+    // Change counts by severity
+    if (diff.breakingCount > 0 || diff.warningCount > 0 || diff.infoCount > 0) {
+        const counts = [];
+        if (diff.breakingCount > 0)
+            counts.push(`🔴 ${diff.breakingCount} breaking`);
+        if (diff.warningCount > 0)
+            counts.push(`🟠 ${diff.warningCount} warnings`);
+        if (diff.infoCount > 0)
+            counts.push(`🔵 ${diff.infoCount} info`);
+        lines.push(counts.join(' | '));
+    }
+    return {
+        title: 'Summary',
+        content: lines.join('\n'),
+        priority: 1,
+    };
+}
+/**
+ * Generate breaking changes section.
+ */
+function generateBreakingChangesSection(diff, config) {
+    const breakingChanges = diff.behaviorChanges.filter(c => c.severity === 'breaking');
+    if (breakingChanges.length === 0) {
+        return null;
+    }
+    const lines = ['> ⚠️ **These changes may break existing integrations**', ''];
+    // Group by tool
+    const byTool = new Map();
+    for (const change of breakingChanges) {
+        const existing = byTool.get(change.tool) || [];
+        existing.push(change);
+        byTool.set(change.tool, existing);
+    }
+    let toolCount = 0;
+    for (const [toolName, changes] of byTool) {
+        if (toolCount >= (config.maxDetailedTools ?? PR_COMMENTS.MAX_DETAILED_TOOLS)) {
+            lines.push(`\n*...and ${byTool.size - toolCount} more tools with breaking changes*`);
+            break;
+        }
+        lines.push(`### \`${toolName}\``);
+        lines.push('');
+        const displayChanges = changes.slice(0, config.maxChangesPerTool ?? PR_COMMENTS.MAX_CHANGES_PER_TOOL);
+        for (const change of displayChanges) {
+            lines.push(`- **${formatAspect(change.aspect)}**: ${change.description}`);
+            if (change.before && change.after) {
+                lines.push(`  - Before: \`${truncate(change.before)}\``);
+                lines.push(`  - After: \`${truncate(change.after)}\``);
+            }
+        }
+        if (changes.length > displayChanges.length) {
+            lines.push(`  - *...and ${changes.length - displayChanges.length} more changes*`);
+        }
+        lines.push('');
+        toolCount++;
+    }
+    return {
+        title: '🔴 Breaking Changes',
+        content: lines.join('\n'),
+        priority: 2,
+    };
+}
+/**
+ * Generate tools added section.
+ */
+function generateToolsAddedSection(diff) {
+    if (diff.toolsAdded.length === 0) {
+        return null;
+    }
+    const lines = [];
+    for (const tool of diff.toolsAdded.slice(0, PR_COMMENTS.MAX_DETAILED_TOOLS)) {
+        lines.push(`- \`${tool}\``);
+    }
+    if (diff.toolsAdded.length > PR_COMMENTS.MAX_DETAILED_TOOLS) {
+        lines.push(`- *...and ${diff.toolsAdded.length - PR_COMMENTS.MAX_DETAILED_TOOLS} more*`);
+    }
+    return {
+        title: '✅ Tools Added',
+        content: lines.join('\n'),
+        priority: 4,
+    };
+}
+/**
+ * Generate tools removed section.
+ */
+function generateToolsRemovedSection(diff) {
+    if (diff.toolsRemoved.length === 0) {
+        return null;
+    }
+    const lines = [
+        '> ⚠️ **Removing tools is a breaking change for consumers**',
+        '',
+    ];
+    for (const tool of diff.toolsRemoved) {
+        lines.push(`- ~~\`${tool}\`~~`);
+    }
+    return {
+        title: '❌ Tools Removed',
+        content: lines.join('\n'),
+        priority: 3,
+    };
+}
+/**
+ * Generate modified tools section.
+ */
+function generateModifiedToolsSection(diff, config) {
+    // Filter out breaking changes (shown separately)
+    const nonBreakingModified = diff.toolsModified.filter(t => !t.changes.some(c => c.severity === 'breaking'));
+    if (nonBreakingModified.length === 0) {
+        return null;
+    }
+    const lines = [];
+    const maxTools = config.maxDetailedTools ?? PR_COMMENTS.MAX_DETAILED_TOOLS;
+    for (const toolDiff of nonBreakingModified.slice(0, maxTools)) {
+        lines.push(`### \`${toolDiff.tool}\``);
+        lines.push('');
+        const changes = toolDiff.changes.slice(0, config.maxChangesPerTool ?? PR_COMMENTS.MAX_CHANGES_PER_TOOL);
+        for (const change of changes) {
+            const icon = change.severity === 'warning' ? '🟠' : '🔵';
+            lines.push(`- ${icon} **${formatAspect(change.aspect)}**: ${change.description}`);
+        }
+        if (toolDiff.changes.length > changes.length) {
+            lines.push(`- *...and ${toolDiff.changes.length - changes.length} more changes*`);
+        }
+        lines.push('');
+    }
+    if (nonBreakingModified.length > maxTools) {
+        lines.push(`*...and ${nonBreakingModified.length - maxTools} more modified tools*`);
+    }
+    return {
+        title: '📝 Tools Modified',
+        content: lines.join('\n'),
+        priority: 5,
+        collapsed: nonBreakingModified.length > 3,
+    };
+}
+/**
+ * Generate migration guide section.
+ */
+function generateMigrationSection(guide, config) {
+    if (!guide || guide.steps.length === 0 || !config.includeMigrationExamples) {
+        return null;
+    }
+    const lines = [
+        `**Estimated effort**: ${guide.estimatedEffort.toUpperCase()}`,
+        `**Breaking changes**: ${guide.stats.breakingChangesCount}`,
+        '',
+        '### Steps',
+        '',
+    ];
+    const maxSteps = config.maxMigrationExamples ?? PR_COMMENTS.MAX_MIGRATION_EXAMPLES;
+    const steps = guide.steps.slice(0, maxSteps);
+    for (let i = 0; i < steps.length; i++) {
+        const step = steps[i];
+        lines.push(`${i + 1}. **${step.title}**`);
+        lines.push(`   - Tool: \`${step.toolName}\``);
+        if (step.codeExamples && step.codeExamples.length > 0) {
+            const example = step.codeExamples[0];
+            lines.push('');
+            lines.push('   ```' + (example.language || ''));
+            lines.push('   // Before:');
+            lines.push('   ' + example.before.split('\n').join('\n   '));
+            lines.push('');
+            lines.push('   // After:');
+            lines.push('   ' + example.after.split('\n').join('\n   '));
+            lines.push('   ```');
+        }
+        lines.push('');
+    }
+    if (guide.steps.length > maxSteps) {
+        lines.push(`*...and ${guide.steps.length - maxSteps} more migration steps*`);
+    }
+    return {
+        title: '🔄 Migration Guide',
+        content: lines.join('\n'),
+        priority: 6,
+        collapsed: true,
+    };
+}
+/**
+ * Generate affected workflows section.
+ */
+function generateAffectedWorkflowsSection(workflows, config) {
+    if (!workflows || workflows.length === 0) {
+        return null;
+    }
+    const lines = [];
+    const maxWorkflows = config.maxAffectedWorkflows ?? PR_COMMENTS.MAX_AFFECTED_WORKFLOWS;
+    for (const workflow of workflows.slice(0, maxWorkflows)) {
+        const icon = workflow.severity === 'breaking' ? '🔴' : workflow.severity === 'warning' ? '🟠' : '🔵';
+        lines.push(`- ${icon} **${workflow.name}**: ${workflow.description}`);
+        lines.push(`  - Tools: ${workflow.affectedTools.map(t => `\`${t}\``).join(', ')}`);
+    }
+    if (workflows.length > maxWorkflows) {
+        lines.push(`\n*...and ${workflows.length - maxWorkflows} more affected workflows*`);
+    }
+    return {
+        title: '⚡ Affected Workflows',
+        content: lines.join('\n'),
+        priority: 7,
+        collapsed: true,
+    };
+}
+/**
+ * Generate action items based on changes.
+ */
+function generateActionItems(diff) {
+    const items = [];
+    if (diff.toolsRemoved.length > 0) {
+        items.push('[ ] Update consumer applications to handle removed tools');
+    }
+    if (diff.breakingCount > 0) {
+        items.push('[ ] Review breaking changes with team before merging');
+        items.push('[ ] Update documentation for changed tool schemas');
+    }
+    const schemaChanges = diff.toolsModified.filter(t => t.schemaChanged);
+    if (schemaChanges.length > 0) {
+        items.push('[ ] Verify client SDKs are updated for schema changes');
+    }
+    if (diff.toolsAdded.length > 0) {
+        items.push('[ ] Add tests for new tools');
+        items.push('[ ] Update API documentation');
+    }
+    // Add testing recommendations
+    if (diff.severity === 'breaking' || diff.severity === 'warning') {
+        items.push('[ ] Run integration tests with downstream consumers');
+    }
+    return items;
+}
+/**
+ * Format a behavior aspect for display.
+ */
+function formatAspect(aspect) {
+    return aspect
+        .replace(/_/g, ' ')
+        .replace(/\b\w/g, c => c.toUpperCase());
+}
+/**
+ * Truncate a string for display.
+ */
+function truncate(value, maxLength = PR_COMMENTS.VALUE_TRUNCATE_LENGTH) {
+    if (value.length <= maxLength)
+        return value;
+    return value.substring(0, maxLength - 3) + '...';
+}
+/**
+ * Render a collapsible section.
+ */
+function renderCollapsibleSection(title, content) {
+    return `<details>\n<summary>${title}</summary>\n\n${content}\n\n</details>`;
+}
+/**
+ * Generate footer with metadata.
+ */
+function generateFooter(config) {
+    const lines = ['---'];
+    const parts = [];
+    if (config.baseBranch && config.headBranch) {
+        parts.push(`Comparing \`${config.baseBranch}\` → \`${config.headBranch}\``);
+    }
+    parts.push('Generated by [Bellwether](https://bellwether.sh)');
+    lines.push(parts.join(' | '));
+    return lines.join('\n');
+}
+/**
+ * Generate a complete PR comment for a behavioral diff.
+ */
+export function generatePRComment(diff, config = {}, migrationGuide, affectedWorkflows) {
+    const { includeMigrationExamples = true, useCollapsibleSections = true, } = config;
+    // Determine badge
+    const badgeLabel = 'Schema Drift';
+    let badgeMessage;
+    let badgeColor;
+    if (diff.severity === 'breaking') {
+        badgeMessage = `${diff.breakingCount} breaking`;
+        badgeColor = 'red';
+    }
+    else if (diff.severity === 'warning') {
+        badgeMessage = `${diff.warningCount} warnings`;
+        badgeColor = 'orange';
+    }
+    else if (diff.severity === 'info') {
+        badgeMessage = 'changes detected';
+        badgeColor = 'blue';
+    }
+    else {
+        badgeMessage = 'no changes';
+        badgeColor = 'green';
+    }
+    // Generate sections
+    const sections = [];
+    sections.push(generateSummarySection(diff));
+    const breakingSection = generateBreakingChangesSection(diff, config);
+    if (breakingSection)
+        sections.push(breakingSection);
+    const removedSection = generateToolsRemovedSection(diff);
+    if (removedSection)
+        sections.push(removedSection);
+    const addedSection = generateToolsAddedSection(diff);
+    if (addedSection)
+        sections.push(addedSection);
+    const modifiedSection = generateModifiedToolsSection(diff, config);
+    if (modifiedSection)
+        sections.push(modifiedSection);
+    if (includeMigrationExamples) {
+        const migrationSection = generateMigrationSection(migrationGuide, config);
+        if (migrationSection)
+            sections.push(migrationSection);
+    }
+    const workflowsSection = generateAffectedWorkflowsSection(affectedWorkflows || [], config);
+    if (workflowsSection)
+        sections.push(workflowsSection);
+    // Sort by priority
+    sections.sort((a, b) => a.priority - b.priority);
+    // Generate action items
+    const actionItems = generateActionItems(diff);
+    // Generate footer
+    const footer = generateFooter(config);
+    // Generate title
+    const title = diff.severity === 'none'
+        ? '✅ No Schema Drift Detected'
+        : diff.severity === 'breaking'
+            ? '🚨 Breaking Schema Changes Detected'
+            : diff.severity === 'warning'
+                ? '⚠️ Schema Changes Detected'
+                : 'ℹ️ Minor Schema Changes';
+    // Render full markdown
+    const markdownLines = [
+        `## ${title}`,
+        '',
+        generateBadgeMarkdown(badgeLabel, badgeMessage, badgeColor),
+        '',
+    ];
+    // Add sections
+    for (const section of sections) {
+        if (useCollapsibleSections && section.collapsed) {
+            markdownLines.push(renderCollapsibleSection(`### ${section.title}`, section.content));
+        }
+        else {
+            markdownLines.push(`### ${section.title}`);
+            markdownLines.push('');
+            markdownLines.push(section.content);
+        }
+        markdownLines.push('');
+    }
+    // Add action items
+    if (actionItems.length > 0) {
+        markdownLines.push('### 📋 Action Items');
+        markdownLines.push('');
+        for (const item of actionItems) {
+            markdownLines.push(`- ${item}`);
+        }
+        markdownLines.push('');
+    }
+    // Add footer
+    markdownLines.push(footer);
+    return {
+        title,
+        summary: diff.summary,
+        badge: {
+            label: badgeLabel,
+            color: badgeColor,
+            message: badgeMessage,
+        },
+        sections,
+        actionItems,
+        footer,
+        markdown: markdownLines.join('\n'),
+    };
+}
+/**
+ * Generate a compact PR comment for simple diffs.
+ */
+export function generateCompactPRComment(diff) {
+    if (diff.severity === 'none') {
+        return '✅ **No schema drift detected** - baseline matches current server state.';
+    }
+    const lines = [];
+    // Header with badge
+    const badgeColor = getBadgeColor(diff.severity);
+    const badgeMessage = diff.severity === 'breaking'
+        ? `${diff.breakingCount} breaking`
+        : diff.severity === 'warning'
+            ? `${diff.warningCount} warnings`
+            : 'changes detected';
+    lines.push(`## Schema Drift Report ${generateBadgeMarkdown('drift', badgeMessage, badgeColor)}`);
+    lines.push('');
+    // Quick stats
+    const stats = [];
+    if (diff.toolsAdded.length > 0)
+        stats.push(`+${diff.toolsAdded.length} added`);
+    if (diff.toolsRemoved.length > 0)
+        stats.push(`-${diff.toolsRemoved.length} removed`);
+    if (diff.toolsModified.length > 0)
+        stats.push(`~${diff.toolsModified.length} modified`);
+    if (stats.length > 0) {
+        lines.push(`**Tools**: ${stats.join(', ')}`);
+    }
+    // Breaking changes summary
+    if (diff.breakingCount > 0) {
+        lines.push('');
+        lines.push('**Breaking changes:**');
+        const breakingChanges = diff.behaviorChanges
+            .filter(c => c.severity === 'breaking')
+            .slice(0, 5);
+        for (const change of breakingChanges) {
+            lines.push(`- \`${change.tool}\`: ${change.description}`);
+        }
+        if (diff.breakingCount > 5) {
+            lines.push(`- *...and ${diff.breakingCount - 5} more*`);
+        }
+    }
+    lines.push('');
+    lines.push('---');
+    lines.push('*Generated by [Bellwether](https://bellwether.sh)*');
+    return lines.join('\n');
+}
+/**
+ * Generate a status check summary for CI.
+ */
+export function generateCIStatusSummary(diff) {
+    if (diff.severity === 'breaking') {
+        return {
+            conclusion: 'failure',
+            title: `${diff.breakingCount} breaking change(s) detected`,
+            summary: `Schema drift analysis found ${diff.breakingCount} breaking changes that may affect consumers.`,
+        };
+    }
+    if (diff.severity === 'warning') {
+        return {
+            conclusion: 'neutral',
+            title: `${diff.warningCount} warning(s) detected`,
+            summary: `Schema drift analysis found ${diff.warningCount} changes that should be reviewed.`,
+        };
+    }
+    if (diff.severity === 'info') {
+        return {
+            conclusion: 'success',
+            title: 'Minor changes detected',
+            summary: `Schema drift analysis found ${diff.infoCount} informational change(s).`,
+        };
+    }
+    return {
+        conclusion: 'success',
+        title: 'No drift detected',
+        summary: 'Schema matches the baseline - no changes detected.',
+    };
+}
+/**
+ * Generate diff visualization as a table.
+ */
+export function generateDiffTable(toolDiffs) {
+    if (toolDiffs.length === 0) {
+        return '*No tool modifications*';
+    }
+    const lines = [
+        '| Tool | Changes | Schema | Description |',
+        '|------|---------|--------|-------------|',
+    ];
+    for (const toolDiff of toolDiffs.slice(0, PR_COMMENTS.MAX_DETAILED_TOOLS)) {
+        const breakingCount = toolDiff.changes.filter(c => c.severity === 'breaking').length;
+        const warningCount = toolDiff.changes.filter(c => c.severity === 'warning').length;
+        const infoCount = toolDiff.changes.filter(c => c.severity === 'info').length;
+        const changeIndicators = [];
+        if (breakingCount > 0)
+            changeIndicators.push(`🔴${breakingCount}`);
+        if (warningCount > 0)
+            changeIndicators.push(`🟠${warningCount}`);
+        if (infoCount > 0)
+            changeIndicators.push(`🔵${infoCount}`);
+        const schemaStatus = toolDiff.schemaChanged ? '⚠️ Changed' : '✅ OK';
+        const descStatus = toolDiff.descriptionChanged ? '⚠️ Changed' : '✅ OK';
+        lines.push(`| \`${toolDiff.tool}\` | ${changeIndicators.join(' ') || '—'} | ${schemaStatus} | ${descStatus} |`);
+    }
+    if (toolDiffs.length > PR_COMMENTS.MAX_DETAILED_TOOLS) {
+        lines.push(`| *...and ${toolDiffs.length - PR_COMMENTS.MAX_DETAILED_TOOLS} more* | | | |`);
+    }
+    return lines.join('\n');
+}
+/**
+ * Determine if a PR comment should block merge.
+ */
+export function shouldBlockMerge(diff, strictMode = true) {
+    if (strictMode) {
+        return diff.severity === 'breaking';
+    }
+    return false;
+}
+/**
+ * Get emoji for severity level.
+ */
+export function getSeverityEmoji(severity) {
+    switch (severity) {
+        case 'breaking':
+            return '🔴';
+        case 'warning':
+            return '🟠';
+        case 'info':
+            return '🔵';
+        case 'none':
+            return '✅';
+    }
+}
+//# sourceMappingURL=pr-comment-generator.js.map

package/dist/baseline/response-fingerprint.d.ts

@@ -0,0 +1,127 @@
+/**
+ * Response fingerprinting for structural drift detection.
+ *
+ * Analyzes MCP tool responses to create deterministic fingerprints
+ * that capture response structure, shape, and characteristics without
+ * requiring LLM analysis.
+ */
+import type { MCPToolCallResult } from '../transport/types.js';
+import type { EnhancedErrorAnalysis } from './error-analyzer.js';
+/**
+ * Content type classification for responses.
+ */
+export type ResponseContentType = 'text' | 'object' | 'array' | 'primitive' | 'empty' | 'error' | 'mixed' | 'binary';
+/**
+ * Size classification for responses.
+ */
+export type ResponseSize = 'tiny' | 'small' | 'medium' | 'large';
+/**
+ * Fingerprint of a tool's response structure.
+ */
+export interface ResponseFingerprint {
+    /** Hash of the response structure (keys, types, nesting) */
+    structureHash: string;
+    /** Primary content type of the response */
+    contentType: ResponseContentType;
+    /** Top-level field names if response is an object */
+    fields?: string[];
+    /** Structure hash of array items (if array response) */
+    arrayItemStructure?: string;
+    /** Approximate response size category */
+    size: ResponseSize;
+    /** Whether the response is empty/has no meaningful content */
+    isEmpty: boolean;
+    /** Number of successful responses used to build this fingerprint */
+    sampleCount: number;
+    /** Confidence score (0-1) based on response consistency */
+    confidence: number;
+}
+/**
+ * Inferred JSON schema from response samples.
+ */
+export interface InferredSchema {
+    type: string;
+    properties?: Record<string, InferredSchema>;
+    items?: InferredSchema;
+    required?: string[];
+    nullable?: boolean;
+    enum?: unknown[];
+}
+/**
+ * Normalized error pattern for drift detection.
+ */
+export interface ErrorPattern {
+    /** Normalized error category */
+    category: 'validation' | 'not_found' | 'permission' | 'timeout' | 'internal' | 'unknown';
+    /** Pattern hash for comparison */
+    patternHash: string;
+    /** Example error message (first occurrence) */
+    example: string;
+    /** Count of occurrences */
+    count: number;
+}
+/**
+ * Result of analyzing multiple tool responses.
+ */
+export interface ResponseAnalysis {
+    /** Aggregated response fingerprint */
+    fingerprint: ResponseFingerprint;
+    /** Inferred output schema from successful responses */
+    inferredSchema?: InferredSchema;
+    /** Error patterns observed */
+    errorPatterns: ErrorPattern[];
+    /** Enhanced error analyses with root cause and remediation */
+    enhancedErrorAnalyses?: EnhancedErrorAnalysis[];
+    /** Whether responses were consistent across samples */
+    isConsistent: boolean;
+}
+/**
+ * Analyze multiple tool responses to create a comprehensive fingerprint.
+ */
+export declare function analyzeResponses(responses: Array<{
+    response: MCPToolCallResult | null;
+    error: string | null;
+}>): ResponseAnalysis;
+/**
+ * Infer a JSON schema from a sample value.
+ */
+export declare function inferSchemaFromValue(value: unknown): InferredSchema;
+/**
+ * Compare two response fingerprints and return differences.
+ */
+export interface FingerprintDiff {
+    /** Whether the fingerprints are identical */
+    identical: boolean;
+    /** List of changes detected */
+    changes: FingerprintChange[];
+    /** Overall significance of changes */
+    significance: 'none' | 'low' | 'medium' | 'high';
+}
+export interface FingerprintChange {
+    aspect: 'structure' | 'content_type' | 'fields' | 'array_items' | 'size' | 'emptiness';
+    description: string;
+    before: string;
+    after: string;
+    breaking: boolean;
+}
+/**
+ * Compare two response fingerprints.
+ */
+export declare function compareFingerprints(previous: ResponseFingerprint | undefined, current: ResponseFingerprint | undefined): FingerprintDiff;
+/**
+ * Compare error patterns between baselines.
+ */
+export interface ErrorPatternDiff {
+    /** New error patterns that didn't exist before */
+    added: ErrorPattern[];
+    /** Error patterns that no longer occur */
+    removed: ErrorPattern[];
+    /** Whether error behavior changed significantly */
+    behaviorChanged: boolean;
+}
+export declare function compareErrorPatterns(previous: ErrorPattern[] | undefined, current: ErrorPattern[] | undefined): ErrorPatternDiff;
+/**
+ * Compute a hash for the inferred schema for comparison.
+ */
+export declare function computeInferredSchemaHash(schema: InferredSchema | undefined): string;
+//# sourceMappingURL=response-fingerprint.d.ts.map