@dotsetlabs/bellwether 0.10.0

package/dist/baseline/converter.js

@@ -0,0 +1,585 @@
+/**
+ * Baseline format converter.
+ *
+ * Converts between local BehavioralBaseline format and cloud BellwetherBaseline format.
+ *
+ * ## Severity Type Mappings
+ *
+ * The codebase uses three different severity type systems for different contexts:
+ *
+ * ### ChangeSeverity (baseline/types.ts)
+ * Used for drift detection change classification. Maps to CLI exit codes.
+ * Values: 'none' | 'info' | 'warning' | 'breaking'
+ *
+ * ### ErrorSeverity (errors/types.ts)
+ * Used for error severity classification in error handling.
+ * Values: 'low' | 'medium' | 'high' | 'critical'
+ *
+ * ### CloudAssertionSeverity (cloud/types.ts)
+ * Used for cloud assertions and PersonaFinding severity levels.
+ * Values: 'info' | 'low' | 'medium' | 'high' | 'critical'
+ *
+ * ### Conversion Mappings
+ *
+ * ChangeSeverity → CloudAssertionSeverity:
+ * - 'none'     → 'info'     (no change, informational)
+ * - 'info'     → 'low'      (minor changes)
+ * - 'warning'  → 'medium'   (moderate changes)
+ * - 'breaking' → 'critical' (breaking changes)
+ *
+ * CloudAssertionSeverity → ChangeSeverity (for display/filtering):
+ * - 'info'     → 'info'
+ * - 'low'      → 'info'
+ * - 'medium'   → 'warning'
+ * - 'high'     → 'warning'
+ * - 'critical' → 'breaking'
+ */
+import { createHash } from 'crypto';
+import { analyzeResponses } from './response-fingerprint.js';
+import { getBaselineVersion } from './version.js';
+import { VERSION } from '../version.js';
+/**
+ * Map ChangeSeverity to CloudAssertionSeverity.
+ * Used when converting local baselines to cloud format.
+ */
+export const CHANGE_TO_CLOUD_SEVERITY = {
+    none: 'info',
+    info: 'low',
+    warning: 'medium',
+    breaking: 'critical',
+};
+/**
+ * Map CloudAssertionSeverity to ChangeSeverity.
+ * Used when filtering or displaying cloud data locally.
+ */
+export const CLOUD_TO_CHANGE_SEVERITY = {
+    info: 'info',
+    low: 'info',
+    medium: 'warning',
+    high: 'warning',
+    critical: 'breaking',
+};
+/**
+ * Hash a string using SHA-256.
+ */
+function hashString(input) {
+    return createHash('sha256').update(input).digest('hex').slice(0, 16);
+}
+/**
+ * Convert a local BehavioralAssertion to cloud CloudAssertion format.
+ *
+ * Mapping:
+ * - isPositive=true + security aspect → 'requires' (critical security requirement)
+ * - isPositive=true + other aspect → 'expects' (expected behavior)
+ * - isPositive=false + security aspect → 'warns' (security warning)
+ * - isPositive=false + other aspect → 'notes' (limitation/note)
+ */
+function convertAssertion(assertion) {
+    // Determine assertion type based on isPositive and aspect
+    let type;
+    if (assertion.isPositive) {
+        type = assertion.aspect === 'security' ? 'requires' : 'expects';
+    }
+    else {
+        type = assertion.aspect === 'security' ? 'warns' : 'notes';
+    }
+    // Determine severity based on aspect and content
+    let severity = 'info';
+    const lowerAssertion = assertion.assertion.toLowerCase();
+    if (assertion.aspect === 'security') {
+        if (lowerAssertion.includes('critical') ||
+            lowerAssertion.includes('injection') ||
+            lowerAssertion.includes('rce')) {
+            severity = 'critical';
+        }
+        else if (lowerAssertion.includes('high') ||
+            lowerAssertion.includes('dangerous') ||
+            lowerAssertion.includes('exploit')) {
+            severity = 'high';
+        }
+        else if (lowerAssertion.includes('medium') ||
+            lowerAssertion.includes('sensitive') ||
+            lowerAssertion.includes('leak')) {
+            severity = 'medium';
+        }
+        else {
+            severity = 'low';
+        }
+    }
+    else if (assertion.aspect === 'error_handling') {
+        severity = assertion.isPositive ? 'info' : 'low';
+    }
+    else if (assertion.aspect === 'performance') {
+        severity = 'medium';
+    }
+    return {
+        type,
+        condition: assertion.assertion,
+        tool: assertion.tool,
+        severity,
+    };
+}
+/**
+ * Convert an array of BehavioralAssertions to CloudAssertions.
+ */
+function convertAssertions(assertions) {
+    return assertions.map(convertAssertion);
+}
+/**
+ * Derive baseline mode from result metadata.
+ * Returns 'check' for check mode results, 'explore' for explore mode results.
+ * Note: Baselines should only be created from check mode results,
+ * but explore uploads are still supported for documentation tracking.
+ */
+function deriveCloudMode(resultModel) {
+    // Check mode results have model === 'check'
+    if (resultModel === 'check')
+        return 'check';
+    // Everything else (explore mode with LLM model names) is explore
+    return 'explore';
+}
+/**
+ * Convert a BehavioralBaseline to cloud BellwetherBaseline format.
+ */
+export function convertToCloudBaseline(baseline, discovery, interviewResult) {
+    // Derive mode from result metadata
+    const mode = deriveCloudMode(interviewResult?.metadata.model);
+    // Build metadata
+    const metadata = {
+        mode,
+        generatedAt: baseline.createdAt.toISOString(),
+        cliVersion: VERSION,
+        serverCommand: baseline.serverCommand,
+        serverName: baseline.server.name,
+        durationMs: interviewResult?.metadata.durationMs ?? 0,
+        personas: extractPersonas(interviewResult),
+        model: interviewResult?.metadata.model ?? 'unknown',
+    };
+    // Build server fingerprint
+    const server = {
+        name: baseline.server.name,
+        version: baseline.server.version,
+        protocolVersion: baseline.server.protocolVersion,
+        capabilities: baseline.server.capabilities,
+    };
+    // Build capabilities
+    const capabilities = buildCapabilities(baseline, discovery);
+    // Build interviews
+    const interviews = buildInterviews(interviewResult);
+    // Build tool profiles (with converted assertions)
+    const toolProfiles = baseline.tools.map(convertToolFingerprint);
+    // Build workflows
+    const workflows = baseline.workflowSignatures;
+    // Convert assertions to cloud format
+    const assertions = convertAssertions(baseline.assertions);
+    // Build content hash
+    const contentForHash = JSON.stringify({
+        metadata,
+        server,
+        capabilities,
+        interviews,
+        toolProfiles,
+        workflows,
+        assertions,
+        summary: baseline.summary,
+    });
+    const hash = hashString(contentForHash);
+    return {
+        version: getBaselineVersion(),
+        metadata,
+        server,
+        capabilities,
+        interviews,
+        toolProfiles,
+        workflows,
+        assertions,
+        summary: baseline.summary,
+        hash,
+    };
+}
+/**
+ * Extract persona names from interview result.
+ */
+function extractPersonas(result) {
+    if (!result?.metadata.personas) {
+        return ['technical_writer']; // Default persona
+    }
+    return result.metadata.personas.map((p) => p.id);
+}
+/**
+ * Build capabilities from baseline and discovery.
+ */
+function buildCapabilities(baseline, discovery) {
+    // Build tool capabilities
+    // Prefer discovery schema, fall back to stored inputSchema from baseline
+    // Include response fingerprinting data from baseline
+    const tools = baseline.tools.map((tool) => ({
+        name: tool.name,
+        description: tool.description,
+        inputSchema: discovery
+            ? getToolSchema(discovery, tool.name)
+            : (tool.inputSchema ?? {}),
+        schemaHash: tool.schemaHash,
+        // Response fingerprinting (check mode enhancement)
+        responseFingerprint: tool.responseFingerprint,
+        inferredOutputSchema: tool.inferredOutputSchema,
+        errorPatterns: tool.errorPatterns,
+    }));
+    // Build resource capabilities (from discovery if available)
+    let resources;
+    // Note: Resources would come from discovery.resources if we had them
+    // Build prompt capabilities (from discovery if available)
+    let prompts;
+    if (discovery?.prompts && discovery.prompts.length > 0) {
+        prompts = discovery.prompts.map((p) => ({
+            name: p.name,
+            description: p.description,
+            arguments: p.arguments?.map((a) => ({
+                name: a.name,
+                description: a.description,
+                required: a.required,
+            })),
+        }));
+    }
+    return {
+        tools,
+        resources,
+        prompts,
+    };
+}
+/**
+ * Get tool schema from discovery.
+ */
+function getToolSchema(discovery, toolName) {
+    const tool = discovery.tools.find((t) => t.name === toolName);
+    return tool?.inputSchema ?? {};
+}
+/**
+ * Build interview summaries from interview result.
+ */
+function buildInterviews(result) {
+    if (!result?.metadata.personas) {
+        // Create a default technical_writer interview
+        const totalQuestions = result?.toolProfiles.reduce((sum, p) => sum + p.interactions.length, 0) ?? 0;
+        return [
+            {
+                persona: 'technical_writer',
+                toolsInterviewed: result?.toolProfiles.length ?? 0,
+                questionsAsked: totalQuestions,
+                findings: extractFindings(result?.toolProfiles ?? []),
+            },
+        ];
+    }
+    // Build interviews per persona
+    return result.metadata.personas.map((persona) => {
+        const personaInteractions = result.toolProfiles.flatMap((profile) => profile.interactions.filter((i) => i.personaId === persona.id));
+        const toolsWithPersonaInteractions = new Set(personaInteractions.map((i) => i.toolName));
+        return {
+            persona: persona.id,
+            toolsInterviewed: toolsWithPersonaInteractions.size,
+            questionsAsked: persona.questionsAsked,
+            findings: extractFindingsForPersona(result.toolProfiles, persona.id),
+        };
+    });
+}
+/**
+ * Extract findings from tool profiles.
+ */
+function extractFindings(toolProfiles) {
+    const findings = [];
+    for (const profile of toolProfiles) {
+        // Add security findings
+        for (const note of profile.securityNotes) {
+            findings.push({
+                tool: profile.name,
+                category: 'security',
+                severity: classifySeverity(note),
+                description: note,
+            });
+        }
+        // Add limitation findings
+        for (const limitation of profile.limitations) {
+            findings.push({
+                tool: profile.name,
+                category: 'reliability',
+                severity: 'low',
+                description: limitation,
+            });
+        }
+        // Add behavioral findings (first few as behavior category)
+        for (const note of profile.behavioralNotes.slice(0, 3)) {
+            findings.push({
+                tool: profile.name,
+                category: 'behavior',
+                severity: 'info',
+                description: note,
+            });
+        }
+    }
+    return findings;
+}
+/**
+ * Extract findings for a specific persona.
+ */
+function extractFindingsForPersona(toolProfiles, personaId) {
+    const findings = [];
+    for (const profile of toolProfiles) {
+        const personaFindings = profile.findingsByPersona?.find((f) => f.personaId === personaId);
+        if (!personaFindings)
+            continue;
+        // Add security findings
+        for (const note of personaFindings.securityNotes) {
+            findings.push({
+                tool: profile.name,
+                category: 'security',
+                severity: classifySeverity(note),
+                description: note,
+            });
+        }
+        // Add limitation findings
+        for (const limitation of personaFindings.limitations) {
+            findings.push({
+                tool: profile.name,
+                category: 'reliability',
+                severity: 'low',
+                description: limitation,
+            });
+        }
+        // Add behavioral findings
+        for (const note of personaFindings.behavioralNotes.slice(0, 3)) {
+            findings.push({
+                tool: profile.name,
+                category: 'behavior',
+                severity: 'info',
+                description: note,
+            });
+        }
+    }
+    return findings;
+}
+/**
+ * Classify severity based on note content.
+ */
+function classifySeverity(note) {
+    const lowerNote = note.toLowerCase();
+    if (lowerNote.includes('critical') ||
+        lowerNote.includes('severe') ||
+        lowerNote.includes('injection') ||
+        lowerNote.includes('rce') ||
+        lowerNote.includes('remote code')) {
+        return 'critical';
+    }
+    if (lowerNote.includes('high') ||
+        lowerNote.includes('dangerous') ||
+        lowerNote.includes('exploit') ||
+        lowerNote.includes('bypass')) {
+        return 'high';
+    }
+    if (lowerNote.includes('medium') ||
+        lowerNote.includes('moderate') ||
+        lowerNote.includes('sensitive') ||
+        lowerNote.includes('leak')) {
+        return 'medium';
+    }
+    if (lowerNote.includes('low') ||
+        lowerNote.includes('minor') ||
+        lowerNote.includes('potential')) {
+        return 'low';
+    }
+    return 'info';
+}
+/**
+ * Convert ToolFingerprint to CloudToolProfile.
+ */
+function convertToolFingerprint(tool) {
+    return {
+        name: tool.name,
+        description: tool.description,
+        schemaHash: tool.schemaHash,
+        assertions: convertAssertions(tool.assertions),
+        securityNotes: tool.securityNotes,
+        limitations: tool.limitations,
+        behavioralNotes: tool.assertions
+            .filter((a) => a.aspect === 'response_format' && a.isPositive)
+            .map((a) => a.assertion),
+    };
+}
+/**
+ * Create a BellwetherBaseline directly from InterviewResult.
+ *
+ * This is the preferred method when you have fresh interview results.
+ */
+export function createCloudBaseline(result, serverCommand) {
+    // Derive mode from result metadata
+    const mode = deriveCloudMode(result.metadata.model);
+    // Build metadata
+    const metadata = {
+        mode,
+        generatedAt: new Date().toISOString(),
+        cliVersion: VERSION,
+        serverCommand,
+        serverName: result.discovery.serverInfo.name,
+        durationMs: result.metadata.durationMs,
+        personas: result.metadata.personas?.map((p) => p.id) ?? ['technical_writer'],
+        model: result.metadata.model ?? 'unknown',
+    };
+    // Build server fingerprint
+    const server = {
+        name: result.discovery.serverInfo.name,
+        version: result.discovery.serverInfo.version,
+        protocolVersion: result.discovery.protocolVersion,
+        capabilities: buildCapabilityList(result.discovery),
+    };
+    // Build response fingerprint map from tool profiles
+    const fingerprintMap = new Map();
+    for (const profile of result.toolProfiles) {
+        const responseData = profile.interactions.map(i => ({
+            response: i.response,
+            error: i.error,
+        }));
+        fingerprintMap.set(profile.name, analyzeResponses(responseData));
+    }
+    // Build capabilities (including response fingerprinting)
+    const tools = result.discovery.tools.map((tool) => {
+        const analysis = fingerprintMap.get(tool.name);
+        return {
+            name: tool.name,
+            description: tool.description ?? '',
+            inputSchema: tool.inputSchema ?? {},
+            schemaHash: hashString(JSON.stringify(tool.inputSchema ?? {})),
+            // Response fingerprinting (check mode enhancement)
+            responseFingerprint: analysis?.fingerprint,
+            inferredOutputSchema: analysis?.inferredSchema,
+            errorPatterns: analysis?.errorPatterns.length ? analysis.errorPatterns : undefined,
+        };
+    });
+    const prompts = result.discovery.prompts.length > 0
+        ? result.discovery.prompts.map((p) => ({
+            name: p.name,
+            description: p.description,
+            arguments: p.arguments?.map((a) => ({
+                name: a.name,
+                description: a.description,
+                required: a.required,
+            })),
+        }))
+        : undefined;
+    // Build interviews
+    const interviews = buildInterviews(result);
+    // Build tool profiles (with converted assertions)
+    const toolProfiles = result.toolProfiles.map((profile) => ({
+        name: profile.name,
+        description: profile.description,
+        schemaHash: hashString(JSON.stringify(result.discovery.tools.find((t) => t.name === profile.name)?.inputSchema ?? {})),
+        assertions: convertAssertions(extractToolAssertions(profile)),
+        securityNotes: profile.securityNotes,
+        limitations: profile.limitations,
+        behavioralNotes: profile.behavioralNotes,
+    }));
+    // Build workflows
+    const workflows = result.workflowResults?.map((wr) => ({
+        id: wr.workflow.id,
+        name: wr.workflow.name,
+        toolSequence: wr.workflow.steps.map((s) => s.tool),
+        succeeded: wr.success,
+        summary: wr.summary,
+    }));
+    // Build assertions (convert to cloud format)
+    const assertions = convertAssertions(extractAllAssertions(result));
+    // Build content hash
+    const contentForHash = JSON.stringify({
+        metadata,
+        server,
+        capabilities: { tools, prompts },
+        interviews,
+        toolProfiles,
+        workflows,
+        assertions,
+        summary: result.summary,
+    });
+    const hash = hashString(contentForHash);
+    return {
+        version: getBaselineVersion(),
+        metadata,
+        server,
+        capabilities: { tools, prompts },
+        interviews,
+        toolProfiles,
+        workflows,
+        assertions,
+        summary: result.summary,
+        hash,
+    };
+}
+/**
+ * Build capability list from discovery.
+ */
+function buildCapabilityList(discovery) {
+    const capabilities = [];
+    if (discovery.capabilities.tools)
+        capabilities.push('tools');
+    if (discovery.capabilities.prompts)
+        capabilities.push('prompts');
+    if (discovery.capabilities.resources)
+        capabilities.push('resources');
+    if (discovery.capabilities.logging)
+        capabilities.push('logging');
+    return capabilities;
+}
+/**
+ * Extract behavioral assertions from a tool profile.
+ */
+function extractToolAssertions(profile) {
+    const assertions = [];
+    // Behavioral notes as positive assertions
+    for (const note of profile.behavioralNotes) {
+        assertions.push({
+            tool: profile.name,
+            aspect: 'response_format',
+            assertion: note,
+            isPositive: true,
+        });
+    }
+    // Limitations as negative assertions
+    for (const limitation of profile.limitations) {
+        assertions.push({
+            tool: profile.name,
+            aspect: 'error_handling',
+            assertion: limitation,
+            isPositive: false,
+        });
+    }
+    // Security notes as security assertions
+    for (const secNote of profile.securityNotes) {
+        assertions.push({
+            tool: profile.name,
+            aspect: 'security',
+            assertion: secNote,
+            isPositive: !secNote.toLowerCase().includes('risk') &&
+                !secNote.toLowerCase().includes('vulnerab') &&
+                !secNote.toLowerCase().includes('dangerous'),
+        });
+    }
+    return assertions;
+}
+/**
+ * Extract all assertions from interview result.
+ */
+function extractAllAssertions(result) {
+    const assertions = [];
+    // Extract from each tool
+    for (const profile of result.toolProfiles) {
+        assertions.push(...extractToolAssertions(profile));
+    }
+    // Add overall limitations as server assertions
+    for (const limitation of result.limitations) {
+        assertions.push({
+            tool: 'server',
+            aspect: 'error_handling',
+            assertion: limitation,
+            isPositive: false,
+        });
+    }
+    return assertions;
+}
+//# sourceMappingURL=converter.js.map

package/dist/baseline/dependency-analyzer.d.ts

@@ -0,0 +1,89 @@
+/**
+ * Cross-Tool Dependency Analyzer.
+ *
+ * Analyzes tool descriptions and schemas to identify dependencies
+ * between tools, enabling better testing strategies and documentation.
+ *
+ * Detection strategies:
+ * 1. Description analysis - mentions of other tool names
+ * 2. Parameter name matching - params that match output field names
+ * 3. Resource reference patterns - common ID/token patterns
+ * 4. Workflow step order - sequence implied by tool naming
+ */
+import type { MCPTool } from '../transport/types.js';
+/**
+ * A dependency edge between two tools.
+ */
+export interface DependencyEdge {
+    /** Source tool that provides data */
+    from: string;
+    /** Target tool that consumes data */
+    to: string;
+    /** Type of dependency relationship */
+    type: DependencyType;
+    /** Confidence level of this dependency detection (0-1) */
+    confidence: number;
+    /** Description of the relationship */
+    description: string;
+    /** Which field/parameter creates the dependency */
+    field?: string;
+}
+/**
+ * Types of dependencies between tools.
+ */
+export type DependencyType = 'mention' | 'output_input' | 'resource_ref' | 'sequence' | 'shared_resource';
+/**
+ * Full dependency graph for a set of tools.
+ */
+export interface DependencyGraph {
+    /** All dependency edges */
+    edges: DependencyEdge[];
+    /** Tools grouped by layer (no dependencies -> most dependencies) */
+    layers: string[][];
+    /** Tools with no dependencies (entry points) */
+    entryPoints: string[];
+    /** Tools with no dependents (terminal operations) */
+    terminalPoints: string[];
+    /** Detected cycles (if any) */
+    cycles: string[][];
+}
+/**
+ * Statistics about the dependency graph.
+ */
+export interface DependencyStats {
+    /** Total number of edges */
+    totalEdges: number;
+    /** Edges by type */
+    byType: Record<DependencyType, number>;
+    /** Average dependencies per tool */
+    avgDependencies: number;
+    /** Maximum dependency chain length */
+    maxChainLength: number;
+    /** Tools with most dependencies */
+    mostDependent: Array<{
+        tool: string;
+        count: number;
+    }>;
+    /** Tools most depended upon */
+    mostDependedUpon: Array<{
+        tool: string;
+        count: number;
+    }>;
+}
+/**
+ * Analyze dependencies between tools.
+ */
+export declare function analyzeDependencies(tools: MCPTool[]): DependencyGraph;
+/**
+ * Calculate statistics about the dependency graph.
+ */
+export declare function calculateDependencyStats(graph: DependencyGraph): DependencyStats;
+/**
+ * Generate a Mermaid diagram for the dependency graph.
+ */
+export declare function generateDependencyMermaid(graph: DependencyGraph): string;
+/**
+ * Generate markdown documentation for dependencies.
+ */
+export declare function generateDependencyMarkdown(graph: DependencyGraph, stats: DependencyStats): string;
+//# sourceMappingURL=dependency-analyzer.d.ts.map