@dotsetlabs/bellwether 0.10.0

package/dist/verification/verifier.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Verification module for the Verified by Bellwether program.
+ */
+import type { VerificationResult, VerificationReport, VerificationBadge, VerificationConfig } from './types.js';
+import type { InterviewResult } from '../interview/types.js';
+/**
+ * Generate a verification result from an interview result.
+ */
+export declare function generateVerificationResult(interview: InterviewResult, config: VerificationConfig): VerificationResult;
+/**
+ * Generate a full verification report.
+ */
+export declare function generateVerificationReport(interview: InterviewResult, config: VerificationConfig): VerificationReport;
+/**
+ * Generate a verification badge for embedding.
+ */
+export declare function generateVerificationBadge(result: VerificationResult): VerificationBadge;
+/**
+ * Generate a Shields.io compatible badge URL.
+ */
+export declare function generateBadgeUrl(result: VerificationResult): string;
+/**
+ * Generate a verification badge markdown.
+ */
+export declare function generateBadgeMarkdown(result: VerificationResult, reportUrl?: string): string;
+/**
+ * Check if a verification result is still valid.
+ */
+export declare function isVerificationValid(result: VerificationResult): boolean;
+//# sourceMappingURL=verifier.d.ts.map

package/dist/verification/verifier.js

@@ -0,0 +1,309 @@
+/**
+ * Verification module for the Verified by Bellwether program.
+ */
+import { createHash } from 'crypto';
+import { getLogger } from '../logging/logger.js';
+import { VERSION } from '../version.js';
+import { TIME_CONSTANTS, DISPLAY_LIMITS, VERIFICATION_TIERS } from '../constants.js';
+const logger = getLogger('verification');
+/** Verification validity period in days */
+const VERIFICATION_VALIDITY_DAYS = TIME_CONSTANTS.VERIFICATION_VALIDITY_DAYS;
+/**
+ * Generate a verification result from an interview result.
+ */
+export function generateVerificationResult(interview, config) {
+    const now = new Date();
+    const expiresAt = new Date(now.getTime() + VERIFICATION_VALIDITY_DAYS * TIME_CONSTANTS.MS_PER_DAY);
+    // Calculate test statistics
+    const { testsPassed, testsTotal } = calculateTestStats(interview);
+    const passRate = testsTotal > 0 ? Math.round((testsPassed / testsTotal) * 100) : 0;
+    // Determine tier based on coverage and pass rate
+    const tier = determineTier(interview, passRate);
+    // Determine status
+    const status = determineStatus(passRate, tier, config.targetTier);
+    // Generate report hash
+    const reportHash = generateReportHash(interview);
+    const result = {
+        serverId: config.serverId,
+        version: config.version ?? interview.discovery.serverInfo.version,
+        status,
+        tier: status === 'verified' ? tier : undefined,
+        verifiedAt: now.toISOString(),
+        expiresAt: expiresAt.toISOString(),
+        toolsVerified: interview.toolProfiles.length,
+        testsPassed,
+        testsTotal,
+        passRate,
+        reportHash,
+        bellwetherVersion: VERSION,
+    };
+    logger.info({
+        serverId: result.serverId,
+        status: result.status,
+        tier: result.tier,
+        passRate: result.passRate,
+    }, 'Verification result generated');
+    return result;
+}
+/**
+ * Generate a full verification report.
+ */
+export function generateVerificationReport(interview, config) {
+    const result = generateVerificationResult(interview, config);
+    // Build tool verification details
+    const tools = interview.toolProfiles.map(profile => {
+        const passed = profile.interactions.filter(i => !i.error && !i.response?.isError).length;
+        const errors = profile.interactions
+            .filter(i => i.error || i.response?.isError)
+            .map(i => i.error ?? 'Tool returned error');
+        return {
+            name: profile.name,
+            testsRun: profile.interactions.length,
+            testsPassed: passed,
+            errors,
+        };
+    });
+    // Build prompt verification details
+    const prompts = interview.promptProfiles?.map(profile => {
+        const passed = profile.interactions.filter(i => !i.error).length;
+        const errors = profile.interactions
+            .filter(i => i.error)
+            .map(i => i.error ?? 'Prompt returned error');
+        return {
+            name: profile.name,
+            testsRun: profile.interactions.length,
+            testsPassed: passed,
+            errors,
+        };
+    });
+    // Build resource verification details
+    const resources = interview.resourceProfiles?.map(profile => {
+        const passed = profile.interactions.filter(i => !i.error).length;
+        const errors = profile.interactions
+            .filter(i => i.error)
+            .map(i => i.error ?? 'Resource read error');
+        return {
+            uri: profile.uri,
+            name: profile.name,
+            testsRun: profile.interactions.length,
+            testsPassed: passed,
+            errors,
+        };
+    });
+    return {
+        result,
+        serverInfo: {
+            name: interview.discovery.serverInfo.name,
+            version: interview.discovery.serverInfo.version,
+            description: interview.summary,
+        },
+        tools,
+        prompts,
+        resources,
+        environment: {
+            os: process.platform,
+            nodeVersion: process.version,
+            bellwetherVersion: VERSION,
+        },
+    };
+}
+/**
+ * Generate a verification badge for embedding.
+ */
+export function generateVerificationBadge(result) {
+    const badge = {
+        label: 'bellwether',
+        message: 'not verified',
+        color: 'lightgrey',
+    };
+    switch (result.status) {
+        case 'verified':
+            badge.message = result.tier ?? 'verified';
+            badge.color = getTierColor(result.tier);
+            badge.icon = getTierIcon(result.tier);
+            badge.verifiedAt = result.verifiedAt;
+            break;
+        case 'pending':
+            badge.message = 'pending';
+            badge.color = 'yellow';
+            break;
+        case 'failed':
+            badge.message = 'failed';
+            badge.color = 'red';
+            break;
+        case 'expired':
+            badge.message = 'expired';
+            badge.color = 'orange';
+            break;
+    }
+    return badge;
+}
+/**
+ * Generate a Shields.io compatible badge URL.
+ */
+export function generateBadgeUrl(result) {
+    const badge = generateVerificationBadge(result);
+    const encodedLabel = encodeURIComponent(badge.label);
+    const encodedMessage = encodeURIComponent(badge.message);
+    const encodedColor = encodeURIComponent(badge.color);
+    return `https://img.shields.io/badge/${encodedLabel}-${encodedMessage}-${encodedColor}`;
+}
+/**
+ * Generate a verification badge markdown.
+ */
+export function generateBadgeMarkdown(result, reportUrl) {
+    const badgeUrl = generateBadgeUrl(result);
+    const altText = `Bellwether ${result.status}: ${result.tier ?? result.status}`;
+    if (reportUrl) {
+        return `[![${altText}](${badgeUrl})](${reportUrl})`;
+    }
+    return `![${altText}](${badgeUrl})`;
+}
+/**
+ * Check if a verification result is still valid.
+ */
+export function isVerificationValid(result) {
+    const expiresAt = new Date(result.expiresAt);
+    return result.status === 'verified' && expiresAt > new Date();
+}
+/**
+ * Calculate test statistics from interview results.
+ */
+function calculateTestStats(interview) {
+    let testsPassed = 0;
+    let testsTotal = 0;
+    const toolStats = new Map();
+    // Count tool tests
+    for (const profile of interview.toolProfiles) {
+        const total = profile.interactions.length;
+        const passed = profile.interactions.filter(i => !i.error && !i.response?.isError).length;
+        toolStats.set(profile.name, { passed, total });
+        testsPassed += passed;
+        testsTotal += total;
+    }
+    // Count prompt tests
+    if (interview.promptProfiles) {
+        for (const profile of interview.promptProfiles) {
+            const total = profile.interactions.length;
+            const passed = profile.interactions.filter(i => !i.error).length;
+            testsPassed += passed;
+            testsTotal += total;
+        }
+    }
+    // Count resource tests
+    if (interview.resourceProfiles) {
+        for (const profile of interview.resourceProfiles) {
+            const total = profile.interactions.length;
+            const passed = profile.interactions.filter(i => !i.error).length;
+            testsPassed += passed;
+            testsTotal += total;
+        }
+    }
+    // Count scenario tests
+    if (interview.scenarioResults) {
+        for (const result of interview.scenarioResults) {
+            testsTotal++;
+            if (result.passed) {
+                testsPassed++;
+            }
+        }
+    }
+    return { testsPassed, testsTotal, toolStats };
+}
+function determineTier(interview, passRate) {
+    const hasSecurityTesting = interview.metadata.personas?.some(p => p.name.toLowerCase().includes('security'));
+    const personaCount = interview.metadata.personas?.length ?? 1;
+    const hasPrompts = (interview.promptProfiles?.length ?? 0) > 0;
+    const hasResources = (interview.resourceProfiles?.length ?? 0) > 0;
+    // Platinum: Security testing + all personas + high pass rate
+    if (hasSecurityTesting &&
+        personaCount >= VERIFICATION_TIERS.PLATINUM.MIN_PERSONAS &&
+        passRate >= VERIFICATION_TIERS.PLATINUM.MIN_PASS_RATE) {
+        return 'platinum';
+    }
+    // Gold: Multiple personas + good coverage + high pass rate
+    if (personaCount >= VERIFICATION_TIERS.GOLD.MIN_PERSONAS &&
+        passRate >= VERIFICATION_TIERS.GOLD.MIN_PASS_RATE &&
+        (hasPrompts || hasResources)) {
+        return 'gold';
+    }
+    // Silver: Error handling tested + decent pass rate
+    if (personaCount >= VERIFICATION_TIERS.SILVER.MIN_PERSONAS &&
+        passRate >= VERIFICATION_TIERS.SILVER.MIN_PASS_RATE) {
+        return 'silver';
+    }
+    // Bronze: Basic testing
+    return 'bronze';
+}
+/**
+ * Determine verification status based on results.
+ */
+function determineStatus(passRate, achievedTier, targetTier) {
+    // Minimum pass rate for verification
+    if (passRate < VERIFICATION_TIERS.MIN_PASS_RATE_FOR_VERIFICATION) {
+        return 'failed';
+    }
+    // Check if target tier is met
+    if (targetTier) {
+        const tierOrder = ['bronze', 'silver', 'gold', 'platinum'];
+        const achievedIndex = tierOrder.indexOf(achievedTier);
+        const targetIndex = tierOrder.indexOf(targetTier);
+        if (achievedIndex < targetIndex) {
+            return 'failed';
+        }
+    }
+    return 'verified';
+}
+/**
+ * Generate a hash of the interview results for verification.
+ */
+function generateReportHash(interview) {
+    const data = {
+        serverName: interview.discovery.serverInfo.name,
+        serverVersion: interview.discovery.serverInfo.version,
+        toolCount: interview.toolProfiles.length,
+        toolNames: interview.toolProfiles.map(p => p.name).sort(),
+        promptCount: interview.promptProfiles?.length ?? 0,
+        resourceCount: interview.resourceProfiles?.length ?? 0,
+        timestamp: interview.metadata.startTime.toISOString(),
+    };
+    return createHash('sha256')
+        .update(JSON.stringify(data))
+        .digest('hex')
+        .substring(0, DISPLAY_LIMITS.HASH_DISPLAY_LENGTH);
+}
+/**
+ * Get color for a verification tier.
+ */
+function getTierColor(tier) {
+    switch (tier) {
+        case 'platinum':
+            return '00CED1'; // Dark cyan
+        case 'gold':
+            return 'FFD700'; // Gold
+        case 'silver':
+            return 'C0C0C0'; // Silver
+        case 'bronze':
+            return 'CD7F32'; // Bronze
+        default:
+            return 'brightgreen';
+    }
+}
+/**
+ * Get icon for a verification tier.
+ */
+function getTierIcon(tier) {
+    switch (tier) {
+        case 'platinum':
+            return '💎';
+        case 'gold':
+            return '🥇';
+        case 'silver':
+            return '🥈';
+        case 'bronze':
+            return '🥉';
+        default:
+            return '✓';
+    }
+}
+//# sourceMappingURL=verifier.js.map

package/dist/version.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Centralized version management.
+ *
+ * All version references should import from this module rather than
+ * hardcoding the version string.
+ */
+/**
+ * The current Bellwether version.
+ */
+export declare const VERSION: string;
+/**
+ * Package name.
+ */
+export declare const PACKAGE_NAME = "@dotsetlabs/bellwether";
+/**
+ * User-Agent string for HTTP requests.
+ */
+export declare const USER_AGENT: string;
+//# sourceMappingURL=version.d.ts.map

package/dist/version.js

@@ -0,0 +1,48 @@
+/**
+ * Centralized version management.
+ *
+ * All version references should import from this module rather than
+ * hardcoding the version string.
+ */
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+import { readFileSync } from 'fs';
+/**
+ * Get the package version.
+ *
+ * This reads from package.json at runtime, which works correctly
+ * whether running in development or as an installed global package.
+ */
+function getPackageVersion() {
+    try {
+        // First try npm_package_version (works when run via npm scripts)
+        if (process.env.npm_package_version) {
+            return process.env.npm_package_version;
+        }
+        // Otherwise read from package.json relative to this module
+        const __filename = fileURLToPath(import.meta.url);
+        const __dirname = dirname(__filename);
+        // In dist/, we need to go up one level to find package.json
+        // In src/, we also need to go up one level
+        const packagePath = join(__dirname, '..', 'package.json');
+        const packageJson = JSON.parse(readFileSync(packagePath, 'utf-8'));
+        return packageJson.version;
+    }
+    catch {
+        // Fallback version - should match package.json
+        return '0.10.0';
+    }
+}
+/**
+ * The current Bellwether version.
+ */
+export const VERSION = getPackageVersion();
+/**
+ * Package name.
+ */
+export const PACKAGE_NAME = '@dotsetlabs/bellwether';
+/**
+ * User-Agent string for HTTP requests.
+ */
+export const USER_AGENT = `bellwether/${VERSION}`;
+//# sourceMappingURL=version.js.map

package/dist/workflow/auto-generator.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Workflow auto-generator - generates workflow YAML from discovered tools.
+ *
+ * This module analyzes tool descriptions and schemas to automatically
+ * generate test workflows that represent common usage patterns.
+ * Unlike the LLM-powered WorkflowDiscoverer, this uses heuristics
+ * for deterministic workflow generation in check mode.
+ */
+import type { MCPTool } from '../transport/types.js';
+import type { Workflow, WorkflowDiscoveryOptions } from './types.js';
+/**
+ * Generate workflows from available tools using pattern matching.
+ *
+ * This function analyzes tool names, descriptions, and schemas to
+ * identify common workflow patterns like CRUD operations, auth flows,
+ * and chained data processing.
+ *
+ * @param tools - Available MCP tools
+ * @param options - Generation options
+ * @returns Array of generated workflows
+ */
+export declare function generateWorkflowsFromTools(tools: MCPTool[], options?: WorkflowDiscoveryOptions): Workflow[];
+/**
+ * Generate YAML content for workflows.
+ */
+export declare function generateWorkflowYamlContent(workflows: Workflow[]): string;
+//# sourceMappingURL=auto-generator.d.ts.map