@dotsetlabs/bellwether 0.10.0

package/dist/docs/shared.d.ts

@@ -0,0 +1,35 @@
+import type { ToolProfile } from '../interview/types.js';
+import type { PerformanceConfidence } from '../baseline/types.js';
+/**
+ * Performance metrics for a single tool.
+ */
+export interface ToolPerformanceMetrics {
+    toolName: string;
+    callCount: number;
+    avgMs: number;
+    minMs: number;
+    maxMs: number;
+    p50Ms: number;
+    p95Ms: number;
+    stdDevMs: number;
+    errorRate: number;
+    /** Average time for tool execution only (MCP transport) */
+    avgToolMs?: number;
+    /** Average time for LLM analysis only */
+    avgAnalysisMs?: number;
+    /** Statistical confidence metrics */
+    confidence?: PerformanceConfidence;
+}
+/**
+ * Extract parameter list from schema.
+ */
+export declare function extractParameters(schema: Record<string, unknown> | undefined): string;
+/**
+ * Heuristic detection of error responses for summarization.
+ */
+export declare function looksLikeError(text: string): boolean;
+/**
+ * Calculate performance metrics for all tools.
+ */
+export declare function calculatePerformanceMetrics(profiles: ToolProfile[]): ToolPerformanceMetrics[];
+//# sourceMappingURL=shared.d.ts.map

package/dist/docs/shared.js

@@ -0,0 +1,162 @@
+import { PERFORMANCE_CONFIDENCE } from '../constants.js';
+/**
+ * Extract parameter list from schema.
+ */
+export function extractParameters(schema) {
+    if (!schema || typeof schema !== 'object') {
+        return '*none*';
+    }
+    const properties = schema.properties;
+    if (!properties || Object.keys(properties).length === 0) {
+        return '*none*';
+    }
+    const required = schema.required || [];
+    const params = Object.entries(properties).map(([name, prop]) => {
+        const propObj = prop;
+        const type = propObj.type || 'any';
+        const isRequired = required.includes(name);
+        return `\`${name}\`${isRequired ? '*' : ''}: ${type}`;
+    });
+    return params.slice(0, 3).join(', ') + (params.length > 3 ? ', ...' : '');
+}
+/**
+ * Heuristic detection of error responses for summarization.
+ */
+export function looksLikeError(text) {
+    const errorPatterns = [
+        /^error\s*[-:]/i,
+        /:\s*error\s*-/i,
+        /access denied/i,
+        /permission denied/i,
+        /not allowed/i,
+        /outside allowed/i,
+        /outside.*(predefined|allowed)/i,
+        /path outside/i,
+        /invalid path/i,
+        /failed to/i,
+        /could not/i,
+        /unable to/i,
+        /cannot\s+(access|read|write|create)/i,
+        /restricted to/i,
+    ];
+    return errorPatterns.some((pattern) => pattern.test(text));
+}
+/**
+ * Calculate performance metrics for all tools.
+ */
+export function calculatePerformanceMetrics(profiles) {
+    const metrics = [];
+    for (const profile of profiles) {
+        const interactions = profile.interactions.filter(i => !i.mocked);
+        if (interactions.length === 0) {
+            continue;
+        }
+        const durations = interactions.map(i => i.durationMs);
+        const errorCount = interactions.filter(i => i.error || i.response?.isError).length;
+        // Sort durations for percentile calculations
+        const sortedDurations = [...durations].sort((a, b) => a - b);
+        const p50Index = Math.floor(sortedDurations.length * 0.5);
+        const p95Index = Math.floor(sortedDurations.length * 0.95);
+        // Calculate separate averages for tool execution and LLM analysis
+        const toolDurations = interactions
+            .map(i => i.toolExecutionMs)
+            .filter((d) => d !== undefined && d > 0);
+        const analysisDurations = interactions
+            .map(i => i.llmAnalysisMs)
+            .filter((d) => d !== undefined && d > 0);
+        const avgToolMs = toolDurations.length > 0
+            ? Math.round(toolDurations.reduce((a, b) => a + b, 0) / toolDurations.length)
+            : undefined;
+        const avgAnalysisMs = analysisDurations.length > 0
+            ? Math.round(analysisDurations.reduce((a, b) => a + b, 0) / analysisDurations.length)
+            : undefined;
+        // Calculate average and standard deviation
+        const avgMs = durations.reduce((a, b) => a + b, 0) / durations.length;
+        const squaredDiffs = durations.map(d => Math.pow(d - avgMs, 2));
+        const variance = squaredDiffs.reduce((a, b) => a + b, 0) / durations.length;
+        const stdDevMs = Math.sqrt(variance);
+        // Calculate coefficient of variation and confidence using successful tool executions only
+        const successfulToolDurations = interactions
+            .filter(i => i.toolExecutionMs !== undefined && !i.error && !i.response?.isError)
+            .map(i => i.toolExecutionMs);
+        let confidenceCV;
+        let confidenceStdDev;
+        let confidenceSampleCount;
+        if (successfulToolDurations.length === 0) {
+            // All calls failed - low confidence
+            confidenceCV = 0;
+            confidenceStdDev = 0;
+            confidenceSampleCount = 0;
+        }
+        else {
+            const successMean = successfulToolDurations.reduce((a, b) => a + b, 0) / successfulToolDurations.length;
+            const successVariance = successfulToolDurations.reduce((sum, d) => sum + Math.pow(d - successMean, 2), 0) / successfulToolDurations.length;
+            confidenceStdDev = Math.sqrt(successVariance);
+            confidenceCV = successMean > 0 ? confidenceStdDev / successMean : 0;
+            confidenceSampleCount = successfulToolDurations.length;
+        }
+        const confidence = calculateConfidenceFromMetrics(confidenceSampleCount, confidenceCV, confidenceStdDev);
+        metrics.push({
+            toolName: profile.name,
+            callCount: interactions.length,
+            avgMs: Math.round(avgMs),
+            minMs: Math.min(...durations),
+            maxMs: Math.max(...durations),
+            p50Ms: sortedDurations[p50Index] ?? 0,
+            p95Ms: sortedDurations[Math.min(p95Index, sortedDurations.length - 1)] ?? 0,
+            stdDevMs: Math.round(stdDevMs),
+            errorRate: errorCount / interactions.length,
+            avgToolMs,
+            avgAnalysisMs,
+            confidence,
+        });
+    }
+    // Sort by avg response time (slowest first)
+    return metrics.sort((a, b) => b.avgMs - a.avgMs);
+}
+function calculateConfidenceFromMetrics(sampleCount, coefficientOfVariation, standardDeviation, totalTests) {
+    // Handle no samples case
+    if (sampleCount === 0) {
+        return {
+            sampleCount: 0,
+            successfulSamples: 0,
+            validationSamples: 0,
+            totalTests: totalTests ?? 0,
+            standardDeviation: 0,
+            coefficientOfVariation: 0,
+            confidenceLevel: 'low',
+            recommendation: PERFORMANCE_CONFIDENCE.RECOMMENDATIONS.NO_SAMPLES,
+        };
+    }
+    // Determine confidence level
+    let confidenceLevel;
+    let recommendation;
+    if (sampleCount >= PERFORMANCE_CONFIDENCE.HIGH.MIN_SAMPLES &&
+        coefficientOfVariation <= PERFORMANCE_CONFIDENCE.HIGH.MAX_CV) {
+        confidenceLevel = 'high';
+    }
+    else if (sampleCount >= PERFORMANCE_CONFIDENCE.MEDIUM.MIN_SAMPLES &&
+        coefficientOfVariation <= PERFORMANCE_CONFIDENCE.MEDIUM.MAX_CV) {
+        confidenceLevel = 'medium';
+    }
+    else {
+        confidenceLevel = 'low';
+        if (sampleCount < PERFORMANCE_CONFIDENCE.HIGH.MIN_SAMPLES) {
+            recommendation = PERFORMANCE_CONFIDENCE.RECOMMENDATIONS.LOW_SAMPLES(sampleCount, PERFORMANCE_CONFIDENCE.HIGH.MIN_SAMPLES);
+        }
+        else {
+            recommendation = PERFORMANCE_CONFIDENCE.RECOMMENDATIONS.HIGH_VARIABILITY;
+        }
+    }
+    return {
+        sampleCount,
+        successfulSamples: sampleCount, // Assumes passed in count is successful samples
+        validationSamples: 0, // Not tracked at this level, set to 0
+        totalTests: totalTests ?? sampleCount,
+        standardDeviation,
+        coefficientOfVariation,
+        confidenceLevel,
+        recommendation,
+    };
+}
+//# sourceMappingURL=shared.js.map

package/dist/docs/templates.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Documentation templates for AGENTS.md generation.
+ */
+export declare const AGENTS_MD_HEADER = "# {{serverName}}\n\n> Generated by [Bellwether](https://github.com/dotsetlabs/bellwether) on {{date}}\n\n";
+export declare const OVERVIEW_SECTION = "## Overview\n\n{{summary}}\n\n**Server Version:** {{version}}\n**Protocol Version:** {{protocolVersion}}\n\n";
+export declare const CAPABILITIES_SECTION = "## Capabilities\n\n{{#each capabilities}}\n- **{{name}}**: {{count}} available\n{{/each}}\n\n";
+export declare const TOOL_SECTION_HEADER = "## Tools\n\n";
+export declare const TOOL_ENTRY = "### {{name}}\n\n{{description}}\n\n**Input Schema:**\n```json\n{{inputSchema}}\n```\n\n{{#if behavioralNotes}}\n**Observed Behavior:**\n{{#each behavioralNotes}}\n- {{this}}\n{{/each}}\n{{/if}}\n\n{{#if limitations}}\n**Limitations:**\n{{#each limitations}}\n- {{this}}\n{{/each}}\n{{/if}}\n\n{{#if securityNotes}}\n**Security Considerations:**\n{{#each securityNotes}}\n- {{this}}\n{{/each}}\n{{/if}}\n\n";
+export declare const LIMITATIONS_SECTION = "## Known Limitations\n\n{{#each limitations}}\n- {{this}}\n{{/each}}\n\n";
+export declare const RECOMMENDATIONS_SECTION = "## Recommendations\n\n{{#each recommendations}}\n- {{this}}\n{{/each}}\n\n";
+export declare const INTERVIEW_METADATA = "---\n\n*Interview completed in {{duration}}ms with {{toolCalls}} tool calls.*\n";
+//# sourceMappingURL=templates.d.ts.map

package/dist/docs/templates.js

@@ -0,0 +1,76 @@
+/**
+ * Documentation templates for AGENTS.md generation.
+ */
+export const AGENTS_MD_HEADER = `# {{serverName}}
+
+> Generated by [Bellwether](https://github.com/dotsetlabs/bellwether) on {{date}}
+
+`;
+export const OVERVIEW_SECTION = `## Overview
+
+{{summary}}
+
+**Server Version:** {{version}}
+**Protocol Version:** {{protocolVersion}}
+
+`;
+export const CAPABILITIES_SECTION = `## Capabilities
+
+{{#each capabilities}}
+- **{{name}}**: {{count}} available
+{{/each}}
+
+`;
+export const TOOL_SECTION_HEADER = `## Tools
+
+`;
+export const TOOL_ENTRY = `### {{name}}
+
+{{description}}
+
+**Input Schema:**
+\`\`\`json
+{{inputSchema}}
+\`\`\`
+
+{{#if behavioralNotes}}
+**Observed Behavior:**
+{{#each behavioralNotes}}
+- {{this}}
+{{/each}}
+{{/if}}
+
+{{#if limitations}}
+**Limitations:**
+{{#each limitations}}
+- {{this}}
+{{/each}}
+{{/if}}
+
+{{#if securityNotes}}
+**Security Considerations:**
+{{#each securityNotes}}
+- {{this}}
+{{/each}}
+{{/if}}
+
+`;
+export const LIMITATIONS_SECTION = `## Known Limitations
+
+{{#each limitations}}
+- {{this}}
+{{/each}}
+
+`;
+export const RECOMMENDATIONS_SECTION = `## Recommendations
+
+{{#each recommendations}}
+- {{this}}
+{{/each}}
+
+`;
+export const INTERVIEW_METADATA = `---
+
+*Interview completed in {{duration}}ms with {{toolCalls}} tool calls.*
+`;
+//# sourceMappingURL=templates.js.map

package/dist/errors/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Error types and utilities.
+ */
+export * from './types.js';
+export * from './retry.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/errors/index.js

@@ -0,0 +1,6 @@
+/**
+ * Error types and utilities.
+ */
+export * from './types.js';
+export * from './retry.js';
+//# sourceMappingURL=index.js.map

package/dist/errors/retry.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Retry logic with exponential backoff.
+ */
+import { type ErrorContext } from './types.js';
+/**
+ * Retry options.
+ */
+export interface RetryOptions {
+    /** Maximum number of retry attempts (default: 3) */
+    maxAttempts?: number;
+    /** Initial delay in milliseconds (default: 1000) */
+    initialDelayMs?: number;
+    /** Maximum delay in milliseconds (default: 30000) */
+    maxDelayMs?: number;
+    /** Backoff multiplier (default: 2) */
+    backoffMultiplier?: number;
+    /** Add jitter to delays (default: true) */
+    jitter?: boolean;
+    /** Custom retry condition (default: isRetryable) */
+    shouldRetry?: (error: unknown, attempt: number) => boolean;
+    /** Callback on retry (for logging) */
+    onRetry?: (error: unknown, attempt: number, delayMs: number) => void;
+    /** Operation name for logging */
+    operation?: string;
+    /** Additional context for errors */
+    context?: ErrorContext;
+}
+/**
+ * Execute a function with retry logic.
+ *
+ * @example
+ * ```typescript
+ * const result = await withRetry(
+ *   async () => await llmClient.complete(prompt),
+ *   { maxAttempts: 3, operation: 'LLM completion' }
+ * );
+ * ```
+ */
+export declare function withRetry<T>(fn: () => Promise<T>, options?: RetryOptions): Promise<T>;
+/**
+ * Create a retry wrapper for a specific operation.
+ *
+ * @example
+ * ```typescript
+ * const retryableFetch = createRetryWrapper({
+ *   maxAttempts: 5,
+ *   operation: 'API fetch'
+ * });
+ *
+ * const data = await retryableFetch(() => fetchData());
+ * ```
+ */
+export declare function createRetryWrapper(defaultOptions: RetryOptions): <T>(fn: () => Promise<T>, overrides?: RetryOptions) => Promise<T>;
+/**
+ * Retry options specifically tuned for LLM calls.
+ */
+export declare const LLM_RETRY_OPTIONS: RetryOptions;
+/**
+ * Retry options for MCP transport operations.
+ */
+export declare const TRANSPORT_RETRY_OPTIONS: RetryOptions;
+/**
+ * Retry options for tool calls during interviews.
+ */
+export declare const TOOL_CALL_RETRY_OPTIONS: RetryOptions;
+/**
+ * Circuit breaker options.
+ */
+export interface CircuitBreakerOptions {
+    /** Number of failures before opening circuit (default: 5) */
+    failureThreshold?: number;
+    /** Time in ms before attempting to close circuit (default: 30000) */
+    resetTimeMs?: number;
+    /** Time window for counting failures in ms (default: 60000) */
+    failureWindowMs?: number;
+}
+/**
+ * Create a circuit breaker for an operation.
+ *
+ * @example
+ * ```typescript
+ * const protectedCall = createCircuitBreaker('llm-api');
+ *
+ * try {
+ *   const result = await protectedCall(() => llmClient.complete(prompt));
+ * } catch (error) {
+ *   // Either operation error or circuit open error
+ * }
+ * ```
+ */
+export declare function createCircuitBreaker(name: string, options?: CircuitBreakerOptions): <T>(fn: () => Promise<T>) => Promise<T>;
+//# sourceMappingURL=retry.d.ts.map