@dotsetlabs/bellwether 0.10.0

package/dist/llm/anthropic.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Anthropic Claude LLM client implementation.
+ */
+import type { LLMClient, Message, CompletionOptions, ProviderInfo, StreamingOptions, StreamingResult } from './client.js';
+export interface AnthropicClientOptions {
+    /** API key (defaults to ANTHROPIC_API_KEY env var) */
+    apiKey?: string;
+    /** Default model to use */
+    model?: string;
+    /** Base URL for API (for proxies/alternatives) */
+    baseURL?: string;
+    /** Callback to receive token usage from each API call */
+    onUsage?: (inputTokens: number, outputTokens: number) => void;
+}
+/**
+ * Anthropic Claude LLM client implementation.
+ */
+export declare class AnthropicClient implements LLMClient {
+    private client;
+    private defaultModel;
+    private logger;
+    private onUsage?;
+    constructor(options?: AnthropicClientOptions);
+    getProviderInfo(): ProviderInfo;
+    chat(messages: Message[], options?: CompletionOptions): Promise<string>;
+    /**
+     * Check for content filtering refusal in Anthropic response.
+     * Anthropic uses stop_reason to indicate why generation stopped.
+     */
+    private checkForRefusal;
+    /**
+     * Normalize message order for Claude's requirements.
+     * Claude requires alternating user/assistant messages starting with user.
+     */
+    private normalizeMessageOrder;
+    complete(prompt: string, options?: CompletionOptions): Promise<string>;
+    parseJSON<T>(response: string): T;
+    stream(prompt: string, options?: StreamingOptions): Promise<StreamingResult>;
+    streamChat(messages: Message[], options?: StreamingOptions): Promise<StreamingResult>;
+}
+//# sourceMappingURL=anthropic.d.ts.map

package/dist/llm/anthropic.js

@@ -0,0 +1,355 @@
+/**
+ * Anthropic Claude LLM client implementation.
+ */
+import Anthropic from '@anthropic-ai/sdk';
+import { DEFAULT_MODELS, parseJSONResponse } from './client.js';
+import { LLM_DEFAULTS } from '../constants.js';
+import { withRetry, LLM_RETRY_OPTIONS } from '../errors/retry.js';
+import { LLMAuthError, LLMRateLimitError, LLMQuotaError, LLMConnectionError, LLMRefusalError, } from '../errors/index.js';
+import { getLogger } from '../logging/logger.js';
+/**
+ * Placeholder messages for Claude message normalization.
+ * Claude requires alternating user/assistant messages starting with user.
+ */
+const PLACEHOLDER_CONTINUE = 'Continue.';
+const PLACEHOLDER_GREETING = 'Hello.';
+/**
+ * Anthropic Claude LLM client implementation.
+ */
+export class AnthropicClient {
+    client;
+    defaultModel;
+    logger = getLogger('anthropic');
+    onUsage;
+    constructor(options) {
+        const apiKey = options?.apiKey ?? process.env.ANTHROPIC_API_KEY;
+        if (!apiKey) {
+            throw new LLMAuthError('anthropic');
+        }
+        this.client = new Anthropic({
+            apiKey,
+            baseURL: options?.baseURL,
+        });
+        this.defaultModel = options?.model ?? DEFAULT_MODELS.anthropic;
+        this.onUsage = options?.onUsage;
+    }
+    getProviderInfo() {
+        return {
+            id: 'anthropic',
+            name: 'Anthropic Claude',
+            supportsJSON: false, // Claude doesn't have a JSON mode like OpenAI
+            supportsStreaming: true,
+            defaultModel: this.defaultModel,
+        };
+    }
+    async chat(messages, options) {
+        const model = options?.model ?? this.defaultModel;
+        return withRetry(async () => {
+            // Separate system message from conversation messages
+            const systemPrompt = options?.systemPrompt;
+            // Convert messages to Anthropic format
+            const anthropicMessages = messages.map(m => ({
+                role: m.role === 'assistant' ? 'assistant' : 'user',
+                content: m.content,
+            }));
+            // If first message has system role, extract it
+            let system = systemPrompt;
+            if (messages.length > 0 && messages[0].role === 'system') {
+                system = system ? `${system}\n\n${messages[0].content}` : messages[0].content;
+                anthropicMessages.shift();
+            }
+            // Claude requires messages to start with user role
+            // If we have no messages after removing system, add a placeholder
+            if (anthropicMessages.length === 0) {
+                throw new Error('At least one user message is required');
+            }
+            // Ensure messages alternate between user and assistant
+            // Claude is strict about this
+            const normalizedMessages = this.normalizeMessageOrder(anthropicMessages);
+            try {
+                const response = await this.client.messages.create({
+                    model,
+                    max_tokens: options?.maxTokens ?? LLM_DEFAULTS.MAX_TOKENS,
+                    system: system,
+                    messages: normalizedMessages,
+                });
+                // Track actual token usage from API response
+                if (this.onUsage && response.usage) {
+                    this.onUsage(response.usage.input_tokens, response.usage.output_tokens);
+                }
+                // Check for content filtering refusal
+                this.checkForRefusal(response, model);
+                // Extract text content from response
+                const textBlocks = response.content.filter(block => block.type === 'text');
+                if (textBlocks.length === 0) {
+                    throw new Error('No text content in Claude response');
+                }
+                return textBlocks.map(block => block.text).join('');
+            }
+            catch (error) {
+                // Don't re-process errors that are already typed LLM errors
+                if (error instanceof LLMRefusalError || error instanceof LLMAuthError ||
+                    error instanceof LLMRateLimitError || error instanceof LLMQuotaError ||
+                    error instanceof LLMConnectionError) {
+                    throw error;
+                }
+                // Convert to typed errors for retry logic
+                if (error instanceof Error) {
+                    const message = error.message.toLowerCase();
+                    if (message.includes('401') || message.includes('authentication')) {
+                        throw new LLMAuthError('anthropic', model);
+                    }
+                    if (message.includes('429') || message.includes('rate limit')) {
+                        // Extract retry-after-ms header if available from Anthropic API error
+                        let retryAfterMs;
+                        const apiError = error;
+                        if (apiError.headers?.get) {
+                            // Anthropic uses retry-after-ms header (milliseconds)
+                            const retryAfterMsHeader = apiError.headers.get('retry-after-ms');
+                            if (retryAfterMsHeader) {
+                                const ms = parseInt(retryAfterMsHeader, 10);
+                                if (!isNaN(ms)) {
+                                    retryAfterMs = ms;
+                                    this.logger.debug({ retryAfterMs }, 'Extracted retry-after-ms header');
+                                }
+                            }
+                            // Fall back to standard retry-after header (seconds)
+                            if (!retryAfterMs) {
+                                const retryAfter = apiError.headers.get('retry-after');
+                                if (retryAfter) {
+                                    const seconds = parseInt(retryAfter, 10);
+                                    if (!isNaN(seconds)) {
+                                        retryAfterMs = seconds * 1000;
+                                        this.logger.debug({ retryAfterMs }, 'Extracted retry-after header');
+                                    }
+                                }
+                            }
+                        }
+                        throw new LLMRateLimitError('anthropic', retryAfterMs, model);
+                    }
+                    if (message.includes('insufficient') || message.includes('credit')) {
+                        throw new LLMQuotaError('anthropic', model);
+                    }
+                    if (message.includes('econnrefused') || message.includes('fetch failed')) {
+                        throw new LLMConnectionError('anthropic', model);
+                    }
+                }
+                throw error;
+            }
+        }, {
+            ...LLM_RETRY_OPTIONS,
+            operation: 'Anthropic chat completion',
+            context: { component: 'anthropic', metadata: { model } },
+            onRetry: (error, attempt, delayMs) => {
+                this.logger.debug({
+                    attempt,
+                    delayMs: Math.round(delayMs),
+                    error: error instanceof Error ? error.message : String(error),
+                    msg: `Retrying Anthropic API call`,
+                });
+            },
+        });
+    }
+    /**
+     * Check for content filtering refusal in Anthropic response.
+     * Anthropic uses stop_reason to indicate why generation stopped.
+     */
+    checkForRefusal(response, model) {
+        // Check stop_reason for content filtering
+        if (response.stop_reason === 'content_filter') {
+            throw new LLMRefusalError('anthropic', 'Content was filtered', model);
+        }
+        // Check for safety-related stop reasons
+        if (response.stop_reason === 'safety') {
+            throw new LLMRefusalError('anthropic', 'Response blocked due to safety concerns', model);
+        }
+        // Check content for refusal indicators
+        const textBlocks = response.content.filter(block => block.type === 'text');
+        if (textBlocks.length > 0) {
+            const fullText = textBlocks.map(block => block.text ?? '').join('').toLowerCase();
+            // Check for common refusal patterns in Claude's responses
+            const refusalPatterns = [
+                'i cannot help with',
+                'i am not able to',
+                "i can't assist with",
+                'i cannot assist with',
+                'i am unable to',
+                "i won't be able to",
+                'i must decline',
+                'i need to refuse',
+                'against my guidelines',
+                'violates my ethical guidelines',
+                'goes against my values',
+                'i cannot provide',
+                'i cannot generate',
+                'harmful content',
+                'dangerous content',
+                'illegal content',
+            ];
+            for (const pattern of refusalPatterns) {
+                if (fullText.includes(pattern)) {
+                    // Extract more context around the refusal
+                    const startIdx = Math.max(0, fullText.indexOf(pattern) - 20);
+                    const endIdx = Math.min(fullText.length, fullText.indexOf(pattern) + pattern.length + 100);
+                    const context = fullText.slice(startIdx, endIdx).trim();
+                    throw new LLMRefusalError('anthropic', `Model declined: "${context}..."`, model);
+                }
+            }
+        }
+    }
+    /**
+     * Normalize message order for Claude's requirements.
+     * Claude requires alternating user/assistant messages starting with user.
+     */
+    normalizeMessageOrder(messages) {
+        const result = [];
+        for (const msg of messages) {
+            if (result.length === 0) {
+                // First message must be from user
+                if (msg.role === 'assistant') {
+                    // Insert placeholder user message
+                    result.push({ role: 'user', content: PLACEHOLDER_CONTINUE });
+                }
+                result.push(msg);
+            }
+            else {
+                const lastRole = result[result.length - 1].role;
+                if (msg.role === lastRole) {
+                    // Same role as previous - merge content
+                    result[result.length - 1].content += '\n\n' + msg.content;
+                }
+                else {
+                    result.push(msg);
+                }
+            }
+        }
+        // Ensure we end with the messages array not empty
+        if (result.length === 0) {
+            result.push({ role: 'user', content: PLACEHOLDER_GREETING });
+        }
+        return result;
+    }
+    async complete(prompt, options) {
+        return this.chat([{ role: 'user', content: prompt }], options);
+    }
+    parseJSON(response) {
+        return parseJSONResponse(response);
+    }
+    async stream(prompt, options) {
+        return this.streamChat([{ role: 'user', content: prompt }], options);
+    }
+    async streamChat(messages, options) {
+        const model = options?.model ?? this.defaultModel;
+        return withRetry(async () => {
+            // Separate system message from conversation messages
+            const systemPrompt = options?.systemPrompt;
+            // Convert messages to Anthropic format
+            const anthropicMessages = messages.map(m => ({
+                role: m.role === 'assistant' ? 'assistant' : 'user',
+                content: m.content,
+            }));
+            // If first message has system role, extract it
+            let system = systemPrompt;
+            if (messages.length > 0 && messages[0].role === 'system') {
+                system = system ? `${system}\n\n${messages[0].content}` : messages[0].content;
+                anthropicMessages.shift();
+            }
+            // Claude requires messages to start with user role
+            if (anthropicMessages.length === 0) {
+                throw new Error('At least one user message is required');
+            }
+            // Ensure messages alternate between user and assistant
+            const normalizedMessages = this.normalizeMessageOrder(anthropicMessages);
+            try {
+                const stream = await this.client.messages.stream({
+                    model,
+                    max_tokens: options?.maxTokens ?? LLM_DEFAULTS.MAX_TOKENS,
+                    system: system,
+                    messages: normalizedMessages,
+                });
+                let fullText = '';
+                for await (const event of stream) {
+                    if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') {
+                        const text = event.delta.text;
+                        if (text) {
+                            fullText += text;
+                            options?.onChunk?.(text);
+                        }
+                    }
+                }
+                // Get final message for token usage
+                const finalMessage = await stream.finalMessage();
+                // Track actual token usage from API response
+                if (this.onUsage && finalMessage.usage) {
+                    this.onUsage(finalMessage.usage.input_tokens, finalMessage.usage.output_tokens);
+                }
+                // Check for content filtering refusal
+                this.checkForRefusal(finalMessage, model);
+                if (!fullText) {
+                    throw new Error('No text content in streaming response');
+                }
+                options?.onComplete?.(fullText);
+                return { text: fullText, completed: true };
+            }
+            catch (error) {
+                options?.onError?.(error instanceof Error ? error : new Error(String(error)));
+                // Don't re-process errors that are already typed LLM errors
+                if (error instanceof LLMRefusalError || error instanceof LLMAuthError ||
+                    error instanceof LLMRateLimitError || error instanceof LLMQuotaError ||
+                    error instanceof LLMConnectionError) {
+                    throw error;
+                }
+                // Convert to typed errors for retry logic (same as chat method)
+                if (error instanceof Error) {
+                    const message = error.message.toLowerCase();
+                    if (message.includes('401') || message.includes('authentication')) {
+                        throw new LLMAuthError('anthropic', model);
+                    }
+                    if (message.includes('429') || message.includes('rate limit')) {
+                        let retryAfterMs;
+                        const apiError = error;
+                        if (apiError.headers?.get) {
+                            const retryAfterMsHeader = apiError.headers.get('retry-after-ms');
+                            if (retryAfterMsHeader) {
+                                const ms = parseInt(retryAfterMsHeader, 10);
+                                if (!isNaN(ms)) {
+                                    retryAfterMs = ms;
+                                }
+                            }
+                            if (!retryAfterMs) {
+                                const retryAfter = apiError.headers.get('retry-after');
+                                if (retryAfter) {
+                                    const seconds = parseInt(retryAfter, 10);
+                                    if (!isNaN(seconds)) {
+                                        retryAfterMs = seconds * 1000;
+                                    }
+                                }
+                            }
+                        }
+                        throw new LLMRateLimitError('anthropic', retryAfterMs, model);
+                    }
+                    if (message.includes('insufficient') || message.includes('credit')) {
+                        throw new LLMQuotaError('anthropic', model);
+                    }
+                    if (message.includes('econnrefused') || message.includes('fetch failed')) {
+                        throw new LLMConnectionError('anthropic', model);
+                    }
+                }
+                throw error;
+            }
+        }, {
+            ...LLM_RETRY_OPTIONS,
+            operation: 'Anthropic streaming chat completion',
+            context: { component: 'anthropic', metadata: { model } },
+            onRetry: (error, attempt, delayMs) => {
+                this.logger.debug({
+                    attempt,
+                    delayMs: Math.round(delayMs),
+                    error: error instanceof Error ? error.message : String(error),
+                    msg: `Retrying Anthropic streaming API call`,
+                });
+            },
+        });
+    }
+}
+//# sourceMappingURL=anthropic.js.map

package/dist/llm/client.d.ts

@@ -0,0 +1,123 @@
+/**
+ * LLM client interface for abstracting different providers.
+ */
+export interface Message {
+    role: 'system' | 'user' | 'assistant';
+    content: string;
+}
+export interface CompletionOptions {
+    /** Model to use */
+    model?: string;
+    /** Maximum tokens in response */
+    maxTokens?: number;
+    /** Temperature for sampling (0-2) */
+    temperature?: number;
+    /** Response format */
+    responseFormat?: 'text' | 'json';
+    /** System prompt to set context */
+    systemPrompt?: string;
+}
+/**
+ * Options for streaming completions.
+ */
+export interface StreamingOptions extends CompletionOptions {
+    /** Callback invoked with each chunk of text as it arrives */
+    onChunk?: (chunk: string) => void;
+    /** Callback invoked when streaming completes with full text */
+    onComplete?: (fullText: string) => void;
+    /** Callback invoked if an error occurs during streaming */
+    onError?: (error: Error) => void;
+}
+/**
+ * Result of a streaming completion.
+ */
+export interface StreamingResult {
+    /** The complete text (after streaming finishes) */
+    text: string;
+    /** Whether streaming completed successfully */
+    completed: boolean;
+}
+/**
+ * Provider capabilities and metadata.
+ */
+export interface ProviderInfo {
+    /** Unique provider identifier */
+    id: string;
+    /** Human-readable provider name */
+    name: string;
+    /** Whether the provider supports JSON mode */
+    supportsJSON: boolean;
+    /** Whether the provider supports streaming */
+    supportsStreaming: boolean;
+    /** Default model for this provider */
+    defaultModel: string;
+}
+export interface LLMClient {
+    /**
+     * Get provider information.
+     */
+    getProviderInfo(): ProviderInfo;
+    /**
+     * Generate a completion from a list of messages.
+     */
+    chat(messages: Message[], options?: CompletionOptions): Promise<string>;
+    /**
+     * Generate a completion from a single prompt.
+     */
+    complete(prompt: string, options?: CompletionOptions): Promise<string>;
+    /**
+     * Stream a completion from a single prompt.
+     * Yields text chunks as they arrive from the LLM.
+     * Returns the complete response text when done.
+     */
+    stream(prompt: string, options?: StreamingOptions): Promise<StreamingResult>;
+    /**
+     * Stream a chat completion from a list of messages.
+     * Yields text chunks as they arrive from the LLM.
+     * Returns the complete response text when done.
+     */
+    streamChat(messages: Message[], options?: StreamingOptions): Promise<StreamingResult>;
+    /**
+     * Parse JSON from LLM response, handling common formatting issues.
+     */
+    parseJSON<T>(response: string): T;
+}
+/**
+ * Default model configurations per provider.
+ * Uses budget-friendly models by default for cost efficiency.
+ * @see constants.ts for the source values
+ */
+export declare const DEFAULT_MODELS: Record<LLMProviderId, string>;
+/**
+ * Premium model configurations for --quality flag.
+ * Higher quality output but more expensive.
+ * @see constants.ts for the source values
+ */
+export declare const PREMIUM_MODELS: Record<LLMProviderId, string>;
+/**
+ * Provider IDs.
+ */
+export type LLMProviderId = 'openai' | 'anthropic' | 'ollama';
+/**
+ * Configuration for LLM providers.
+ */
+export interface LLMConfig {
+    /** Provider to use */
+    provider: LLMProviderId;
+    /** Model to use (provider-specific) */
+    model?: string;
+    /** API key (or env var name containing it) */
+    apiKey?: string;
+    /** Environment variable containing API key */
+    apiKeyEnvVar?: string;
+    /** Base URL for API (for proxies/self-hosted) */
+    baseUrl?: string;
+    /** Callback to receive actual token usage from each API call */
+    onUsage?: (inputTokens: number, outputTokens: number) => void;
+}
+/**
+ * Parse JSON from LLM response, handling common formatting issues.
+ * Shared utility function for all providers.
+ */
+export declare function parseJSONResponse<T>(response: string): T;
+//# sourceMappingURL=client.d.ts.map

package/dist/llm/client.js

@@ -0,0 +1,42 @@
+/**
+ * LLM client interface for abstracting different providers.
+ */
+import { DEFAULT_MODELS as DEFAULT_MODELS_CONST, PREMIUM_MODELS as PREMIUM_MODELS_CONST, } from '../constants.js';
+/**
+ * Default model configurations per provider.
+ * Uses budget-friendly models by default for cost efficiency.
+ * @see constants.ts for the source values
+ */
+export const DEFAULT_MODELS = DEFAULT_MODELS_CONST;
+/**
+ * Premium model configurations for --quality flag.
+ * Higher quality output but more expensive.
+ * @see constants.ts for the source values
+ */
+export const PREMIUM_MODELS = PREMIUM_MODELS_CONST;
+/**
+ * Parse JSON from LLM response, handling common formatting issues.
+ * Shared utility function for all providers.
+ */
+export function parseJSONResponse(response) {
+    // Handle markdown code blocks
+    let cleaned = response.trim();
+    // Remove ```json or ``` wrappers
+    if (cleaned.startsWith('```')) {
+        const lines = cleaned.split('\n');
+        // Remove first line (```json or ```)
+        lines.shift();
+        // Remove last line (```)
+        if (lines[lines.length - 1]?.trim() === '```') {
+            lines.pop();
+        }
+        cleaned = lines.join('\n');
+    }
+    try {
+        return JSON.parse(cleaned);
+    }
+    catch (error) {
+        throw new Error(`Failed to parse JSON from LLM response: ${error}`);
+    }
+}
+//# sourceMappingURL=client.js.map

package/dist/llm/factory.d.ts

@@ -0,0 +1,38 @@
+/**
+ * LLM provider factory - creates and auto-detects providers.
+ */
+import type { LLMClient, LLMConfig, LLMProviderId } from './client.js';
+/**
+ * Create an LLM client from configuration.
+ */
+export declare function createLLMClient(config: LLMConfig): LLMClient;
+/**
+ * Auto-detect the best available provider based on environment.
+ */
+export declare function detectProvider(): LLMProviderId;
+/**
+ * Create an LLM client with auto-detection.
+ * Uses environment variables to determine the best provider.
+ */
+export declare function createAutoClient(modelOverride?: string): LLMClient;
+/**
+ * Get information about available providers.
+ */
+export interface ProviderAvailability {
+    provider: LLMProviderId;
+    available: boolean;
+    reason?: string;
+}
+/**
+ * Check which providers are available.
+ */
+export declare function checkProviderAvailability(): Promise<ProviderAvailability[]>;
+/**
+ * Get the default model for a provider.
+ */
+export declare function getDefaultModel(provider: LLMProviderId): string;
+/**
+ * List all supported providers.
+ */
+export declare function getSupportedProviders(): LLMProviderId[];
+//# sourceMappingURL=factory.d.ts.map