@dotsetlabs/bellwether 0.10.0

package/dist/llm/openai.js

@@ -0,0 +1,320 @@
+import OpenAI from 'openai';
+import { DEFAULT_MODELS, parseJSONResponse } from './client.js';
+import { withRetry, LLM_RETRY_OPTIONS } from '../errors/retry.js';
+import { LLMAuthError, LLMRateLimitError, LLMQuotaError, LLMConnectionError, LLMRefusalError, } from '../errors/index.js';
+import { getLogger } from '../logging/logger.js';
+import { LLM_DEFAULTS } from '../constants.js';
+/**
+ * OpenAI LLM client implementation.
+ */
+/**
+ * Models that require special parameter handling.
+ * This includes reasoning models (o1, o3) and newer GPT models (gpt-5+).
+ * These models:
+ * - Require max_completion_tokens instead of max_tokens
+ * - Don't support custom temperature (only default of 1)
+ * - Use reasoning tokens that come out of the completion token budget
+ */
+const MODELS_WITH_RESTRICTED_PARAMS = [
+    'o1',
+    'o1-mini',
+    'o1-preview',
+    'o3',
+    'o3-mini',
+    'gpt-5',
+];
+/**
+ * Minimum max_completion_tokens for reasoning models.
+ * Reasoning models use tokens for internal "thinking" before producing output.
+ * We need a high minimum to ensure there are enough tokens left for actual output.
+ */
+const REASONING_MODEL_MIN_TOKENS = 8192;
+/**
+ * Check if a model has restricted parameters (newer reasoning/GPT-5 models).
+ */
+function hasRestrictedParams(model) {
+    const modelLower = model.toLowerCase();
+    return MODELS_WITH_RESTRICTED_PARAMS.some(prefix => modelLower.startsWith(prefix));
+}
+/**
+ * Get the effective max tokens for a model, accounting for reasoning overhead.
+ */
+function getEffectiveMaxTokens(model, requestedMaxTokens) {
+    if (hasRestrictedParams(model)) {
+        // Reasoning models need much higher token limits because reasoning tokens
+        // come out of the completion budget. Use at least REASONING_MODEL_MIN_TOKENS.
+        return Math.max(requestedMaxTokens, REASONING_MODEL_MIN_TOKENS);
+    }
+    return requestedMaxTokens;
+}
+export class OpenAIClient {
+    client;
+    defaultModel;
+    logger = getLogger('openai');
+    onUsage;
+    constructor(options) {
+        const apiKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
+        if (!apiKey) {
+            throw new LLMAuthError('openai');
+        }
+        this.client = new OpenAI({
+            apiKey,
+            baseURL: options?.baseURL,
+        });
+        this.defaultModel = options?.model ?? DEFAULT_MODELS.openai;
+        this.onUsage = options?.onUsage;
+    }
+    getProviderInfo() {
+        return {
+            id: 'openai',
+            name: 'OpenAI',
+            supportsJSON: true,
+            supportsStreaming: true,
+            defaultModel: this.defaultModel,
+        };
+    }
+    async chat(messages, options) {
+        const model = options?.model ?? this.defaultModel;
+        return withRetry(async () => {
+            // Prepend system message if provided
+            const allMessages = options?.systemPrompt
+                ? [{ role: 'system', content: options.systemPrompt }, ...messages]
+                : messages;
+            try {
+                // Build request parameters - newer models have restricted params
+                const requestedMaxTokens = options?.maxTokens ?? LLM_DEFAULTS.MAX_TOKENS;
+                const restrictedParams = hasRestrictedParams(model);
+                // For reasoning models, ensure we have enough tokens for both reasoning AND output
+                const maxTokensValue = getEffectiveMaxTokens(model, requestedMaxTokens);
+                const response = await this.client.chat.completions.create({
+                    model,
+                    messages: allMessages.map(m => ({
+                        role: m.role,
+                        content: m.content,
+                    })),
+                    // Use max_completion_tokens for newer models (o1, o3, gpt-5+)
+                    // Use max_tokens for older models (gpt-4, gpt-3.5, etc.)
+                    ...(restrictedParams
+                        ? { max_completion_tokens: maxTokensValue }
+                        : { max_tokens: maxTokensValue }),
+                    // Newer models (o1, o3, gpt-5+) don't support custom temperature
+                    // Only include temperature for models that support it
+                    ...(restrictedParams
+                        ? {}
+                        : { temperature: options?.temperature ?? LLM_DEFAULTS.TEMPERATURE }),
+                    response_format: options?.responseFormat === 'json'
+                        ? { type: 'json_object' }
+                        : undefined,
+                });
+                // Track actual token usage from API response
+                if (this.onUsage && response.usage) {
+                    this.onUsage(response.usage.prompt_tokens, response.usage.completion_tokens);
+                }
+                const choice = response.choices[0];
+                let content = choice?.message?.content;
+                // Check for refusal - but sometimes the model puts valid content in the refusal field!
+                if (!content && choice?.message?.refusal) {
+                    const refusal = choice.message.refusal;
+                    // Check if the refusal actually contains JSON (model mistake)
+                    if (refusal.includes('[') || refusal.includes('{')) {
+                        // Extract JSON from refusal text
+                        const jsonMatch = refusal.match(/```json\s*([\s\S]*?)\s*```/) ||
+                            refusal.match(/(\[[\s\S]*\])/) ||
+                            refusal.match(/(\{[\s\S]*\})/);
+                        if (jsonMatch) {
+                            content = jsonMatch[1];
+                        }
+                    }
+                    if (!content) {
+                        throw new LLMRefusalError('openai', refusal, model);
+                    }
+                }
+                if (!content) {
+                    this.logger.error({ response: JSON.stringify(response) }, 'No content in OpenAI response');
+                    throw new Error('No content in LLM response');
+                }
+                return content;
+            }
+            catch (error) {
+                // Convert to typed errors for retry logic
+                if (error instanceof Error) {
+                    const message = error.message.toLowerCase();
+                    if (message.includes('401')) {
+                        throw new LLMAuthError('openai', model);
+                    }
+                    if (message.includes('429')) {
+                        // Extract Retry-After header if available from OpenAI API error
+                        let retryAfterMs;
+                        const apiError = error;
+                        if (apiError.headers?.get) {
+                            const retryAfter = apiError.headers.get('retry-after');
+                            if (retryAfter) {
+                                // retry-after can be seconds or HTTP date
+                                const seconds = parseInt(retryAfter, 10);
+                                if (!isNaN(seconds)) {
+                                    retryAfterMs = seconds * 1000;
+                                    this.logger.debug({ retryAfterMs }, 'Extracted Retry-After header');
+                                }
+                            }
+                            // Also check x-ratelimit-reset-requests header (OpenAI specific)
+                            const resetRequests = apiError.headers.get('x-ratelimit-reset-requests');
+                            if (resetRequests && !retryAfterMs) {
+                                // Format: "1s", "1m", etc.
+                                const match = resetRequests.match(/(\d+)([smh])/);
+                                if (match) {
+                                    const value = parseInt(match[1], 10);
+                                    const unit = match[2];
+                                    const multipliers = { s: 1000, m: 60000, h: 3600000 };
+                                    retryAfterMs = value * (multipliers[unit] || 1000);
+                                    this.logger.debug({ retryAfterMs, resetRequests }, 'Extracted x-ratelimit-reset-requests');
+                                }
+                            }
+                        }
+                        throw new LLMRateLimitError('openai', retryAfterMs, model);
+                    }
+                    if (message.includes('insufficient_quota')) {
+                        throw new LLMQuotaError('openai', model);
+                    }
+                    if (message.includes('econnrefused') || message.includes('fetch failed')) {
+                        throw new LLMConnectionError('openai', model);
+                    }
+                }
+                throw error;
+            }
+        }, {
+            ...LLM_RETRY_OPTIONS,
+            operation: 'OpenAI chat completion',
+            context: { component: 'openai', metadata: { model } },
+            onRetry: (error, attempt, delayMs) => {
+                this.logger.debug({
+                    attempt,
+                    delayMs: Math.round(delayMs),
+                    error: error instanceof Error ? error.message : String(error),
+                    msg: `Retrying OpenAI API call`,
+                });
+            },
+        });
+    }
+    async complete(prompt, options) {
+        return this.chat([{ role: 'user', content: prompt }], options);
+    }
+    parseJSON(response) {
+        return parseJSONResponse(response);
+    }
+    async stream(prompt, options) {
+        return this.streamChat([{ role: 'user', content: prompt }], options);
+    }
+    async streamChat(messages, options) {
+        const model = options?.model ?? this.defaultModel;
+        return withRetry(async () => {
+            // Prepend system message if provided
+            const allMessages = options?.systemPrompt
+                ? [{ role: 'system', content: options.systemPrompt }, ...messages]
+                : messages;
+            try {
+                // Build request parameters - newer models have restricted params
+                const requestedMaxTokens = options?.maxTokens ?? LLM_DEFAULTS.MAX_TOKENS;
+                const restrictedParams = hasRestrictedParams(model);
+                // For reasoning models, ensure we have enough tokens for both reasoning AND output
+                const maxTokensValue = getEffectiveMaxTokens(model, requestedMaxTokens);
+                const stream = await this.client.chat.completions.create({
+                    model,
+                    messages: allMessages.map(m => ({
+                        role: m.role,
+                        content: m.content,
+                    })),
+                    // Use max_completion_tokens for newer models (o1, o3, gpt-5+)
+                    // Use max_tokens for older models (gpt-4, gpt-3.5, etc.)
+                    ...(restrictedParams
+                        ? { max_completion_tokens: maxTokensValue }
+                        : { max_tokens: maxTokensValue }),
+                    // Newer models (o1, o3, gpt-5+) don't support custom temperature
+                    // Only include temperature for models that support it
+                    ...(restrictedParams
+                        ? {}
+                        : { temperature: options?.temperature ?? LLM_DEFAULTS.TEMPERATURE }),
+                    response_format: options?.responseFormat === 'json'
+                        ? { type: 'json_object' }
+                        : undefined,
+                    stream: true,
+                });
+                let fullText = '';
+                let inputTokens = 0;
+                let outputTokens = 0;
+                let finishReason = null;
+                for await (const chunk of stream) {
+                    const choice = chunk.choices[0];
+                    const content = choice?.delta?.content;
+                    if (content) {
+                        fullText += content;
+                        options?.onChunk?.(content);
+                    }
+                    // Track finish reason from final chunk
+                    if (choice?.finish_reason) {
+                        finishReason = choice.finish_reason;
+                    }
+                    // Track usage from final chunk if available
+                    if (chunk.usage) {
+                        inputTokens = chunk.usage.prompt_tokens;
+                        outputTokens = chunk.usage.completion_tokens;
+                    }
+                }
+                // Report token usage if callback provided
+                if (this.onUsage && (inputTokens > 0 || outputTokens > 0)) {
+                    this.onUsage(inputTokens, outputTokens);
+                }
+                // Handle empty responses gracefully - don't throw, let caller handle
+                // This can happen with content filters, refusals, or empty model responses
+                if (!fullText && finishReason) {
+                    this.logger.debug({ finishReason }, 'Streaming completed with no content');
+                }
+                options?.onComplete?.(fullText);
+                return { text: fullText, completed: fullText.length > 0 };
+            }
+            catch (error) {
+                options?.onError?.(error instanceof Error ? error : new Error(String(error)));
+                // Convert to typed errors for retry logic (same as chat method)
+                if (error instanceof Error) {
+                    const message = error.message.toLowerCase();
+                    if (message.includes('401')) {
+                        throw new LLMAuthError('openai', model);
+                    }
+                    if (message.includes('429')) {
+                        let retryAfterMs;
+                        const apiError = error;
+                        if (apiError.headers?.get) {
+                            const retryAfter = apiError.headers.get('retry-after');
+                            if (retryAfter) {
+                                const seconds = parseInt(retryAfter, 10);
+                                if (!isNaN(seconds)) {
+                                    retryAfterMs = seconds * 1000;
+                                }
+                            }
+                        }
+                        throw new LLMRateLimitError('openai', retryAfterMs, model);
+                    }
+                    if (message.includes('insufficient_quota')) {
+                        throw new LLMQuotaError('openai', model);
+                    }
+                    if (message.includes('econnrefused') || message.includes('fetch failed')) {
+                        throw new LLMConnectionError('openai', model);
+                    }
+                }
+                throw error;
+            }
+        }, {
+            ...LLM_RETRY_OPTIONS,
+            operation: 'OpenAI streaming chat completion',
+            context: { component: 'openai', metadata: { model } },
+            onRetry: (error, attempt, delayMs) => {
+                this.logger.debug({
+                    attempt,
+                    delayMs: Math.round(delayMs),
+                    error: error instanceof Error ? error.message : String(error),
+                    msg: `Retrying OpenAI streaming API call`,
+                });
+            },
+        });
+    }
+}
+//# sourceMappingURL=openai.js.map

package/dist/llm/token-budget.d.ts

@@ -0,0 +1,161 @@
+/**
+ * Token budget enforcement for LLM API calls.
+ *
+ * Provides pre-estimation, budget tracking, context window management,
+ * and graceful truncation.
+ */
+import type { Message, LLMClient, CompletionOptions, StreamingOptions, StreamingResult, ProviderInfo } from './client.js';
+/**
+ * Token budget configuration options.
+ */
+export interface TokenBudgetOptions {
+    /** Maximum total tokens for the session (input + output) */
+    maxTotalTokens?: number;
+    /** Maximum input tokens per request */
+    maxInputTokensPerRequest?: number;
+    /** Maximum output tokens per request */
+    maxOutputTokensPerRequest?: number;
+    /** Warning threshold (0-1) - warn when this percentage of budget is used */
+    warningThreshold?: number;
+    /** Reserve tokens for output (don't use entire context window for input) */
+    outputReserve?: number;
+    /** Callback when approaching budget limit */
+    onBudgetWarning?: (used: number, total: number, percentage: number) => void;
+    /** Callback when budget is exceeded */
+    onBudgetExceeded?: (used: number, total: number) => void;
+}
+/**
+ * Token estimation result.
+ */
+export interface TokenEstimate {
+    /** Estimated token count */
+    tokens: number;
+    /** Whether this exceeds the context window */
+    exceedsContext: boolean;
+    /** Context window size for the model */
+    contextWindow: number;
+    /** Available tokens after this input */
+    availableForOutput: number;
+}
+/**
+ * Budget status.
+ */
+export interface BudgetStatus {
+    /** Total tokens used */
+    totalUsed: number;
+    /** Total budget */
+    totalBudget: number;
+    /** Percentage of budget used */
+    percentageUsed: number;
+    /** Whether warning threshold is exceeded */
+    warningTriggered: boolean;
+    /** Whether budget is exceeded */
+    budgetExceeded: boolean;
+    /** Remaining tokens */
+    remaining: number;
+}
+/**
+ * Estimate tokens from text using a character-based heuristic.
+ * This is a rough approximation (~4 characters per token for English text).
+ */
+export declare function estimateTokens(text: string): number;
+/**
+ * Estimate tokens for a message array.
+ */
+export declare function estimateMessagesTokens(messages: Message[]): number;
+/**
+ * Get context window size for a model.
+ */
+export declare function getContextWindow(model: string): number;
+/**
+ * Estimate tokens and check against context window.
+ */
+export declare function estimateWithContext(text: string | Message[], model: string, outputReserve?: number): TokenEstimate;
+/**
+ * Truncate messages to fit within token budget using sliding window.
+ * Keeps system message and most recent messages.
+ */
+export declare function truncateMessages(messages: Message[], maxTokens: number, options?: {
+    keepSystemMessage?: boolean;
+    minMessages?: number;
+}): Message[];
+/**
+ * Truncate text to approximately fit within token budget.
+ */
+export declare function truncateText(text: string, maxTokens: number): string;
+/**
+ * Token budget tracker for a session.
+ */
+export declare class TokenBudgetTracker {
+    private readonly options;
+    private readonly onBudgetWarning?;
+    private readonly onBudgetExceeded?;
+    private totalInputTokens;
+    private totalOutputTokens;
+    private warningEmitted;
+    constructor(options?: TokenBudgetOptions);
+    /**
+     * Record token usage.
+     */
+    recordUsage(inputTokens: number, outputTokens: number): void;
+    /**
+     * Get current budget status.
+     */
+    getStatus(): BudgetStatus;
+    /**
+     * Check if a request would exceed the budget.
+     */
+    wouldExceedBudget(estimatedInputTokens: number, expectedOutputTokens?: number): boolean;
+    /**
+     * Get maximum safe input tokens for next request.
+     */
+    getMaxSafeInputTokens(): number;
+    /**
+     * Reset the tracker.
+     */
+    reset(): void;
+}
+/**
+ * Budget-aware error.
+ */
+export declare class TokenBudgetExceededError extends Error {
+    readonly used: number;
+    readonly budget: number;
+    readonly requested: number;
+    constructor(used: number, budget: number, requested: number);
+}
+/**
+ * Wrapper that enforces token budget on an LLM client.
+ */
+export declare class BudgetEnforcedLLMClient implements LLMClient {
+    private readonly client;
+    private readonly tracker;
+    private readonly model;
+    private readonly strict;
+    constructor(client: LLMClient, options?: TokenBudgetOptions & {
+        model?: string;
+        strict?: boolean;
+    });
+    getProviderInfo(): ProviderInfo;
+    chat(messages: Message[], options?: CompletionOptions): Promise<string>;
+    complete(prompt: string, options?: CompletionOptions): Promise<string>;
+    stream(prompt: string, options?: StreamingOptions): Promise<StreamingResult>;
+    streamChat(messages: Message[], options?: StreamingOptions): Promise<StreamingResult>;
+    parseJSON<T>(response: string): T;
+    /**
+     * Get current budget status.
+     */
+    getBudgetStatus(): BudgetStatus;
+    /**
+     * Reset the budget tracker.
+     */
+    resetBudget(): void;
+}
+/**
+ * Create a budget-enforced wrapper around an LLM client.
+ */
+export declare function withTokenBudget(client: LLMClient, options?: TokenBudgetOptions & {
+    model?: string;
+    strict?: boolean;
+}): BudgetEnforcedLLMClient;
+//# sourceMappingURL=token-budget.d.ts.map