@dotsetlabs/bellwether 0.10.0

package/dist/llm/token-budget.js

@@ -0,0 +1,395 @@
+/**
+ * Token budget enforcement for LLM API calls.
+ *
+ * Provides pre-estimation, budget tracking, context window management,
+ * and graceful truncation.
+ */
+import { createLogger } from '../logging/logger.js';
+import { MATH_FACTORS } from '../constants.js';
+const logger = createLogger({ name: 'token-budget' });
+/**
+ * Context window sizes for different models (in tokens).
+ */
+const CONTEXT_WINDOWS = {
+    // OpenAI GPT-5 series
+    'gpt-5.2': 256_000,
+    'gpt-5.1': 256_000,
+    'gpt-5': 200_000,
+    'gpt-5-mini': 128_000,
+    'gpt-5-nano': 64_000,
+    // OpenAI GPT-4 series
+    'gpt-4.1': 128_000,
+    'gpt-4.1-mini': 128_000,
+    'gpt-4.1-nano': 64_000,
+    'gpt-4o': 128_000,
+    'gpt-4o-mini': 128_000,
+    'gpt-4-turbo': 128_000,
+    'gpt-4': 8_000,
+    'gpt-3.5-turbo': 16_000,
+    // Anthropic Claude 4.5
+    'claude-opus-4-5': 200_000,
+    'claude-opus-4-5-20251101': 200_000,
+    'claude-sonnet-4-5': 200_000,
+    'claude-sonnet-4-5-20250929': 200_000,
+    'claude-haiku-4-5': 200_000,
+    'claude-haiku-4-5-20251001': 200_000,
+    // Anthropic Claude 4
+    'claude-opus-4-20250514': 200_000,
+    'claude-sonnet-4-20250514': 200_000,
+    // Anthropic Claude 3.x
+    'claude-3-5-sonnet-20241022': 200_000,
+    'claude-3-5-haiku-20241022': 200_000,
+    'claude-3-opus-20240229': 200_000,
+    'claude-3-sonnet-20240229': 200_000,
+    'claude-3-haiku-20240307': 200_000,
+    // Ollama (varies by model, using conservative defaults)
+    'llama3.2': 128_000,
+    'llama3.1': 128_000,
+    'mixtral': 32_000,
+    'codellama': 16_000,
+};
+/**
+ * Default context window for unknown models.
+ */
+const DEFAULT_CONTEXT_WINDOW = 16_000;
+const DEFAULT_OPTIONS = {
+    maxTotalTokens: 1_000_000, // 1M tokens default budget
+    maxInputTokensPerRequest: 100_000,
+    maxOutputTokensPerRequest: 8_000,
+    warningThreshold: MATH_FACTORS.WORD_BOUNDARY_THRESHOLD,
+    outputReserve: 4_000,
+};
+/**
+ * Estimate tokens from text using a character-based heuristic.
+ * This is a rough approximation (~4 characters per token for English text).
+ */
+export function estimateTokens(text) {
+    if (!text)
+        return 0;
+    // More accurate estimation considering:
+    // - Whitespace tokens
+    // - Special characters
+    // - Code tends to have more tokens per character
+    const chars = text.length;
+    const words = text.split(/\s+/).filter(w => w.length > 0).length;
+    const specialChars = (text.match(/[{}[\]()<>:;,."'`]/g) || []).length;
+    // Weighted estimate
+    const baseTokens = chars / 4;
+    const wordAdjustment = words * 0.3;
+    const specialAdjustment = specialChars * MATH_FACTORS.SPECIAL_CHAR_MULTIPLIER;
+    return Math.ceil(baseTokens + wordAdjustment + specialAdjustment);
+}
+/**
+ * Estimate tokens for a message array.
+ */
+export function estimateMessagesTokens(messages) {
+    let total = 0;
+    for (const msg of messages) {
+        // Role overhead (~4 tokens per message for role/formatting)
+        total += 4;
+        total += estimateTokens(msg.content);
+    }
+    // Add overhead for message structure
+    total += 3;
+    return total;
+}
+/**
+ * Get context window size for a model.
+ */
+export function getContextWindow(model) {
+    // Try exact match first
+    if (model in CONTEXT_WINDOWS) {
+        return CONTEXT_WINDOWS[model];
+    }
+    // Try prefix matching for versioned models
+    for (const [key, value] of Object.entries(CONTEXT_WINDOWS)) {
+        if (model.startsWith(key)) {
+            return value;
+        }
+    }
+    return DEFAULT_CONTEXT_WINDOW;
+}
+/**
+ * Estimate tokens and check against context window.
+ */
+export function estimateWithContext(text, model, outputReserve = DEFAULT_OPTIONS.outputReserve) {
+    const tokens = typeof text === 'string'
+        ? estimateTokens(text)
+        : estimateMessagesTokens(text);
+    const contextWindow = getContextWindow(model);
+    const availableForInput = contextWindow - outputReserve;
+    const availableForOutput = Math.max(0, contextWindow - tokens);
+    return {
+        tokens,
+        exceedsContext: tokens > availableForInput,
+        contextWindow,
+        availableForOutput,
+    };
+}
+/**
+ * Truncate messages to fit within token budget using sliding window.
+ * Keeps system message and most recent messages.
+ */
+export function truncateMessages(messages, maxTokens, options = {}) {
+    const { keepSystemMessage = true, minMessages = 2 } = options;
+    if (messages.length === 0)
+        return [];
+    // Separate system message from conversation
+    let systemMessage = null;
+    let conversation = [];
+    if (keepSystemMessage && messages[0]?.role === 'system') {
+        systemMessage = messages[0];
+        conversation = messages.slice(1);
+    }
+    else {
+        conversation = [...messages];
+    }
+    // Calculate system message tokens
+    const systemTokens = systemMessage ? estimateTokens(systemMessage.content) + 4 : 0;
+    const availableTokens = maxTokens - systemTokens;
+    if (availableTokens <= 0) {
+        // System message alone exceeds budget - truncate it
+        if (systemMessage) {
+            const truncatedSystem = truncateText(systemMessage.content, maxTokens - 10);
+            return [{ role: 'system', content: truncatedSystem }];
+        }
+        return [];
+    }
+    // Use sliding window - keep most recent messages that fit
+    const result = [];
+    let currentTokens = 0;
+    // Work backwards from most recent
+    for (let i = conversation.length - 1; i >= 0; i--) {
+        const msg = conversation[i];
+        const msgTokens = estimateTokens(msg.content) + 4;
+        if (currentTokens + msgTokens <= availableTokens || result.length < minMessages) {
+            result.unshift(msg);
+            currentTokens += msgTokens;
+        }
+        else {
+            break;
+        }
+    }
+    // Add system message back at the start
+    if (systemMessage) {
+        result.unshift(systemMessage);
+    }
+    return result;
+}
+/**
+ * Truncate text to approximately fit within token budget.
+ */
+export function truncateText(text, maxTokens) {
+    const estimated = estimateTokens(text);
+    if (estimated <= maxTokens)
+        return text;
+    // Calculate approximate character limit
+    const ratio = maxTokens / estimated;
+    const charLimit = Math.floor(text.length * ratio * MATH_FACTORS.TOKEN_SAFETY_MARGIN); // 5% safety margin
+    // Try to truncate at word boundary
+    let truncated = text.slice(0, charLimit);
+    const lastSpace = truncated.lastIndexOf(' ');
+    if (lastSpace > charLimit * MATH_FACTORS.WORD_BOUNDARY_THRESHOLD) {
+        truncated = truncated.slice(0, lastSpace);
+    }
+    return truncated + '...';
+}
+/**
+ * Token budget tracker for a session.
+ */
+export class TokenBudgetTracker {
+    options;
+    onBudgetWarning;
+    onBudgetExceeded;
+    totalInputTokens = 0;
+    totalOutputTokens = 0;
+    warningEmitted = false;
+    constructor(options = {}) {
+        this.options = { ...DEFAULT_OPTIONS, ...options };
+        this.onBudgetWarning = options.onBudgetWarning;
+        this.onBudgetExceeded = options.onBudgetExceeded;
+    }
+    /**
+     * Record token usage.
+     */
+    recordUsage(inputTokens, outputTokens) {
+        this.totalInputTokens += inputTokens;
+        this.totalOutputTokens += outputTokens;
+        const status = this.getStatus();
+        // Check for budget exceeded
+        if (status.budgetExceeded && this.onBudgetExceeded) {
+            this.onBudgetExceeded(status.totalUsed, status.totalBudget);
+        }
+        // Check for warning threshold
+        else if (status.warningTriggered && !this.warningEmitted && this.onBudgetWarning) {
+            this.onBudgetWarning(status.totalUsed, status.totalBudget, status.percentageUsed);
+            this.warningEmitted = true;
+        }
+    }
+    /**
+     * Get current budget status.
+     */
+    getStatus() {
+        const totalUsed = this.totalInputTokens + this.totalOutputTokens;
+        const percentageUsed = (totalUsed / this.options.maxTotalTokens) * 100;
+        return {
+            totalUsed,
+            totalBudget: this.options.maxTotalTokens,
+            percentageUsed,
+            warningTriggered: percentageUsed >= this.options.warningThreshold * 100,
+            budgetExceeded: totalUsed >= this.options.maxTotalTokens,
+            remaining: Math.max(0, this.options.maxTotalTokens - totalUsed),
+        };
+    }
+    /**
+     * Check if a request would exceed the budget.
+     */
+    wouldExceedBudget(estimatedInputTokens, expectedOutputTokens = 1000) {
+        const status = this.getStatus();
+        return (status.totalUsed + estimatedInputTokens + expectedOutputTokens) > this.options.maxTotalTokens;
+    }
+    /**
+     * Get maximum safe input tokens for next request.
+     */
+    getMaxSafeInputTokens() {
+        const status = this.getStatus();
+        const remaining = status.remaining - this.options.outputReserve;
+        return Math.min(remaining, this.options.maxInputTokensPerRequest);
+    }
+    /**
+     * Reset the tracker.
+     */
+    reset() {
+        this.totalInputTokens = 0;
+        this.totalOutputTokens = 0;
+        this.warningEmitted = false;
+    }
+}
+/**
+ * Budget-aware error.
+ */
+export class TokenBudgetExceededError extends Error {
+    used;
+    budget;
+    requested;
+    constructor(used, budget, requested) {
+        super(`Token budget exceeded: used ${used} + requested ${requested} > budget ${budget}`);
+        this.name = 'TokenBudgetExceededError';
+        this.used = used;
+        this.budget = budget;
+        this.requested = requested;
+    }
+}
+/**
+ * Wrapper that enforces token budget on an LLM client.
+ */
+export class BudgetEnforcedLLMClient {
+    client;
+    tracker;
+    model;
+    strict;
+    constructor(client, options = {}) {
+        this.client = client;
+        this.tracker = new TokenBudgetTracker(options);
+        this.model = options.model ?? client.getProviderInfo().defaultModel;
+        this.strict = options.strict ?? false;
+    }
+    getProviderInfo() {
+        return this.client.getProviderInfo();
+    }
+    async chat(messages, options) {
+        const model = options?.model ?? this.model;
+        const estimate = estimateWithContext(messages, model);
+        // Check if we need to truncate
+        if (estimate.exceedsContext || this.tracker.wouldExceedBudget(estimate.tokens)) {
+            if (this.strict) {
+                const status = this.tracker.getStatus();
+                throw new TokenBudgetExceededError(status.totalUsed, status.totalBudget, estimate.tokens);
+            }
+            // Truncate messages to fit
+            const maxTokens = Math.min(estimate.contextWindow - (options?.maxTokens ?? DEFAULT_OPTIONS.outputReserve), this.tracker.getMaxSafeInputTokens());
+            logger.warn({ originalTokens: estimate.tokens, maxTokens }, 'Truncating messages to fit budget');
+            messages = truncateMessages(messages, maxTokens);
+        }
+        const result = await this.client.chat(messages, options);
+        // Estimate output tokens and record
+        const outputTokens = estimateTokens(result);
+        const inputTokens = estimateMessagesTokens(messages);
+        this.tracker.recordUsage(inputTokens, outputTokens);
+        return result;
+    }
+    async complete(prompt, options) {
+        const model = options?.model ?? this.model;
+        const estimate = estimateWithContext(prompt, model);
+        if (estimate.exceedsContext || this.tracker.wouldExceedBudget(estimate.tokens)) {
+            if (this.strict) {
+                const status = this.tracker.getStatus();
+                throw new TokenBudgetExceededError(status.totalUsed, status.totalBudget, estimate.tokens);
+            }
+            const maxTokens = Math.min(estimate.contextWindow - (options?.maxTokens ?? DEFAULT_OPTIONS.outputReserve), this.tracker.getMaxSafeInputTokens());
+            logger.warn({ originalTokens: estimate.tokens, maxTokens }, 'Truncating prompt to fit budget');
+            prompt = truncateText(prompt, maxTokens);
+        }
+        const result = await this.client.complete(prompt, options);
+        const outputTokens = estimateTokens(result);
+        const inputTokens = estimateTokens(prompt);
+        this.tracker.recordUsage(inputTokens, outputTokens);
+        return result;
+    }
+    async stream(prompt, options) {
+        const model = options?.model ?? this.model;
+        const estimate = estimateWithContext(prompt, model);
+        if (estimate.exceedsContext || this.tracker.wouldExceedBudget(estimate.tokens)) {
+            if (this.strict) {
+                const status = this.tracker.getStatus();
+                throw new TokenBudgetExceededError(status.totalUsed, status.totalBudget, estimate.tokens);
+            }
+            const maxTokens = Math.min(estimate.contextWindow - (options?.maxTokens ?? DEFAULT_OPTIONS.outputReserve), this.tracker.getMaxSafeInputTokens());
+            prompt = truncateText(prompt, maxTokens);
+        }
+        const result = await this.client.stream(prompt, options);
+        const outputTokens = estimateTokens(result.text);
+        const inputTokens = estimateTokens(prompt);
+        this.tracker.recordUsage(inputTokens, outputTokens);
+        return result;
+    }
+    async streamChat(messages, options) {
+        const model = options?.model ?? this.model;
+        const estimate = estimateWithContext(messages, model);
+        if (estimate.exceedsContext || this.tracker.wouldExceedBudget(estimate.tokens)) {
+            if (this.strict) {
+                const status = this.tracker.getStatus();
+                throw new TokenBudgetExceededError(status.totalUsed, status.totalBudget, estimate.tokens);
+            }
+            const maxTokens = Math.min(estimate.contextWindow - (options?.maxTokens ?? DEFAULT_OPTIONS.outputReserve), this.tracker.getMaxSafeInputTokens());
+            messages = truncateMessages(messages, maxTokens);
+        }
+        const result = await this.client.streamChat(messages, options);
+        const outputTokens = estimateTokens(result.text);
+        const inputTokens = estimateMessagesTokens(messages);
+        this.tracker.recordUsage(inputTokens, outputTokens);
+        return result;
+    }
+    parseJSON(response) {
+        return this.client.parseJSON(response);
+    }
+    /**
+     * Get current budget status.
+     */
+    getBudgetStatus() {
+        return this.tracker.getStatus();
+    }
+    /**
+     * Reset the budget tracker.
+     */
+    resetBudget() {
+        this.tracker.reset();
+    }
+}
+/**
+ * Create a budget-enforced wrapper around an LLM client.
+ */
+export function withTokenBudget(client, options) {
+    return new BudgetEnforcedLLMClient(client, options);
+}
+//# sourceMappingURL=token-budget.js.map

package/dist/logging/logger.d.ts

@@ -0,0 +1,70 @@
+import { Logger as PinoLogger } from 'pino';
+/**
+ * Log levels supported by Bellwether.
+ */
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error' | 'silent';
+/**
+ * Logger configuration options.
+ */
+export interface LoggerConfig {
+    /** Log level (default: 'info') */
+    level?: LogLevel;
+    /** Output file path (default: stderr) */
+    file?: string;
+    /** Enable pretty printing for development */
+    pretty?: boolean;
+    /** Include timestamps in output */
+    timestamp?: boolean;
+    /** Component name for context */
+    name?: string;
+}
+/**
+ * Create a new logger instance.
+ */
+export declare function createLogger(config?: LoggerConfig): PinoLogger;
+/**
+ * Get or create the global logger instance.
+ */
+export declare function getLogger(name?: string): PinoLogger;
+/**
+ * Configure the global logger.
+ */
+export declare function configureLogger(config: LoggerConfig): void;
+/**
+ * Reset the global logger (for testing).
+ */
+export declare function resetLogger(): void;
+/**
+ * Temporarily suppress all logging (set level to 'silent').
+ * Call restoreLogLevel() to restore the previous level.
+ */
+export declare function suppressLogs(): void;
+/**
+ * Restore the log level after suppression.
+ */
+export declare function restoreLogLevel(): void;
+/**
+ * Create a child logger with additional context.
+ */
+export declare function childLogger(parent: PinoLogger, context: Record<string, unknown>): PinoLogger;
+/**
+ * Timing helper for performance measurement.
+ */
+export interface TimingResult {
+    durationMs: number;
+    log: () => void;
+}
+/**
+ * Start a timing measurement.
+ */
+export declare function startTiming(logger: PinoLogger, operation: string): () => TimingResult;
+/**
+ * Log levels as numeric values for comparison.
+ */
+export declare const LOG_LEVEL_VALUES: Record<LogLevel, number>;
+/**
+ * Check if a log level is enabled.
+ */
+export declare function isLevelEnabled(currentLevel: LogLevel, checkLevel: LogLevel): boolean;
+export type { PinoLogger as Logger };
+//# sourceMappingURL=logger.d.ts.map

package/dist/logging/logger.js

@@ -0,0 +1,130 @@
+import pino from 'pino';
+/**
+ * Default configuration.
+ */
+const DEFAULT_CONFIG = {
+    level: 'info',
+    pretty: false,
+    timestamp: true,
+};
+/**
+ * Global logger instance.
+ */
+let globalLogger = null;
+/**
+ * Create a new logger instance.
+ */
+export function createLogger(config = {}) {
+    const level = config.level ?? DEFAULT_CONFIG.level;
+    const pretty = config.pretty ?? DEFAULT_CONFIG.pretty;
+    const timestamp = config.timestamp ?? DEFAULT_CONFIG.timestamp;
+    const options = {
+        level,
+        name: config.name,
+        timestamp: timestamp ? pino.stdTimeFunctions.isoTime : false,
+        formatters: {
+            level: (label) => ({ level: label }),
+        },
+    };
+    // Use pino-pretty transport for development
+    if (pretty) {
+        options.transport = {
+            target: 'pino-pretty',
+            options: {
+                colorize: true,
+                translateTime: 'SYS:standard',
+                ignore: 'pid,hostname',
+            },
+        };
+    }
+    // Create logger with file destination or stdout
+    if (config.file) {
+        return pino(options, pino.destination(config.file));
+    }
+    return pino(options);
+}
+/**
+ * Get or create the global logger instance.
+ */
+export function getLogger(name) {
+    if (!globalLogger) {
+        globalLogger = createLogger({ level: 'info' });
+    }
+    if (name) {
+        return globalLogger.child({ component: name });
+    }
+    return globalLogger;
+}
+/**
+ * Configure the global logger.
+ */
+export function configureLogger(config) {
+    globalLogger = createLogger(config);
+}
+/**
+ * Reset the global logger (for testing).
+ */
+export function resetLogger() {
+    globalLogger = null;
+}
+/**
+ * Saved log level for restoration after temporary suppression.
+ */
+let savedLogLevel = null;
+/**
+ * Temporarily suppress all logging (set level to 'silent').
+ * Call restoreLogLevel() to restore the previous level.
+ */
+export function suppressLogs() {
+    if (globalLogger && savedLogLevel === null) {
+        savedLogLevel = globalLogger.level;
+        globalLogger.level = 'silent';
+    }
+}
+/**
+ * Restore the log level after suppression.
+ */
+export function restoreLogLevel() {
+    if (globalLogger && savedLogLevel !== null) {
+        globalLogger.level = savedLogLevel;
+        savedLogLevel = null;
+    }
+}
+/**
+ * Create a child logger with additional context.
+ */
+export function childLogger(parent, context) {
+    return parent.child(context);
+}
+/**
+ * Start a timing measurement.
+ */
+export function startTiming(logger, operation) {
+    const startTime = Date.now();
+    return () => {
+        const durationMs = Date.now() - startTime;
+        return {
+            durationMs,
+            log: () => {
+                logger.debug({ operation, durationMs }, `${operation} completed`);
+            },
+        };
+    };
+}
+/**
+ * Log levels as numeric values for comparison.
+ */
+export const LOG_LEVEL_VALUES = {
+    debug: 20,
+    info: 30,
+    warn: 40,
+    error: 50,
+    silent: Infinity,
+};
+/**
+ * Check if a log level is enabled.
+ */
+export function isLevelEnabled(currentLevel, checkLevel) {
+    return LOG_LEVEL_VALUES[checkLevel] >= LOG_LEVEL_VALUES[currentLevel];
+}
+//# sourceMappingURL=logger.js.map

package/dist/metrics/collector.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Metrics collector for comprehensive observability.
+ *
+ * Tracks timing, token usage, errors, and costs across all operations.
+ */
+import type { TokenUsageRecord, TimingRecord, ErrorRecord, CostRecord, OperationType, ErrorCategory, InterviewMetrics, AggregatedMetrics, DashboardMetrics } from './types.js';
+/**
+ * Options for the metrics collector.
+ */
+export interface MetricsCollectorOptions {
+    /** Maximum number of records to keep in memory */
+    maxRecords?: number;
+    /** Whether to track individual records (vs just aggregates) */
+    trackRecords?: boolean;
+}
+/**
+ * Centralized metrics collector for observability.
+ */
+export declare class MetricsCollector {
+    private readonly options;
+    private tokenRecords;
+    private timingRecords;
+    private errorRecords;
+    private costRecords;
+    private tokensByProviderModel;
+    private timingByOperation;
+    private errorsByCategory;
+    private costByProvider;
+    private interviewMetrics;
+    constructor(options?: MetricsCollectorOptions);
+    /**
+     * Start tracking an interview.
+     */
+    startInterview(): void;
+    /**
+     * End interview tracking.
+     */
+    endInterview(): InterviewMetrics | null;
+    /**
+     * Record token usage from an LLM call.
+     */
+    recordTokenUsage(provider: string, model: string, inputTokens: number, outputTokens: number, operation?: string): void;
+    /**
+     * Record operation timing.
+     */
+    recordTiming(operation: OperationType, durationMs: number, success: boolean, metadata?: Record<string, unknown>): void;
+    /**
+     * Record an error.
+     */
+    recordError(category: ErrorCategory, code: string, message: string, retryable: boolean, operation?: string): void;
+    /**
+     * Update interview-specific counters.
+     */
+    updateInterviewCounters(updates: Partial<{
+        toolsDiscovered: number;
+        questionsGenerated: number;
+        personasUsed: number;
+    }>): void;
+    /**
+     * Create a timed operation wrapper.
+     */
+    time<T>(operation: OperationType, fn: () => Promise<T>, metadata?: Record<string, unknown>): Promise<T>;
+    /**
+     * Get current interview metrics.
+     */
+    getInterviewMetrics(): InterviewMetrics | null;
+    /**
+     * Get aggregated metrics for a time period.
+     */
+    getAggregatedMetrics(periodStart?: Date, periodEnd?: Date): AggregatedMetrics;
+    /**
+     * Get dashboard-compatible metrics.
+     */
+    getDashboardMetrics(): DashboardMetrics;
+    /**
+     * Reset all metrics.
+     */
+    reset(): void;
+    /**
+     * Get raw records for external processing.
+     */
+    getRawRecords(): {
+        tokens: TokenUsageRecord[];
+        timing: TimingRecord[];
+        errors: ErrorRecord[];
+        costs: CostRecord[];
+    };
+    private trimRecords;
+    private calculateTokenUsageFromRecords;
+    private calculateTokenUsageFromAggregates;
+    private calculateOperationStatsFromRecords;
+    private calculateOperationStatsFromAggregates;
+    private convertOperationStats;
+    private calculateErrorStatsFromRecords;
+    private calculateErrorStatsFromAggregates;
+    private calculateProjectedCost;
+}
+/**
+ * Get or create the global metrics collector.
+ */
+export declare function getMetricsCollector(options?: MetricsCollectorOptions): MetricsCollector;
+/**
+ * Reset the global metrics collector.
+ */
+export declare function resetMetricsCollector(): void;
+//# sourceMappingURL=collector.d.ts.map