@azumag/opencode-rate-limit-fallback 1.31.0 → 1.36.0

package/dist/src/health/HealthTracker.js

@@ -0,0 +1,353 @@
+/**
+ * Model Health Tracker
+ * Tracks model success rates and response times for health-based selection
+ */
+import { getModelKey } from '../utils/helpers.js';
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
+import { join, dirname } from 'path';
+import { homedir } from 'os';
+/**
+ * Default health persistence path
+ */
+const DEFAULT_HEALTH_PERSISTENCE_PATH = join(homedir(), '.opencode', 'rate-limit-fallback-health.json');
+/**
+ * Minimum requests before health score is considered reliable
+ */
+const MIN_REQUESTS_FOR_RELIABLE_SCORE = 3;
+/**
+ * Default health configuration
+ */
+const DEFAULT_HEALTH_CONFIG = {
+    enabled: true,
+    path: DEFAULT_HEALTH_PERSISTENCE_PATH,
+};
+/**
+ * Model Health Tracker class
+ */
+export class HealthTracker {
+    healthData;
+    persistenceEnabled;
+    persistencePath;
+    healthBasedSelectionEnabled;
+    logger;
+    savePending;
+    saveTimeout;
+    // Configurable thresholds
+    responseTimeThreshold;
+    responseTimePenaltyDivisor;
+    failurePenaltyMultiplier;
+    persistenceDebounceMs;
+    constructor(config, logger) {
+        this.healthData = new Map();
+        // Parse health persistence config
+        const healthPersistence = config.healthPersistence || DEFAULT_HEALTH_CONFIG;
+        this.persistenceEnabled = healthPersistence.enabled !== false;
+        this.persistencePath = healthPersistence.path || DEFAULT_HEALTH_PERSISTENCE_PATH;
+        this.healthBasedSelectionEnabled = config.enableHealthBasedSelection || false;
+        // Initialize logger
+        this.logger = logger;
+        // Initialize save state
+        this.savePending = false;
+        // Initialize configurable thresholds (can be customized via config if needed)
+        this.responseTimeThreshold = 2000; // ms - threshold for response time penalty
+        this.responseTimePenaltyDivisor = 200; // divisor for response time penalty calculation
+        this.failurePenaltyMultiplier = 15; // penalty per consecutive failure
+        this.persistenceDebounceMs = 30000; // 30 seconds debounce for persistence
+        // Load existing state
+        if (this.persistenceEnabled) {
+            this.loadState();
+        }
+    }
+    /**
+     * Record a successful request for a model
+     */
+    recordSuccess(providerID, modelID, responseTime) {
+        const key = getModelKey(providerID, modelID);
+        const now = Date.now();
+        let health = this.healthData.get(key);
+        if (!health) {
+            // Initialize new health entry
+            health = {
+                modelKey: key,
+                providerID,
+                modelID,
+                totalRequests: 0,
+                successfulRequests: 0,
+                failedRequests: 0,
+                consecutiveFailures: 0,
+                avgResponseTime: 0,
+                lastUsed: now,
+                lastSuccess: now,
+                lastFailure: 0,
+                healthScore: 100, // Start with perfect score
+            };
+        }
+        // Update metrics
+        health.totalRequests++;
+        health.successfulRequests++;
+        health.consecutiveFailures = 0;
+        health.lastUsed = now;
+        health.lastSuccess = now;
+        // Update average response time (weighted moving average)
+        if (health.avgResponseTime === 0) {
+            health.avgResponseTime = responseTime;
+        }
+        else {
+            // Weight new response at 30%
+            health.avgResponseTime = Math.round(health.avgResponseTime * 0.7 + responseTime * 0.3);
+        }
+        // Recalculate health score
+        health.healthScore = this.calculateHealthScore(health);
+        this.healthData.set(key, health);
+        // Persist if enabled
+        if (this.persistenceEnabled) {
+            this.saveState();
+        }
+    }
+    /**
+     * Record a failed request for a model
+     */
+    recordFailure(providerID, modelID) {
+        const key = getModelKey(providerID, modelID);
+        const now = Date.now();
+        let health = this.healthData.get(key);
+        if (!health) {
+            // Initialize new health entry
+            health = {
+                modelKey: key,
+                providerID,
+                modelID,
+                totalRequests: 0,
+                successfulRequests: 0,
+                failedRequests: 0,
+                consecutiveFailures: 0,
+                avgResponseTime: 0,
+                lastUsed: now,
+                lastSuccess: 0,
+                lastFailure: now,
+                healthScore: 100,
+            };
+        }
+        // Update metrics
+        health.totalRequests++;
+        health.failedRequests++;
+        health.consecutiveFailures++;
+        health.lastUsed = now;
+        health.lastFailure = now;
+        // Recalculate health score
+        health.healthScore = this.calculateHealthScore(health);
+        this.healthData.set(key, health);
+        // Persist if enabled
+        if (this.persistenceEnabled) {
+            this.saveState();
+        }
+    }
+    /**
+     * Get the health score for a model (0-100)
+     */
+    getHealthScore(providerID, modelID) {
+        const key = getModelKey(providerID, modelID);
+        const health = this.healthData.get(key);
+        if (!health) {
+            return 100; // No data yet - assume healthy
+        }
+        return health.healthScore;
+    }
+    /**
+     * Get full health data for a model
+     */
+    getModelHealth(providerID, modelID) {
+        const key = getModelKey(providerID, modelID);
+        return this.healthData.get(key) || null;
+    }
+    /**
+     * Get all health data
+     */
+    getAllHealthData() {
+        return Array.from(this.healthData.values());
+    }
+    /**
+     * Get healthiest models from a list of candidates
+     * Returns models sorted by health score (highest first)
+     */
+    getHealthiestModels(candidates, limit) {
+        // Map candidates with health scores
+        const scored = candidates.map(model => ({
+            model,
+            score: this.getHealthScore(model.providerID, model.modelID),
+        }));
+        // Sort by health score (descending)
+        scored.sort((a, b) => b.score - a.score);
+        // Return limited results or all
+        const result = scored.map(item => item.model);
+        return limit ? result.slice(0, limit) : result;
+    }
+    /**
+     * Calculate health score based on metrics
+     * Score is 0-100, higher is healthier
+     */
+    calculateHealthScore(health) {
+        let score = 100;
+        // Penalize based on success rate
+        if (health.totalRequests >= MIN_REQUESTS_FOR_RELIABLE_SCORE) {
+            const successRate = health.successfulRequests / health.totalRequests;
+            score = Math.round(score * successRate);
+        }
+        // Penalize consecutive failures heavily
+        const failurePenalty = Math.min(health.consecutiveFailures * this.failurePenaltyMultiplier, 80);
+        score -= failurePenalty;
+        // Penalize slow response times (if we have data)
+        if (health.avgResponseTime > 0) {
+            const responseTimePenalty = Math.min(Math.round((health.avgResponseTime - this.responseTimeThreshold) / this.responseTimePenaltyDivisor), 30);
+            if (responseTimePenalty > 0) {
+                score -= responseTimePenalty;
+            }
+        }
+        // Ensure score is within valid range
+        return Math.max(0, Math.min(100, score));
+    }
+    /**
+     * Save health state to file (with debouncing)
+     */
+    saveState() {
+        if (!this.persistenceEnabled) {
+            return;
+        }
+        // If a save is already pending, don't schedule another one
+        if (this.savePending) {
+            return;
+        }
+        this.savePending = true;
+        // Clear any existing timeout
+        if (this.saveTimeout) {
+            clearTimeout(this.saveTimeout);
+        }
+        // Schedule debounced save
+        this.saveTimeout = setTimeout(() => {
+            this.performSave();
+            this.savePending = false;
+        }, this.persistenceDebounceMs);
+    }
+    /**
+     * Perform the actual save operation
+     */
+    performSave() {
+        try {
+            // Ensure directory exists
+            const dir = dirname(this.persistencePath);
+            if (!existsSync(dir)) {
+                mkdirSync(dir, { recursive: true });
+            }
+            const state = {
+                models: Object.fromEntries(this.healthData.entries()),
+                lastUpdated: Date.now(),
+            };
+            writeFileSync(this.persistencePath, JSON.stringify(state, null, 2), 'utf-8');
+        }
+        catch (error) {
+            // Use logger instead of console
+            this.logger.warn('[HealthTracker] Failed to save state', { error });
+        }
+    }
+    /**
+     * Load health state from file
+     */
+    loadState() {
+        if (!this.persistenceEnabled || !existsSync(this.persistencePath)) {
+            return;
+        }
+        try {
+            const content = readFileSync(this.persistencePath, 'utf-8');
+            const state = JSON.parse(content);
+            // Validate state structure
+            if (state.models && typeof state.models === 'object') {
+                for (const [key, health] of Object.entries(state.models)) {
+                    // Validate health object structure
+                    if (health && typeof health === 'object' && health.modelKey === key) {
+                        this.healthData.set(key, health);
+                    }
+                }
+            }
+        }
+        catch (error) {
+            // Use logger instead of console
+            this.logger.warn('[HealthTracker] Failed to load state, starting fresh', { error });
+        }
+    }
+    /**
+     * Reset health data for a specific model
+     */
+    resetModelHealth(providerID, modelID) {
+        const key = getModelKey(providerID, modelID);
+        this.healthData.delete(key);
+        if (this.persistenceEnabled) {
+            this.saveState();
+        }
+    }
+    /**
+     * Reset all health data
+     */
+    resetAllHealth() {
+        this.healthData.clear();
+        if (this.persistenceEnabled) {
+            this.saveState();
+        }
+    }
+    /**
+     * Check if health-based selection is enabled
+     */
+    isEnabled() {
+        return this.healthBasedSelectionEnabled;
+    }
+    /**
+     * Get statistics about tracked models
+     */
+    getStats() {
+        const models = Array.from(this.healthData.values());
+        const totalRequests = models.reduce((sum, h) => sum + h.totalRequests, 0);
+        const totalSuccesses = models.reduce((sum, h) => sum + h.successfulRequests, 0);
+        const totalFailures = models.reduce((sum, h) => sum + h.failedRequests, 0);
+        const avgHealthScore = models.length > 0
+            ? Math.round(models.reduce((sum, h) => sum + h.healthScore, 0) / models.length)
+            : 100;
+        const modelsWithReliableData = models.filter(h => h.totalRequests >= MIN_REQUESTS_FOR_RELIABLE_SCORE).length;
+        return {
+            totalTracked: models.length,
+            totalRequests,
+            totalSuccesses,
+            totalFailures,
+            avgHealthScore,
+            modelsWithReliableData,
+        };
+    }
+    /**
+     * Clean up old health data (models not used recently)
+     */
+    cleanupOldEntries(maxAgeMs = 30 * 24 * 60 * 60 * 1000) {
+        // Default: 30 days
+        const now = Date.now();
+        let cleaned = 0;
+        for (const [key, health] of this.healthData.entries()) {
+            if (now - health.lastUsed > maxAgeMs) {
+                this.healthData.delete(key);
+                cleaned++;
+            }
+        }
+        if (cleaned > 0 && this.persistenceEnabled) {
+            this.saveState();
+        }
+        return cleaned;
+    }
+    /**
+     * Destroy the health tracker
+     */
+    destroy() {
+        // Cancel any pending save
+        if (this.saveTimeout) {
+            clearTimeout(this.saveTimeout);
+        }
+        // Save state immediately before cleanup
+        this.performSave();
+        this.healthData.clear();
+    }
+}

package/dist/src/metrics/MetricsManager.d.ts

@@ -2,7 +2,7 @@
  * Metrics Manager - Handles metrics collection, aggregation, and reporting
  */
 import type { Logger } from '../../logger.js';
-import type { MetricsConfig, MetricsData, RateLimitMetrics, FallbackTargetMetrics, ModelPerformanceMetrics } from '../types/index.js';
+import type { MetricsConfig, MetricsData, RateLimitMetrics, FallbackTargetMetrics, ModelPerformanceMetrics, CircuitBreakerStateType } from '../types/index.js';
 /**
  * Metrics Manager class for collecting and reporting metrics
  */
@@ -61,6 +61,15 @@ export declare class MetricsManager {
      * Record a failed retry
      */
     recordRetryFailure(): void;
+    /**
+     * Record a circuit breaker state transition
+     */
+    recordCircuitBreakerStateTransition(modelKey: string, oldState: CircuitBreakerStateType, newState: CircuitBreakerStateType): void;
+    /**
+     * Helper method to update circuit breaker state counts
+     * @private
+     */
+    private updateCircuitBreakerStateCounts;
     /**
      * Get a copy of the current metrics
      */

package/dist/src/metrics/MetricsManager.js

@@ -31,6 +31,18 @@ export class MetricsManager {
                 byModel: new Map(),
             },
             modelPerformance: new Map(),
+            circuitBreaker: {
+                total: {
+                    stateTransitions: 0,
+                    opens: 0,
+                    closes: 0,
+                    halfOpens: 0,
+                    currentOpen: 0,
+                    currentHalfOpen: 0,
+                    currentClosed: 0,
+                },
+                byModel: new Map(),
+            },
             startedAt: Date.now(),
             generatedAt: Date.now(),
         };
@@ -71,6 +83,18 @@ export class MetricsManager {
                 byModel: new Map(),
             },
             modelPerformance: new Map(),
+            circuitBreaker: {
+                total: {
+                    stateTransitions: 0,
+                    opens: 0,
+                    closes: 0,
+                    halfOpens: 0,
+                    currentOpen: 0,
+                    currentHalfOpen: 0,
+                    currentClosed: 0,
+                },
+                byModel: new Map(),
+            },
             startedAt: Date.now(),
             generatedAt: Date.now(),
         };
@@ -229,6 +253,62 @@ export class MetricsManager {
             return;
         this.metrics.retries.failed++;
     }
+    /**
+     * Record a circuit breaker state transition
+     */
+    recordCircuitBreakerStateTransition(modelKey, oldState, newState) {
+        if (!this.config.enabled)
+            return;
+        // Update total metrics
+        this.metrics.circuitBreaker.total.stateTransitions++;
+        this.updateCircuitBreakerStateCounts(this.metrics.circuitBreaker.total, oldState, newState);
+        // Update model-specific metrics
+        let modelMetrics = this.metrics.circuitBreaker.byModel.get(modelKey);
+        if (!modelMetrics) {
+            modelMetrics = {
+                stateTransitions: 0,
+                opens: 0,
+                closes: 0,
+                halfOpens: 0,
+                currentOpen: 0,
+                currentHalfOpen: 0,
+                currentClosed: 1, // Start with CLOSED
+            };
+            this.metrics.circuitBreaker.byModel.set(modelKey, modelMetrics);
+        }
+        modelMetrics.stateTransitions++;
+        this.updateCircuitBreakerStateCounts(modelMetrics, oldState, newState);
+        this.metrics.circuitBreaker.byModel.set(modelKey, modelMetrics);
+    }
+    /**
+     * Helper method to update circuit breaker state counts
+     * @private
+     */
+    updateCircuitBreakerStateCounts(metrics, oldState, newState) {
+        // Update state counts based on old state
+        if (oldState === 'OPEN') {
+            metrics.currentOpen--;
+        }
+        else if (oldState === 'HALF_OPEN') {
+            metrics.currentHalfOpen--;
+        }
+        else if (oldState === 'CLOSED') {
+            metrics.currentClosed--;
+        }
+        // Update state counts based on new state
+        if (newState === 'OPEN') {
+            metrics.opens++;
+            metrics.currentOpen++;
+        }
+        else if (newState === 'HALF_OPEN') {
+            metrics.halfOpens++;
+            metrics.currentHalfOpen++;
+        }
+        else if (newState === 'CLOSED') {
+            metrics.closes++;
+            metrics.currentClosed++;
+        }
+    }
     /**
      * Get a copy of the current metrics
      */
@@ -266,6 +346,10 @@ export class MetricsManager {
                 byModel: Object.fromEntries(Array.from(metrics.retries.byModel.entries()).map(([k, v]) => [k, v])),
             },
             modelPerformance: Object.fromEntries(Array.from(metrics.modelPerformance.entries()).map(([k, v]) => [k, v])),
+            circuitBreaker: {
+                ...metrics.circuitBreaker,
+                byModel: Object.fromEntries(Array.from(metrics.circuitBreaker.byModel.entries()).map(([k, v]) => [k, v])),
+            },
             startedAt: metrics.startedAt,
             generatedAt: metrics.generatedAt,
         };
@@ -342,6 +426,31 @@ export class MetricsManager {
             }
         }
         lines.push("");
+        // Circuit Breaker
+        lines.push("Circuit Breaker:");
+        lines.push("-".repeat(40));
+        lines.push(`  State Transitions: ${this.metrics.circuitBreaker.total.stateTransitions}`);
+        lines.push(`  Opens: ${this.metrics.circuitBreaker.total.opens}`);
+        lines.push(`  Closes: ${this.metrics.circuitBreaker.total.closes}`);
+        lines.push(`  Half Opens: ${this.metrics.circuitBreaker.total.halfOpens}`);
+        lines.push("");
+        lines.push("  Current State Distribution:");
+        lines.push(`    CLOSED: ${this.metrics.circuitBreaker.total.currentClosed}`);
+        lines.push(`    HALF_OPEN: ${this.metrics.circuitBreaker.total.currentHalfOpen}`);
+        lines.push(`    OPEN: ${this.metrics.circuitBreaker.total.currentOpen}`);
+        if (this.metrics.circuitBreaker.byModel.size > 0) {
+            lines.push("");
+            lines.push("  By Model:");
+            for (const [model, data] of this.metrics.circuitBreaker.byModel.entries()) {
+                lines.push(`    ${model}:`);
+                lines.push(`      State Transitions: ${data.stateTransitions}`);
+                lines.push(`      Opens: ${data.opens}`);
+                lines.push(`      Closes: ${data.closes}`);
+                lines.push(`      Half Opens: ${data.halfOpens}`);
+                lines.push(`      Current State: ${data.currentOpen > 0 ? 'OPEN' : data.currentHalfOpen > 0 ? 'HALF_OPEN' : 'CLOSED'}`);
+            }
+        }
+        lines.push("");
         // Model Performance
         lines.push("Model Performance:");
         lines.push("-".repeat(40));
@@ -428,6 +537,34 @@ export class MetricsManager {
             ].join(","));
         }
         lines.push("");
+        // Circuit Breaker Summary CSV
+        lines.push("=== CIRCUIT_BREAKER_SUMMARY ===");
+        lines.push(`state_transitions,opens,closes,half_opens,current_open,current_half_open,current_closed`);
+        lines.push([
+            this.metrics.circuitBreaker.total.stateTransitions,
+            this.metrics.circuitBreaker.total.opens,
+            this.metrics.circuitBreaker.total.closes,
+            this.metrics.circuitBreaker.total.halfOpens,
+            this.metrics.circuitBreaker.total.currentOpen,
+            this.metrics.circuitBreaker.total.currentHalfOpen,
+            this.metrics.circuitBreaker.total.currentClosed,
+        ].join(","));
+        lines.push("");
+        // Circuit Breaker by Model CSV
+        lines.push("=== CIRCUIT_BREAKER_BY_MODEL ===");
+        lines.push("model,state_transitions,opens,closes,half_opens,current_state");
+        for (const [model, data] of this.metrics.circuitBreaker.byModel.entries()) {
+            const currentState = data.currentOpen > 0 ? 'OPEN' : data.currentHalfOpen > 0 ? 'HALF_OPEN' : 'CLOSED';
+            lines.push([
+                model,
+                data.stateTransitions,
+                data.opens,
+                data.closes,
+                data.halfOpens,
+                currentState,
+            ].join(","));
+        }
+        lines.push("");
         // Model Performance CSV
         lines.push("=== MODEL_PERFORMANCE ===");
         lines.push("model,requests,successes,failures,avg_response_time_ms,success_rate");

package/dist/src/types/index.d.ts

@@ -33,6 +33,31 @@ export interface RetryPolicy {
     jitterFactor: number;
     timeoutMs?: number;
 }
+/**
+ * Circuit breaker state
+ */
+export type CircuitBreakerStateType = 'CLOSED' | 'OPEN' | 'HALF_OPEN';
+/**
+ * Circuit breaker configuration
+ */
+export interface CircuitBreakerConfig {
+    enabled: boolean;
+    failureThreshold: number;
+    recoveryTimeoutMs: number;
+    halfOpenMaxCalls: number;
+    successThreshold: number;
+}
+/**
+ * Circuit breaker state data
+ */
+export interface CircuitBreakerState {
+    state: CircuitBreakerStateType;
+    failureCount: number;
+    successCount: number;
+    lastFailureTime: number;
+    lastSuccessTime: number;
+    nextAttemptTime: number;
+}
 /**
  * Metrics output configuration
  */
@@ -49,6 +74,53 @@ export interface MetricsConfig {
     output: MetricsOutputConfig;
     resetInterval: "hourly" | "daily" | "weekly";
 }
+/**
+ * Configuration validation options
+ */
+export interface ConfigValidationOptions {
+    strict?: boolean;
+    logWarnings?: boolean;
+}
+/**
+ * Health persistence configuration
+ */
+export interface HealthPersistenceConfig {
+    enabled: boolean;
+    path?: string;
+}
+/**
+ * Health metrics for a model
+ */
+export interface ModelHealth {
+    modelKey: string;
+    providerID: string;
+    modelID: string;
+    totalRequests: number;
+    successfulRequests: number;
+    failedRequests: number;
+    consecutiveFailures: number;
+    avgResponseTime: number;
+    lastUsed: number;
+    lastSuccess: number;
+    lastFailure: number;
+    healthScore: number;
+}
+/**
+ * Error pattern definition
+ */
+export interface ErrorPattern {
+    name: string;
+    provider?: string;
+    patterns: (string | RegExp)[];
+    priority: number;
+}
+/**
+ * Error pattern configuration
+ */
+export interface ErrorPatternsConfig {
+    custom?: ErrorPattern[];
+    enableLearning?: boolean;
+}
 /**
  * Plugin configuration
  */
@@ -60,8 +132,14 @@ export interface PluginConfig {
     maxSubagentDepth?: number;
     enableSubagentFallback?: boolean;
     retryPolicy?: RetryPolicy;
+    circuitBreaker?: CircuitBreakerConfig;
     log?: LogConfig;
     metrics?: MetricsConfig;
+    configValidation?: ConfigValidationOptions;
+    enableHealthBasedSelection?: boolean;
+    healthPersistence?: HealthPersistenceConfig;
+    verbose?: boolean;
+    errorPatterns?: ErrorPatternsConfig;
 }
 /**
  * Fallback state for tracking progress
@@ -183,6 +261,18 @@ export interface ModelPerformanceMetrics {
     failures: number;
     averageResponseTime?: number;
 }
+/**
+ * Circuit breaker metrics
+ */
+export interface CircuitBreakerMetrics {
+    stateTransitions: number;
+    opens: number;
+    closes: number;
+    halfOpens: number;
+    currentOpen: number;
+    currentHalfOpen: number;
+    currentClosed: number;
+}
 /**
  * Retry metrics
  */
@@ -210,6 +300,10 @@ export interface MetricsData {
     };
     retries: RetryMetrics;
     modelPerformance: Map<string, ModelPerformanceMetrics>;
+    circuitBreaker: {
+        total: CircuitBreakerMetrics;
+        byModel: Map<string, CircuitBreakerMetrics>;
+    };
     startedAt: number;
     generatedAt: number;
 }
@@ -314,6 +408,10 @@ export declare const DEFAULT_FALLBACK_MODELS: FallbackModel[];
  * Default retry policy
  */
 export declare const DEFAULT_RETRY_POLICY: RetryPolicy;
+/**
+ * Default circuit breaker configuration
+ */
+export declare const DEFAULT_CIRCUIT_BREAKER_CONFIG: CircuitBreakerConfig;
 /**
  * Valid fallback modes
  */