@azumag/opencode-rate-limit-fallback 1.31.0 → 1.36.0

package/dist/src/errors/PatternRegistry.js

@@ -0,0 +1,234 @@
+/**
+ * Error Pattern Registry for rate limit error detection
+ */
+/**
+ * Error Pattern Registry class
+ * Manages and matches error patterns for rate limit detection
+ */
+export class ErrorPatternRegistry {
+    patterns = [];
+    logger;
+    constructor(logger) {
+        // Initialize logger
+        this.logger = logger || {
+            debug: () => { },
+            info: () => { },
+            warn: () => { },
+            error: () => { },
+        };
+        this.registerDefaultPatterns();
+    }
+    /**
+     * Register default rate limit error patterns
+     */
+    registerDefaultPatterns() {
+        // Common rate limit patterns (provider-agnostic)
+        this.register({
+            name: 'http-429',
+            patterns: [/\\b429\\b/gi], // HTTP 429 status code with word boundaries
+            priority: 100,
+        });
+        this.register({
+            name: 'rate-limit-general',
+            patterns: [
+                'rate limit',
+                'rate_limit',
+                'ratelimit',
+                'too many requests',
+                'quota exceeded',
+            ],
+            priority: 90,
+        });
+        // Anthropic-specific patterns
+        this.register({
+            name: 'anthropic-rate-limit',
+            provider: 'anthropic',
+            patterns: [
+                'rate limit exceeded',
+                'too many requests',
+                'quota exceeded',
+                'rate_limit_error',
+                'overloaded',
+            ],
+            priority: 80,
+        });
+        // Google/Gemini-specific patterns
+        this.register({
+            name: 'google-rate-limit',
+            provider: 'google',
+            patterns: [
+                'quota exceeded',
+                'resource exhausted',
+                'rate limit exceeded',
+                'user rate limit exceeded',
+                'daily limit exceeded',
+                '429',
+            ],
+            priority: 80,
+        });
+        // OpenAI-specific patterns
+        this.register({
+            name: 'openai-rate-limit',
+            provider: 'openai',
+            patterns: [
+                'rate limit exceeded',
+                'you exceeded your current quota',
+                'quota exceeded',
+                'maximum requests per minute reached',
+                'insufficient_quota',
+            ],
+            priority: 80,
+        });
+    }
+    /**
+     * Register a new error pattern
+     */
+    register(pattern) {
+        // Check for duplicate names
+        const existingIndex = this.patterns.findIndex(p => p.name === pattern.name);
+        if (existingIndex >= 0) {
+            // Update existing pattern
+            this.patterns[existingIndex] = pattern;
+        }
+        else {
+            // Add new pattern, sorted by priority (higher priority first)
+            this.patterns.push(pattern);
+            this.patterns.sort((a, b) => b.priority - a.priority);
+        }
+    }
+    /**
+     * Register multiple error patterns
+     */
+    registerMany(patterns) {
+        for (const pattern of patterns) {
+            this.register(pattern);
+        }
+    }
+    /**
+     * Check if an error matches any registered rate limit pattern
+     */
+    isRateLimitError(error) {
+        return this.getMatchedPattern(error) !== null;
+    }
+    /**
+     * Get the matched pattern for an error, or null if no match
+     */
+    getMatchedPattern(error) {
+        if (!error || typeof error !== 'object') {
+            return null;
+        }
+        const err = error;
+        // Extract error text to search
+        const responseBody = String(err.data?.responseBody || '');
+        const message = String(err.data?.message || err.message || '');
+        const name = String(err.name || '');
+        const statusCode = err.data?.statusCode?.toString() || '';
+        // Combine all text sources for matching
+        const allText = [responseBody, message, name, statusCode].join(' ').toLowerCase();
+        // Check each pattern
+        for (const pattern of this.patterns) {
+            for (const patternStr of pattern.patterns) {
+                let match = false;
+                if (typeof patternStr === 'string') {
+                    // String matching (case-insensitive)
+                    if (allText.includes(patternStr.toLowerCase())) {
+                        match = true;
+                    }
+                }
+                else if (patternStr instanceof RegExp) {
+                    // RegExp matching
+                    if (patternStr.test(allText)) {
+                        match = true;
+                    }
+                }
+                if (match) {
+                    return pattern;
+                }
+            }
+        }
+        return null;
+    }
+    /**
+     * Get all registered patterns
+     */
+    getAllPatterns() {
+        return [...this.patterns];
+    }
+    /**
+     * Get patterns for a specific provider
+     */
+    getPatternsForProvider(provider) {
+        return this.patterns.filter(p => !p.provider || p.provider === provider);
+    }
+    /**
+     * Get patterns by name
+     */
+    getPatternByName(name) {
+        return this.patterns.find(p => p.name === name);
+    }
+    /**
+     * Remove a pattern by name
+     */
+    removePattern(name) {
+        const index = this.patterns.findIndex(p => p.name === name);
+        if (index >= 0) {
+            this.patterns.splice(index, 1);
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Clear all patterns (including default ones)
+     */
+    clearAllPatterns() {
+        this.patterns = [];
+    }
+    /**
+     * Reset to default patterns only
+     */
+    resetToDefaults() {
+        this.clearAllPatterns();
+        this.registerDefaultPatterns();
+    }
+    /**
+     * Learn a new pattern from an error (for future ML-based learning)
+     * Currently disabled - patterns must be manually registered
+     */
+    addLearnedPattern(_error) {
+        // Placeholder for future ML-based pattern learning
+        // For now, patterns must be manually registered via config
+        this.logger.warn('[ErrorPatternRegistry] Automatic pattern learning is not enabled. Patterns must be manually registered via configuration.');
+    }
+    /**
+     * Get statistics about registered patterns
+     */
+    getStats() {
+        const byProvider = {};
+        const byPriority = {};
+        for (const pattern of this.patterns) {
+            // Count by provider
+            const provider = pattern.provider || 'generic';
+            byProvider[provider] = (byProvider[provider] || 0) + 1;
+            // Count by priority range
+            const priorityRange = this.getPriorityRange(pattern.priority);
+            byPriority[priorityRange] = (byPriority[priorityRange] || 0) + 1;
+        }
+        return {
+            total: this.patterns.length,
+            byProvider,
+            byPriority,
+        };
+    }
+    /**
+     * Get a readable priority range string
+     */
+    getPriorityRange(priority) {
+        if (priority >= 90)
+            return 'high (90-100)';
+        if (priority >= 70)
+            return 'medium (70-89)';
+        if (priority >= 50)
+            return 'low (50-69)';
+        return 'very low (<50)';
+    }
+}

package/dist/src/fallback/FallbackHandler.d.ts

@@ -5,6 +5,7 @@ import type { Logger } from '../../logger.js';
 import type { FallbackModel, PluginConfig, OpenCodeClient, MessagePart, SessionHierarchy } from '../types/index.js';
 import { MetricsManager } from '../metrics/MetricsManager.js';
 import type { SubagentTracker } from '../session/SubagentTracker.js';
+import type { HealthTracker } from '../health/HealthTracker.js';
 /**
  * Fallback Handler class for orchestrating the fallback retry flow
  */
@@ -21,7 +22,9 @@ export declare class FallbackHandler {
     private metricsManager;
     private subagentTracker;
     private retryManager;
-    constructor(config: PluginConfig, client: OpenCodeClient, logger: Logger, metricsManager: MetricsManager, subagentTracker: SubagentTracker);
+    private circuitBreaker?;
+    private healthTracker?;
+    constructor(config: PluginConfig, client: OpenCodeClient, logger: Logger, metricsManager: MetricsManager, subagentTracker: SubagentTracker, healthTracker?: HealthTracker);
     /**
      * Check and mark fallback in progress for deduplication
      */

package/dist/src/fallback/FallbackHandler.js

@@ -3,6 +3,7 @@
  */
 import { SESSION_ENTRY_TTL_MS } from '../types/index.js';
 import { ModelSelector } from './ModelSelector.js';
+import { CircuitBreaker } from '../circuitbreaker/CircuitBreaker.js';
 import { extractMessageParts, convertPartsToSDKFormat, safeShowToast, getStateKey, getModelKey, DEDUP_WINDOW_MS, STATE_TIMEOUT_MS } from '../utils/helpers.js';
 import { RetryManager } from '../retry/RetryManager.js';
 /**
@@ -24,13 +25,22 @@ export class FallbackHandler {
     subagentTracker;
     // Retry manager reference
     retryManager;
-    constructor(config, client, logger, metricsManager, subagentTracker) {
+    // Circuit breaker reference
+    circuitBreaker;
+    // Health tracker reference
+    healthTracker;
+    constructor(config, client, logger, metricsManager, subagentTracker, healthTracker) {
         this.config = config;
         this.client = client;
         this.logger = logger;
-        this.modelSelector = new ModelSelector(config, client);
         this.metricsManager = metricsManager;
         this.subagentTracker = subagentTracker;
+        this.healthTracker = healthTracker;
+        // Initialize circuit breaker if enabled
+        if (config.circuitBreaker?.enabled) {
+            this.circuitBreaker = new CircuitBreaker(config.circuitBreaker, logger, metricsManager, client);
+        }
+        this.modelSelector = new ModelSelector(config, client, this.circuitBreaker, healthTracker);
         this.currentSessionModel = new Map();
         this.modelRequestStartTimes = new Map();
         this.retryState = new Map();
@@ -154,6 +164,10 @@ export class FallbackHandler {
             if (currentProviderID && currentModelID && this.metricsManager) {
                 this.metricsManager.recordRateLimit(currentProviderID, currentModelID);
             }
+            // Record health failure for current model (if health tracking is enabled)
+            if (this.healthTracker && currentProviderID && currentModelID) {
+                this.healthTracker.recordFailure(currentProviderID, currentModelID);
+            }
             // Abort current session with error handling
             await this.abortSession(targetSessionID);
             await safeShowToast(this.client, {
@@ -269,8 +283,15 @@ export class FallbackHandler {
                 messageID: lastUserMessage.info.id,
                 timestamp: Date.now(),
             });
+            // Record retry start time for health tracking
+            const retryStartTime = Date.now();
             // Retry with the selected model
             await this.retryWithModel(dedupSessionID, nextModel, parts, hierarchy);
+            // Record health success for fallback model
+            if (this.healthTracker) {
+                const responseTime = Date.now() - retryStartTime;
+                this.healthTracker.recordSuccess(nextModel.providerID, nextModel.modelID, responseTime);
+            }
             // Record retry success
             this.retryManager.recordSuccess(dedupSessionID, nextModel.modelID);
             if (this.metricsManager) {
@@ -301,6 +322,11 @@ export class FallbackHandler {
             // Non-rate-limit error - record model failure metric
             const tracked = this.currentSessionModel.get(sessionID);
             if (tracked) {
+                // Record failure to circuit breaker (isRateLimit = false)
+                if (this.circuitBreaker) {
+                    const modelKey = getModelKey(tracked.providerID, tracked.modelID);
+                    this.circuitBreaker.recordFailure(modelKey, false);
+                }
                 if (this.metricsManager) {
                     this.metricsManager.recordModelFailure(tracked.providerID, tracked.modelID);
                     // Check if this was a fallback attempt and record failure
@@ -326,6 +352,11 @@ export class FallbackHandler {
                 // Record fallback success metric
                 const tracked = this.currentSessionModel.get(sessionID);
                 if (tracked) {
+                    // Record success to circuit breaker
+                    if (this.circuitBreaker) {
+                        const modelKey = getModelKey(tracked.providerID, tracked.modelID);
+                        this.circuitBreaker.recordSuccess(modelKey);
+                    }
                     if (this.metricsManager) {
                         this.metricsManager.recordFallbackSuccess(tracked.providerID, tracked.modelID, fallbackInfo.timestamp);
                         // Record model performance metric
@@ -374,6 +405,10 @@ export class FallbackHandler {
         }
         this.modelSelector.cleanupStaleEntries();
         this.retryManager.cleanupStaleEntries(SESSION_ENTRY_TTL_MS);
+        // Clean up circuit breaker stale entries
+        if (this.circuitBreaker) {
+            this.circuitBreaker.cleanupStaleEntries();
+        }
     }
     /**
      * Clean up all resources
@@ -385,5 +420,9 @@ export class FallbackHandler {
         this.fallbackInProgress.clear();
         this.fallbackMessages.clear();
         this.retryManager.destroy();
+        // Destroy circuit breaker
+        if (this.circuitBreaker) {
+            this.circuitBreaker.destroy();
+        }
     }
 }

package/dist/src/fallback/ModelSelector.d.ts

@@ -2,6 +2,8 @@
  * Model selection logic based on fallback mode
  */
 import type { FallbackModel, PluginConfig, OpenCodeClient } from '../types/index.js';
+import type { CircuitBreaker } from '../circuitbreaker/index.js';
+import type { HealthTracker } from '../health/HealthTracker.js';
 /**
  * Model Selector class for handling model selection strategies
  */
@@ -9,7 +11,9 @@ export declare class ModelSelector {
     private rateLimitedModels;
     private config;
     private client;
-    constructor(config: PluginConfig, client: OpenCodeClient);
+    private circuitBreaker?;
+    private healthTracker?;
+    constructor(config: PluginConfig, client: OpenCodeClient, circuitBreaker?: CircuitBreaker, healthTracker?: HealthTracker);
     /**
      * Check if a model is currently rate limited
      */
@@ -22,6 +26,10 @@ export declare class ModelSelector {
      * Find the next available model that is not rate limited
      */
     private findNextAvailableModel;
+    /**
+     * Check if a model is available (not rate limited and not blocked by circuit breaker)
+     */
+    private isModelAvailable;
     /**
      * Apply the fallback mode logic
      */

package/dist/src/fallback/ModelSelector.js

@@ -10,9 +10,13 @@ export class ModelSelector {
     rateLimitedModels;
     config;
     client;
-    constructor(config, client) {
+    circuitBreaker;
+    healthTracker;
+    constructor(config, client, circuitBreaker, healthTracker) {
         this.config = config;
         this.client = client;
+        this.circuitBreaker = circuitBreaker;
+        this.healthTracker = healthTracker;
         this.rateLimitedModels = new Map();
     }
     /**
@@ -44,11 +48,25 @@ export class ModelSelector {
         const startIndex = this.config.fallbackModels.findIndex(m => getModelKey(m.providerID, m.modelID) === currentKey);
         // If current model is not in the fallback list (startIndex is -1), start from 0
         const searchStartIndex = Math.max(0, startIndex);
+        // Get available models
+        const candidates = [];
+        for (let i = 0; i < this.config.fallbackModels.length; i++) {
+            const model = this.config.fallbackModels[i];
+            const key = getModelKey(model.providerID, model.modelID);
+            if (!attemptedModels.has(key) && !this.isModelRateLimited(model.providerID, model.modelID) && this.isModelAvailable(model.providerID, model.modelID)) {
+                candidates.push(model);
+            }
+        }
+        // Sort by health score if health tracker is enabled
+        if (this.healthTracker && this.config.enableHealthBasedSelection) {
+            const healthiest = this.healthTracker.getHealthiestModels(candidates);
+            return healthiest[0] || null;
+        }
         // Search forward from current position
         for (let i = searchStartIndex + 1; i < this.config.fallbackModels.length; i++) {
             const model = this.config.fallbackModels[i];
             const key = getModelKey(model.providerID, model.modelID);
-            if (!attemptedModels.has(key) && !this.isModelRateLimited(model.providerID, model.modelID)) {
+            if (!attemptedModels.has(key) && !this.isModelRateLimited(model.providerID, model.modelID) && this.isModelAvailable(model.providerID, model.modelID)) {
                 return model;
             }
         }
@@ -56,12 +74,23 @@ export class ModelSelector {
         for (let i = 0; i <= searchStartIndex && i < this.config.fallbackModels.length; i++) {
             const model = this.config.fallbackModels[i];
             const key = getModelKey(model.providerID, model.modelID);
-            if (!attemptedModels.has(key) && !this.isModelRateLimited(model.providerID, model.modelID)) {
+            if (!attemptedModels.has(key) && !this.isModelRateLimited(model.providerID, model.modelID) && this.isModelAvailable(model.providerID, model.modelID)) {
                 return model;
             }
         }
         return null;
     }
+    /**
+     * Check if a model is available (not rate limited and not blocked by circuit breaker)
+     */
+    isModelAvailable(providerID, modelID) {
+        // Check circuit breaker if enabled
+        if (this.circuitBreaker && this.config.circuitBreaker?.enabled) {
+            const modelKey = getModelKey(providerID, modelID);
+            return this.circuitBreaker.canExecute(modelKey);
+        }
+        return true;
+    }
     /**
      * Apply the fallback mode logic
      */
@@ -79,7 +108,7 @@ export class ModelSelector {
             const lastModel = this.config.fallbackModels[this.config.fallbackModels.length - 1];
             if (lastModel) {
                 const isLastModelCurrent = currentProviderID === lastModel.providerID && currentModelID === lastModel.modelID;
-                if (!isLastModelCurrent && !this.isModelRateLimited(lastModel.providerID, lastModel.modelID)) {
+                if (!isLastModelCurrent && !this.isModelRateLimited(lastModel.providerID, lastModel.modelID) && this.isModelAvailable(lastModel.providerID, lastModel.modelID)) {
                     // Use the last model for one more try
                     safeShowToast(this.client, {
                         body: {

package/dist/src/health/HealthTracker.d.ts

@@ -0,0 +1,96 @@
+/**
+ * Model Health Tracker
+ * Tracks model success rates and response times for health-based selection
+ */
+import { Logger } from '../../logger.js';
+import type { FallbackModel, PluginConfig, ModelHealth } from '../types/index.js';
+/**
+ * Model Health Tracker class
+ */
+export declare class HealthTracker {
+    private healthData;
+    private persistenceEnabled;
+    private persistencePath;
+    private healthBasedSelectionEnabled;
+    private logger;
+    private savePending;
+    private saveTimeout?;
+    private responseTimeThreshold;
+    private responseTimePenaltyDivisor;
+    private failurePenaltyMultiplier;
+    private persistenceDebounceMs;
+    constructor(config: PluginConfig, logger: Logger);
+    /**
+     * Record a successful request for a model
+     */
+    recordSuccess(providerID: string, modelID: string, responseTime: number): void;
+    /**
+     * Record a failed request for a model
+     */
+    recordFailure(providerID: string, modelID: string): void;
+    /**
+     * Get the health score for a model (0-100)
+     */
+    getHealthScore(providerID: string, modelID: string): number;
+    /**
+     * Get full health data for a model
+     */
+    getModelHealth(providerID: string, modelID: string): ModelHealth | null;
+    /**
+     * Get all health data
+     */
+    getAllHealthData(): ModelHealth[];
+    /**
+     * Get healthiest models from a list of candidates
+     * Returns models sorted by health score (highest first)
+     */
+    getHealthiestModels(candidates: FallbackModel[], limit?: number): FallbackModel[];
+    /**
+     * Calculate health score based on metrics
+     * Score is 0-100, higher is healthier
+     */
+    private calculateHealthScore;
+    /**
+     * Save health state to file (with debouncing)
+     */
+    saveState(): void;
+    /**
+     * Perform the actual save operation
+     */
+    private performSave;
+    /**
+     * Load health state from file
+     */
+    loadState(): void;
+    /**
+     * Reset health data for a specific model
+     */
+    resetModelHealth(providerID: string, modelID: string): void;
+    /**
+     * Reset all health data
+     */
+    resetAllHealth(): void;
+    /**
+     * Check if health-based selection is enabled
+     */
+    isEnabled(): boolean;
+    /**
+     * Get statistics about tracked models
+     */
+    getStats(): {
+        totalTracked: number;
+        totalRequests: number;
+        totalSuccesses: number;
+        totalFailures: number;
+        avgHealthScore: number;
+        modelsWithReliableData: number;
+    };
+    /**
+     * Clean up old health data (models not used recently)
+     */
+    cleanupOldEntries(maxAgeMs?: number): number;
+    /**
+     * Destroy the health tracker
+     */
+    destroy(): void;
+}