@splashcodex/api-key-manager 1.0.0

package/README.md

@@ -0,0 +1,81 @@
+# @splashcodex/api-key-manager
+
+A robust, universal API Key Rotation and Management system designed for high-availability applications using rate-limited APIs (like Google Gemini).
+
+## Features
+
+-   **🔄 Automatic Key Rotation**: Seamlessly switches to the next available key upon exhaustion.
+-   **🔌 Circuit Breaker Pattern**: Automatically "opens" the circuit for keys that return 429s or 500s, preventing wasted requests.
+-   **💾 Persistence**: Remembers key states (failures, cooldowns) across restarts using local storage.
+-   **🧠 Smart Error Classification**: Distinguishes between transient errors (retryable), quota errors (cooldown), and auth errors (dead).
+-   **⏱️ Jittered Exponential Backoff**: Prevents thundering herd problems during retries.
+
+## Installation
+
+```bash
+npm install @splashcodex/api-key-manager
+```
+
+## Usage
+
+### 1. Initialize
+
+```typescript
+import { ApiKeyManager } from '@splashcodex/api-key-manager';
+
+// Initialize with a pool of keys
+const apiKeys = [
+  "AIzaSy...",
+  "AIzaSy...",
+  "AIzaSy..."
+];
+
+const manager = new ApiKeyManager(apiKeys);
+```
+
+### 2. Get a Key
+
+```typescript
+const key = manager.getKey();
+
+if (!key) {
+  throw new Error("All API keys are exhausted or cooling down.");
+}
+
+// Use the key with your API client
+const client = new GoogleGenerativeAI(key);
+```
+
+### 3. Report Results (The Feedback Loop)
+
+Crucial Step: You must report success or failure back to the manager so it can update the circuit state.
+
+```typescript
+try {
+  const response = await client.generateContent(prompt);
+
+  // ✅ REPORT SUCCESS
+  manager.markSuccess(key);
+
+  return response;
+} catch (error) {
+  // ❌ REPORT FAILURE
+  // The manager will automatically classify the error (Quota vs Auth vs Transient)
+  const classification = manager.classifyError(error);
+  manager.markFailed(key, classification);
+
+  throw error; // Re-throw or handle accordingly
+}
+```
+
+## Error Classification
+
+The `classifyError` method automatically detects:
+-   **429 / Quota**: Marks key as 'OPEN' for a cooldown period (default 5 mins).
+-   **403 / Auth**: Marks key as 'DEAD' permanently.
+-   **500 / Transient**: Marks key as 'OPEN' for a short cooldown (default 1 min).
+-   **FinishReason: SAFETY/RECITATION**: specific to Gemini, does not penalize the key.
+
+## License
+
+ISC

package/dist/index.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Universal ApiKeyManager v2.0
+ * Implements: Rotation, Circuit Breaker, Persistence, Exponential Backoff
+ * Gemini-Specific: finishReason handling, Safety blocks, RECITATION detection
+ */
+export interface KeyState {
+    key: string;
+    failCount: number;
+    failedAt: number | null;
+    isQuotaError: boolean;
+    circuitState: 'CLOSED' | 'OPEN' | 'HALF_OPEN' | 'DEAD';
+    lastUsed: number;
+    successCount: number;
+    totalRequests: number;
+    halfOpenTestTime: number | null;
+    customCooldown: number | null;
+}
+export type ErrorType = 'QUOTA' | 'TRANSIENT' | 'AUTH' | 'BAD_REQUEST' | 'SAFETY' | 'RECITATION' | 'UNKNOWN';
+export interface ErrorClassification {
+    type: ErrorType;
+    retryable: boolean;
+    cooldownMs: number;
+    markKeyFailed: boolean;
+    markKeyDead: boolean;
+}
+export interface ApiKeyManagerStats {
+    total: number;
+    healthy: number;
+    cooling: number;
+    dead: number;
+}
+export declare class ApiKeyManager {
+    private keys;
+    private storageKey;
+    private storage;
+    constructor(initialKeys: string[], storage?: any);
+    /**
+     * CLASSIFIES an error to determine handling strategy
+     */
+    classifyError(error: any, finishReason?: string): ErrorClassification;
+    /**
+     * Parses Retry-After header from error response
+     */
+    private parseRetryAfter;
+    /**
+     * HEALTH CHECK
+     * Determines if a key is usable based on Circuit Breaker logic
+     */
+    private isOnCooldown;
+    /**
+     * CORE ROTATION LOGIC
+     * Returns the best available key
+     */
+    getKey(): string | null;
+    /**
+     * Get count of healthy (non-DEAD) keys
+     */
+    getKeyCount(): number;
+    /**
+     * FEEDBACK LOOP: Success
+     */
+    markSuccess(key: string): void;
+    /**
+     * FEEDBACK LOOP: Failure
+     * Enhanced with error classification
+     */
+    markFailed(key: string, classification: ErrorClassification): void;
+    /**
+     * Legacy markFailed for backward compatibility
+     */
+    markFailedLegacy(key: string, isQuota?: boolean): void;
+    /**
+     * Calculate backoff delay with jitter
+     */
+    calculateBackoff(attempt: number): number;
+    /**
+     * Get health statistics
+     */
+    getStats(): ApiKeyManagerStats;
+    _getKeys(): KeyState[];
+    private saveState;
+    private loadState;
+}

package/dist/index.js

@@ -0,0 +1,308 @@
+"use strict";
+/**
+ * Universal ApiKeyManager v2.0
+ * Implements: Rotation, Circuit Breaker, Persistence, Exponential Backoff
+ * Gemini-Specific: finishReason handling, Safety blocks, RECITATION detection
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ApiKeyManager = void 0;
+const CONFIG = {
+    MAX_CONSECUTIVE_FAILURES: 5,
+    COOLDOWN_TRANSIENT: 60 * 1000, // 1 minute
+    COOLDOWN_QUOTA: 5 * 60 * 1000, // 5 minutes (default if no Retry-After)
+    COOLDOWN_QUOTA_DAILY: 60 * 60 * 1000, // 1 hour for RPD exhaustion
+    HALF_OPEN_TEST_DELAY: 60 * 1000, // 1 minute after open
+    MAX_BACKOFF: 64 * 1000, // 64 seconds max
+    BASE_BACKOFF: 1000, // 1 second base
+};
+// Error classification patterns
+const ERROR_PATTERNS = {
+    isQuotaError: /429|quota|exhausted|resource.?exhausted|too.?many.?requests|rate.?limit/i,
+    isAuthError: /403|permission.?denied|invalid.?api.?key|unauthorized|unauthenticated/i,
+    isSafetyBlock: /safety|blocked|recitation|harmful/i,
+    isTransient: /500|502|503|504|internal|unavailable|deadline|timeout|overloaded/i,
+    isBadRequest: /400|invalid.?argument|failed.?precondition|malformed|not.?found|404/i,
+};
+class ApiKeyManager {
+    keys = [];
+    storageKey = 'api_rotation_state_v2';
+    // Simplified Storage interface for Node.js environment
+    storage;
+    constructor(initialKeys, storage) {
+        // Simple in-memory storage mock if none provided (for testing/Node)
+        this.storage = storage || {
+            getItem: () => null,
+            setItem: () => { },
+        };
+        // 1. Sanitize & Deduplicate Keys "Even Better"
+        const uniqueKeys = new Set();
+        initialKeys.forEach(rawKey => {
+            // Handle comma-separated keys if they appear in a single string
+            const parts = rawKey.split(',').map(s => s.trim()).filter(s => s.length > 0);
+            parts.forEach(p => uniqueKeys.add(p));
+        });
+        // 2. Warn about duplicates/empty
+        const sanitizedCount = uniqueKeys.size;
+        const rawLength = initialKeys.length; // Might be misleading if split happened, but roughly
+        if (sanitizedCount < rawLength) {
+            console.warn(`[ApiKeyManager] Optimized key pool: Removed ${rawLength - sanitizedCount} duplicate/empty keys.`);
+        }
+        this.keys = Array.from(uniqueKeys).map(k => ({
+            key: k,
+            failCount: 0,
+            failedAt: null,
+            isQuotaError: false,
+            circuitState: 'CLOSED',
+            lastUsed: 0,
+            successCount: 0,
+            totalRequests: 0,
+            halfOpenTestTime: null,
+            customCooldown: null,
+        }));
+        this.loadState();
+    }
+    /**
+     * CLASSIFIES an error to determine handling strategy
+     */
+    classifyError(error, finishReason) {
+        const status = error?.status || error?.response?.status;
+        const message = error?.message || error?.error?.message || String(error);
+        // 1. Check finishReason first (for 200 responses with content issues)
+        if (finishReason === 'SAFETY') {
+            return { type: 'SAFETY', retryable: false, cooldownMs: 0, markKeyFailed: false, markKeyDead: false };
+        }
+        if (finishReason === 'RECITATION') {
+            return { type: 'RECITATION', retryable: false, cooldownMs: 0, markKeyFailed: false, markKeyDead: false };
+        }
+        // 2. Check HTTP status codes
+        if (status === 403 || ERROR_PATTERNS.isAuthError.test(message)) {
+            return { type: 'AUTH', retryable: false, cooldownMs: Infinity, markKeyFailed: true, markKeyDead: true };
+        }
+        if (status === 429 || ERROR_PATTERNS.isQuotaError.test(message)) {
+            const retryAfter = this.parseRetryAfter(error);
+            return {
+                type: 'QUOTA',
+                retryable: true,
+                cooldownMs: retryAfter || CONFIG.COOLDOWN_QUOTA,
+                markKeyFailed: true,
+                markKeyDead: false
+            };
+        }
+        if (status === 400 || ERROR_PATTERNS.isBadRequest.test(message)) {
+            return { type: 'BAD_REQUEST', retryable: false, cooldownMs: 0, markKeyFailed: false, markKeyDead: false };
+        }
+        if (ERROR_PATTERNS.isTransient.test(message) || [500, 502, 503, 504].includes(status)) {
+            return { type: 'TRANSIENT', retryable: true, cooldownMs: CONFIG.COOLDOWN_TRANSIENT, markKeyFailed: true, markKeyDead: false };
+        }
+        return { type: 'UNKNOWN', retryable: true, cooldownMs: CONFIG.COOLDOWN_TRANSIENT, markKeyFailed: true, markKeyDead: false };
+    }
+    /**
+     * Parses Retry-After header from error response
+     */
+    parseRetryAfter(error) {
+        const retryAfter = error?.response?.headers?.['retry-after'] ||
+            error?.headers?.['retry-after'] ||
+            error?.retryAfter;
+        if (!retryAfter)
+            return null;
+        // If it's a number (seconds)
+        const seconds = parseInt(retryAfter, 10);
+        if (!isNaN(seconds))
+            return seconds * 1000;
+        // If it's a date string
+        const date = Date.parse(retryAfter);
+        if (!isNaN(date))
+            return Math.max(0, date - Date.now());
+        return null;
+    }
+    /**
+     * HEALTH CHECK
+     * Determines if a key is usable based on Circuit Breaker logic
+     */
+    isOnCooldown(k) {
+        // Dead keys are NEVER usable
+        if (k.circuitState === 'DEAD')
+            return true;
+        const now = Date.now();
+        if (k.circuitState === 'OPEN') {
+            // Check if ready for HALF_OPEN test
+            if (k.halfOpenTestTime && now >= k.halfOpenTestTime) {
+                k.circuitState = 'HALF_OPEN';
+                return false;
+            }
+            return true;
+        }
+        // Additional safeguard for custom cooldowns
+        if (k.failedAt && k.customCooldown) {
+            if (now - k.failedAt < k.customCooldown)
+                return true;
+        }
+        // Standard cooldown check
+        if (k.failedAt) {
+            const cooldown = k.isQuotaError ? CONFIG.COOLDOWN_QUOTA : CONFIG.COOLDOWN_TRANSIENT;
+            if (now - k.failedAt < cooldown)
+                return true;
+        }
+        return false;
+    }
+    /**
+     * CORE ROTATION LOGIC
+     * Returns the best available key
+     */
+    getKey() {
+        // 1. Filter out dead and cooling down keys
+        const candidates = this.keys.filter(k => k.circuitState !== 'DEAD' && !this.isOnCooldown(k));
+        if (candidates.length === 0) {
+            // FALLBACK: Return oldest failed key (excluding DEAD)
+            const nonDead = this.keys.filter(k => k.circuitState !== 'DEAD');
+            if (nonDead.length === 0)
+                return null; // All keys are dead!
+            return nonDead.sort((a, b) => (a.failedAt || 0) - (b.failedAt || 0))[0]?.key || null;
+        }
+        // 2. Sort candidates: Pristine > Fewest Failures > Least Recently Used
+        candidates.sort((a, b) => {
+            if (a.failCount !== b.failCount)
+                return a.failCount - b.failCount;
+            return a.lastUsed - b.lastUsed;
+        });
+        const selected = candidates[0];
+        selected.lastUsed = Date.now();
+        this.saveState();
+        return selected.key;
+    }
+    /**
+     * Get count of healthy (non-DEAD) keys
+     */
+    getKeyCount() {
+        return this.keys.filter(k => k.circuitState !== 'DEAD').length;
+    }
+    /**
+     * FEEDBACK LOOP: Success
+     */
+    markSuccess(key) {
+        const k = this.keys.find(x => x.key === key);
+        if (!k)
+            return;
+        if (k.circuitState !== 'CLOSED' && k.circuitState !== 'DEAD') {
+            console.log(`[Key Recovered] ...${key.slice(-4)}`);
+        }
+        k.circuitState = 'CLOSED';
+        k.failCount = 0;
+        k.failedAt = null;
+        k.isQuotaError = false;
+        k.customCooldown = null;
+        k.successCount++;
+        k.totalRequests++;
+        this.saveState();
+    }
+    /**
+     * FEEDBACK LOOP: Failure
+     * Enhanced with error classification
+     */
+    markFailed(key, classification) {
+        const k = this.keys.find(x => x.key === key);
+        if (!k)
+            return;
+        // Don't modify DEAD keys
+        if (k.circuitState === 'DEAD')
+            return;
+        // If this error shouldn't mark the key as failed, skip
+        if (!classification.markKeyFailed)
+            return;
+        k.failedAt = Date.now();
+        k.failCount++;
+        k.totalRequests++;
+        k.isQuotaError = classification.type === 'QUOTA';
+        k.customCooldown = classification.cooldownMs || null;
+        // Permanent death for auth errors
+        if (classification.markKeyDead) {
+            k.circuitState = 'DEAD';
+            console.error(`[Key DEAD] ...${key.slice(-4)} - Permanently removed from rotation`);
+            this.saveState();
+            return;
+        }
+        // State Transitions
+        if (k.circuitState === 'HALF_OPEN') {
+            k.circuitState = 'OPEN';
+            k.halfOpenTestTime = Date.now() + CONFIG.HALF_OPEN_TEST_DELAY;
+        }
+        else if (k.failCount >= CONFIG.MAX_CONSECUTIVE_FAILURES || classification.type === 'QUOTA') {
+            k.circuitState = 'OPEN';
+            k.halfOpenTestTime = Date.now() + (classification.cooldownMs || CONFIG.HALF_OPEN_TEST_DELAY);
+        }
+        this.saveState();
+    }
+    /**
+     * Legacy markFailed for backward compatibility
+     */
+    markFailedLegacy(key, isQuota = false) {
+        this.markFailed(key, {
+            type: isQuota ? 'QUOTA' : 'TRANSIENT',
+            retryable: true,
+            cooldownMs: isQuota ? CONFIG.COOLDOWN_QUOTA : CONFIG.COOLDOWN_TRANSIENT,
+            markKeyFailed: true,
+            markKeyDead: false,
+        });
+    }
+    /**
+     * Calculate backoff delay with jitter
+     */
+    calculateBackoff(attempt) {
+        const exponential = CONFIG.BASE_BACKOFF * Math.pow(2, attempt);
+        const capped = Math.min(exponential, CONFIG.MAX_BACKOFF);
+        const jitter = Math.random() * 1000;
+        return capped + jitter;
+    }
+    // ... inside class ...
+    /**
+     * Get health statistics
+     */
+    getStats() {
+        const total = this.keys.length;
+        const dead = this.keys.filter(k => k.circuitState === 'DEAD').length;
+        const cooling = this.keys.filter(k => k.circuitState === 'OPEN' || k.circuitState === 'HALF_OPEN').length;
+        const healthy = total - dead - cooling;
+        return { total, healthy, cooling, dead };
+    }
+    // Helper for testing
+    _getKeys() {
+        return this.keys;
+    }
+    saveState() {
+        if (!this.storage)
+            return;
+        const state = this.keys.reduce((acc, k) => ({
+            ...acc,
+            [k.key]: {
+                failCount: k.failCount,
+                failedAt: k.failedAt,
+                isQuotaError: k.isQuotaError,
+                circuitState: k.circuitState,
+                lastUsed: k.lastUsed,
+                successCount: k.successCount,
+                totalRequests: k.totalRequests,
+                customCooldown: k.customCooldown,
+            }
+        }), {});
+        this.storage.setItem(this.storageKey, JSON.stringify(state));
+    }
+    loadState() {
+        if (!this.storage)
+            return;
+        try {
+            const raw = this.storage.getItem(this.storageKey);
+            if (!raw)
+                return;
+            const data = JSON.parse(raw);
+            this.keys.forEach(k => {
+                if (data[k.key])
+                    Object.assign(k, data[k.key]);
+            });
+        }
+        catch (e) {
+            console.error("Failed to load key state");
+        }
+    }
+}
+exports.ApiKeyManager = ApiKeyManager;
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;;AAgCH,MAAM,MAAM,GAAG;IACX,wBAAwB,EAAE,CAAC;IAC3B,kBAAkB,EAAE,EAAE,GAAG,IAAI,EAAS,WAAW;IACjD,cAAc,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAS,wCAAwC;IAC9E,oBAAoB,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,4BAA4B;IAClE,oBAAoB,EAAE,EAAE,GAAG,IAAI,EAAO,sBAAsB;IAC5D,WAAW,EAAE,EAAE,GAAG,IAAI,EAAgB,iBAAiB;IACvD,YAAY,EAAE,IAAI,EAAoB,gBAAgB;CACzD,CAAC;AAEF,gCAAgC;AAChC,MAAM,cAAc,GAAG;IACnB,YAAY,EAAE,0EAA0E;IACxF,WAAW,EAAE,wEAAwE;IACrF,aAAa,EAAE,oCAAoC;IACnD,WAAW,EAAE,mEAAmE;IAChF,YAAY,EAAE,sEAAsE;CACvF,CAAC;AASF,MAAa,aAAa;IACd,IAAI,GAAe,EAAE,CAAC;IACtB,UAAU,GAAG,uBAAuB,CAAC;IAC7C,uDAAuD;IAC/C,OAAO,CAAM;IAErB,YAAY,WAAqB,EAAE,OAAa;QAC5C,oEAAoE;QACpE,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI;YACtB,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI;YACnB,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC;SACrB,CAAC;QAEF,+CAA+C;QAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;QACrC,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YACzB,gEAAgE;YAChE,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC7E,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,iCAAiC;QACjC,MAAM,cAAc,GAAG,UAAU,CAAC,IAAI,CAAC;QACvC,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,qDAAqD;QAC3F,IAAI,cAAc,GAAG,SAAS,EAAE,CAAC;YAC7B,OAAO,CAAC,IAAI,CAAC,+CAA+C,SAAS,GAAG,cAAc,wBAAwB,CAAC,CAAC;QACpH,CAAC;QAED,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACzC,GAAG,EAAE,CAAC;YACN,SAAS,EAAE,CAAC;YACZ,QAAQ,EAAE,IAAI;YACd,YAAY,EAAE,KAAK;YACnB,YAAY,EAAE,QAAQ;YACtB,QAAQ,EAAE,CAAC;YACX,YAAY,EAAE,CAAC;YACf,aAAa,EAAE,CAAC;YAChB,gBAAgB,EAAE,IAAI;YACtB,cAAc,EAAE,IAAI;SACvB,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACI,aAAa,CAAC,KAAU,EAAE,YAAqB;QAClD,MAAM,MAAM,GAAG,KAAK,EAAE,MAAM,IAAI,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC;QACxD,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;QAEzE,sEAAsE;QACtE,IAAI,YAAY,KAAK,QAAQ,EAAE,CAAC;YAC5B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;QACzG,CAAC;QACD,IAAI,YAAY,KAAK,YAAY,EAAE,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;QAC7G,CAAC;QAED,6BAA6B;QAC7B,IAAI,MAAM,KAAK,GAAG,IAAI,cAAc,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7D,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAC5G,CAAC;QACD,IAAI,MAAM,KAAK,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9D,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAC/C,OAAO;gBACH,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,IAAI;gBACf,UAAU,EAAE,UAAU,IAAI,MAAM,CAAC,cAAc;gBAC/C,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,KAAK;aACrB,CAAC;QACN,CAAC;QACD,IAAI,MAAM,KAAK,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9D,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;QAC9G,CAAC;QACD,IAAI,cAAc,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACpF,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,kBAAkB,EAAE,aAAa,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;QAClI,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,kBAAkB,EAAE,aAAa,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IAChI,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,KAAU;QAC9B,MAAM,UAAU,GAAG,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,aAAa,CAAC;YACxD,KAAK,EAAE,OAAO,EAAE,CAAC,aAAa,CAAC;YAC/B,KAAK,EAAE,UAAU,CAAC;QAEtB,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC;QAE7B,6BAA6B;QAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,OAAO,OAAO,GAAG,IAAI,CAAC;QAE3C,wBAAwB;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAExD,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;OAGG;IACK,YAAY,CAAC,CAAW;QAC5B,6BAA6B;QAC7B,IAAI,CAAC,CAAC,YAAY,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;QAE3C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,IAAI,CAAC,CAAC,YAAY,KAAK,MAAM,EAAE,CAAC;YAC5B,oCAAoC;YACpC,IAAI,CAAC,CAAC,gBAAgB,IAAI,GAAG,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;gBAClD,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC;gBAC7B,OAAO,KAAK,CAAC;YACjB,CAAC;YACD,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,4CAA4C;QAC5C,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;YACjC,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,cAAc;gBAAE,OAAO,IAAI,CAAC;QACzD,CAAC;QAED,0BAA0B;QAC1B,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC;YACpF,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ,GAAG,QAAQ;gBAAE,OAAO,IAAI,CAAC;QACjD,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACI,MAAM;QACT,2CAA2C;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACpC,CAAC,CAAC,YAAY,KAAK,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CACrD,CAAC;QAEF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,sDAAsD;YACtD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,MAAM,CAAC,CAAC;YACjE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC,CAAC,qBAAqB;YAE5D,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC;QACzF,CAAC;QAED,uEAAuE;QACvE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACrB,IAAI,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS;gBAAE,OAAO,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;YAClE,OAAO,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC/B,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,OAAO,QAAQ,CAAC,GAAG,CAAC;IACxB,CAAC;IAED;;OAEG;IACI,WAAW;QACd,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACnE,CAAC;IAED;;OAEG;IACI,WAAW,CAAC,GAAW;QAC1B,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;QAC7C,IAAI,CAAC,CAAC;YAAE,OAAO;QAEf,IAAI,CAAC,CAAC,YAAY,KAAK,QAAQ,IAAI,CAAC,CAAC,YAAY,KAAK,MAAM,EAAE,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvD,CAAC;QAED,CAAC,CAAC,YAAY,GAAG,QAAQ,CAAC;QAC1B,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;QAChB,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC;QACvB,CAAC,CAAC,cAAc,GAAG,IAAI,CAAC;QACxB,CAAC,CAAC,YAAY,EAAE,CAAC;QACjB,CAAC,CAAC,aAAa,EAAE,CAAC;QAElB,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;;OAGG;IACI,UAAU,CAAC,GAAW,EAAE,cAAmC;QAC9D,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;QAC7C,IAAI,CAAC,CAAC;YAAE,OAAO;QAEf,yBAAyB;QACzB,IAAI,CAAC,CAAC,YAAY,KAAK,MAAM;YAAE,OAAO;QAEtC,uDAAuD;QACvD,IAAI,CAAC,cAAc,CAAC,aAAa;YAAE,OAAO;QAE1C,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxB,CAAC,CAAC,SAAS,EAAE,CAAC;QACd,CAAC,CAAC,aAAa,EAAE,CAAC;QAClB,CAAC,CAAC,YAAY,GAAG,cAAc,CAAC,IAAI,KAAK,OAAO,CAAC;QACjD,CAAC,CAAC,cAAc,GAAG,cAAc,CAAC,UAAU,IAAI,IAAI,CAAC;QAErD,kCAAkC;QAClC,IAAI,cAAc,CAAC,WAAW,EAAE,CAAC;YAC7B,CAAC,CAAC,YAAY,GAAG,MAAM,CAAC;YACxB,OAAO,CAAC,KAAK,CAAC,iBAAiB,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,sCAAsC,CAAC,CAAC;YACpF,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,OAAO;QACX,CAAC;QAED,oBAAoB;QACpB,IAAI,CAAC,CAAC,YAAY,KAAK,WAAW,EAAE,CAAC;YACjC,CAAC,CAAC,YAAY,GAAG,MAAM,CAAC;YACxB,CAAC,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,oBAAoB,CAAC;QAClE,CAAC;aAAM,IAAI,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,wBAAwB,IAAI,cAAc,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC3F,CAAC,CAAC,YAAY,GAAG,MAAM,CAAC;YACxB,CAAC,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,cAAc,CAAC,UAAU,IAAI,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACjG,CAAC;QAED,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACI,gBAAgB,CAAC,GAAW,EAAE,UAAmB,KAAK;QACzD,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE;YACjB,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW;YACrC,SAAS,EAAE,IAAI;YACf,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,kBAAkB;YACvE,aAAa,EAAE,IAAI;YACnB,WAAW,EAAE,KAAK;SACrB,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACI,gBAAgB,CAAC,OAAe;QACnC,MAAM,WAAW,GAAG,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;QACzD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC;QACpC,OAAO,MAAM,GAAG,MAAM,CAAC;IAC3B,CAAC;IAID,uBAAuB;IAEvB;;OAEG;IACI,QAAQ;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;QACrE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,MAAM,IAAI,CAAC,CAAC,YAAY,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;QAC1G,MAAM,OAAO,GAAG,KAAK,GAAG,IAAI,GAAG,OAAO,CAAC;QACvC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC7C,CAAC;IAED,qBAAqB;IACd,QAAQ;QACX,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAEO,SAAS;QACb,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACxC,GAAG,GAAG;YACN,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE;gBACL,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,YAAY,EAAE,CAAC,CAAC,YAAY;gBAC5B,YAAY,EAAE,CAAC,CAAC,YAAY;gBAC5B,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,YAAY,EAAE,CAAC,CAAC,YAAY;gBAC5B,aAAa,EAAE,CAAC,CAAC,aAAa;gBAC9B,cAAc,EAAE,CAAC,CAAC,cAAc;aACnC;SACJ,CAAC,EAAE,EAAE,CAAC,CAAC;QACR,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACjE,CAAC;IAEO,SAAS;QACb,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAC1B,IAAI,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAClD,IAAI,CAAC,GAAG;gBAAE,OAAO;YACjB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;gBAClB,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;oBAAE,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACnD,CAAC,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;IACL,CAAC;CACJ;AAvTD,sCAuTC"}

package/package.json

@@ -0,0 +1,31 @@
+{
+    "name": "@splashcodex/api-key-manager",
+    "version": "1.0.0",
+    "description": "Universal API Key Rotation System for rate-limited APIs",
+    "main": "dist/index.js",
+    "types": "dist/index.d.ts",
+    "files": [
+        "dist",
+        "src"
+    ],
+    "scripts": {
+        "build": "tsc",
+        "test": "ts-node tests/index.test.ts",
+        "prepublishOnly": "npm run build"
+    },
+    "keywords": [
+        "api",
+        "key",
+        "rotation",
+        "gemini",
+        "rate-limit",
+        "circuit-breaker"
+    ],
+    "author": "Antigravity",
+    "license": "ISC",
+    "devDependencies": {
+        "typescript": "^5.3.3",
+        "ts-node": "^10.9.2",
+        "@types/node": "^20.10.5"
+    }
+}

package/src/index.ts

@@ -0,0 +1,374 @@
+/**
+ * Universal ApiKeyManager v2.0
+ * Implements: Rotation, Circuit Breaker, Persistence, Exponential Backoff
+ * Gemini-Specific: finishReason handling, Safety blocks, RECITATION detection
+ */
+
+export interface KeyState {
+    key: string;
+    failCount: number;           // Consecutive failures
+    failedAt: number | null;     // Timestamp of last failure
+    isQuotaError: boolean;       // Was last error a 429?
+    circuitState: 'CLOSED' | 'OPEN' | 'HALF_OPEN' | 'DEAD';
+    lastUsed: number;
+    successCount: number;
+    totalRequests: number;
+    halfOpenTestTime: number | null;
+    customCooldown: number | null; // From Retry-After header
+}
+
+export type ErrorType =
+    | 'QUOTA'       // 429 - Rotate key, respect cooldown
+    | 'TRANSIENT'   // 500/503/504 - Retry with backoff
+    | 'AUTH'        // 403 - Key is dead, remove from pool
+    | 'BAD_REQUEST' // 400 - Do not retry, fix request
+    | 'SAFETY'      // finishReason: SAFETY - Not a key issue
+    | 'RECITATION'  // finishReason: RECITATION - Not a key issue
+    | 'UNKNOWN';    // Catch-all
+
+export interface ErrorClassification {
+    type: ErrorType;
+    retryable: boolean;
+    cooldownMs: number;
+    markKeyFailed: boolean;
+    markKeyDead: boolean;
+}
+
+const CONFIG = {
+    MAX_CONSECUTIVE_FAILURES: 5,
+    COOLDOWN_TRANSIENT: 60 * 1000,        // 1 minute
+    COOLDOWN_QUOTA: 5 * 60 * 1000,        // 5 minutes (default if no Retry-After)
+    COOLDOWN_QUOTA_DAILY: 60 * 60 * 1000, // 1 hour for RPD exhaustion
+    HALF_OPEN_TEST_DELAY: 60 * 1000,      // 1 minute after open
+    MAX_BACKOFF: 64 * 1000,               // 64 seconds max
+    BASE_BACKOFF: 1000,                   // 1 second base
+};
+
+// Error classification patterns
+const ERROR_PATTERNS = {
+    isQuotaError: /429|quota|exhausted|resource.?exhausted|too.?many.?requests|rate.?limit/i,
+    isAuthError: /403|permission.?denied|invalid.?api.?key|unauthorized|unauthenticated/i,
+    isSafetyBlock: /safety|blocked|recitation|harmful/i,
+    isTransient: /500|502|503|504|internal|unavailable|deadline|timeout|overloaded/i,
+    isBadRequest: /400|invalid.?argument|failed.?precondition|malformed|not.?found|404/i,
+};
+
+export interface ApiKeyManagerStats {
+    total: number;
+    healthy: number;
+    cooling: number;
+    dead: number;
+}
+
+export class ApiKeyManager {
+    private keys: KeyState[] = [];
+    private storageKey = 'api_rotation_state_v2';
+    // Simplified Storage interface for Node.js environment
+    private storage: any;
+
+    constructor(initialKeys: string[], storage?: any) {
+        // Simple in-memory storage mock if none provided (for testing/Node)
+        this.storage = storage || {
+            getItem: () => null,
+            setItem: () => { },
+        };
+
+        // 1. Sanitize & Deduplicate Keys "Even Better"
+        const uniqueKeys = new Set<string>();
+        initialKeys.forEach(rawKey => {
+            // Handle comma-separated keys if they appear in a single string
+            const parts = rawKey.split(',').map(s => s.trim()).filter(s => s.length > 0);
+            parts.forEach(p => uniqueKeys.add(p));
+        });
+
+        // 2. Warn about duplicates/empty
+        const sanitizedCount = uniqueKeys.size;
+        const rawLength = initialKeys.length; // Might be misleading if split happened, but roughly
+        if (sanitizedCount < rawLength) {
+            console.warn(`[ApiKeyManager] Optimized key pool: Removed ${rawLength - sanitizedCount} duplicate/empty keys.`);
+        }
+
+        this.keys = Array.from(uniqueKeys).map(k => ({
+            key: k,
+            failCount: 0,
+            failedAt: null,
+            isQuotaError: false,
+            circuitState: 'CLOSED',
+            lastUsed: 0,
+            successCount: 0,
+            totalRequests: 0,
+            halfOpenTestTime: null,
+            customCooldown: null,
+        }));
+
+        this.loadState();
+    }
+
+    /**
+     * CLASSIFIES an error to determine handling strategy
+     */
+    public classifyError(error: any, finishReason?: string): ErrorClassification {
+        const status = error?.status || error?.response?.status;
+        const message = error?.message || error?.error?.message || String(error);
+
+        // 1. Check finishReason first (for 200 responses with content issues)
+        if (finishReason === 'SAFETY') {
+            return { type: 'SAFETY', retryable: false, cooldownMs: 0, markKeyFailed: false, markKeyDead: false };
+        }
+        if (finishReason === 'RECITATION') {
+            return { type: 'RECITATION', retryable: false, cooldownMs: 0, markKeyFailed: false, markKeyDead: false };
+        }
+
+        // 2. Check HTTP status codes
+        if (status === 403 || ERROR_PATTERNS.isAuthError.test(message)) {
+            return { type: 'AUTH', retryable: false, cooldownMs: Infinity, markKeyFailed: true, markKeyDead: true };
+        }
+        if (status === 429 || ERROR_PATTERNS.isQuotaError.test(message)) {
+            const retryAfter = this.parseRetryAfter(error);
+            return {
+                type: 'QUOTA',
+                retryable: true,
+                cooldownMs: retryAfter || CONFIG.COOLDOWN_QUOTA,
+                markKeyFailed: true,
+                markKeyDead: false
+            };
+        }
+        if (status === 400 || ERROR_PATTERNS.isBadRequest.test(message)) {
+            return { type: 'BAD_REQUEST', retryable: false, cooldownMs: 0, markKeyFailed: false, markKeyDead: false };
+        }
+        if (ERROR_PATTERNS.isTransient.test(message) || [500, 502, 503, 504].includes(status)) {
+            return { type: 'TRANSIENT', retryable: true, cooldownMs: CONFIG.COOLDOWN_TRANSIENT, markKeyFailed: true, markKeyDead: false };
+        }
+
+        return { type: 'UNKNOWN', retryable: true, cooldownMs: CONFIG.COOLDOWN_TRANSIENT, markKeyFailed: true, markKeyDead: false };
+    }
+
+    /**
+     * Parses Retry-After header from error response
+     */
+    private parseRetryAfter(error: any): number | null {
+        const retryAfter = error?.response?.headers?.['retry-after'] ||
+            error?.headers?.['retry-after'] ||
+            error?.retryAfter;
+
+        if (!retryAfter) return null;
+
+        // If it's a number (seconds)
+        const seconds = parseInt(retryAfter, 10);
+        if (!isNaN(seconds)) return seconds * 1000;
+
+        // If it's a date string
+        const date = Date.parse(retryAfter);
+        if (!isNaN(date)) return Math.max(0, date - Date.now());
+
+        return null;
+    }
+
+    /**
+     * HEALTH CHECK
+     * Determines if a key is usable based on Circuit Breaker logic
+     */
+    private isOnCooldown(k: KeyState): boolean {
+        // Dead keys are NEVER usable
+        if (k.circuitState === 'DEAD') return true;
+
+        const now = Date.now();
+
+        if (k.circuitState === 'OPEN') {
+            // Check if ready for HALF_OPEN test
+            if (k.halfOpenTestTime && now >= k.halfOpenTestTime) {
+                k.circuitState = 'HALF_OPEN';
+                return false;
+            }
+            return true;
+        }
+
+        // Additional safeguard for custom cooldowns
+        if (k.failedAt && k.customCooldown) {
+            if (now - k.failedAt < k.customCooldown) return true;
+        }
+
+        // Standard cooldown check
+        if (k.failedAt) {
+            const cooldown = k.isQuotaError ? CONFIG.COOLDOWN_QUOTA : CONFIG.COOLDOWN_TRANSIENT;
+            if (now - k.failedAt < cooldown) return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * CORE ROTATION LOGIC
+     * Returns the best available key
+     */
+    public getKey(): string | null {
+        // 1. Filter out dead and cooling down keys
+        const candidates = this.keys.filter(k =>
+            k.circuitState !== 'DEAD' && !this.isOnCooldown(k)
+        );
+
+        if (candidates.length === 0) {
+            // FALLBACK: Return oldest failed key (excluding DEAD)
+            const nonDead = this.keys.filter(k => k.circuitState !== 'DEAD');
+            if (nonDead.length === 0) return null; // All keys are dead!
+
+            return nonDead.sort((a, b) => (a.failedAt || 0) - (b.failedAt || 0))[0]?.key || null;
+        }
+
+        // 2. Sort candidates: Pristine > Fewest Failures > Least Recently Used
+        candidates.sort((a, b) => {
+            if (a.failCount !== b.failCount) return a.failCount - b.failCount;
+            return a.lastUsed - b.lastUsed;
+        });
+
+        const selected = candidates[0];
+        selected.lastUsed = Date.now();
+        this.saveState();
+
+        return selected.key;
+    }
+
+    /**
+     * Get count of healthy (non-DEAD) keys
+     */
+    public getKeyCount(): number {
+        return this.keys.filter(k => k.circuitState !== 'DEAD').length;
+    }
+
+    /**
+     * FEEDBACK LOOP: Success
+     */
+    public markSuccess(key: string) {
+        const k = this.keys.find(x => x.key === key);
+        if (!k) return;
+
+        if (k.circuitState !== 'CLOSED' && k.circuitState !== 'DEAD') {
+            console.log(`[Key Recovered] ...${key.slice(-4)}`);
+        }
+
+        k.circuitState = 'CLOSED';
+        k.failCount = 0;
+        k.failedAt = null;
+        k.isQuotaError = false;
+        k.customCooldown = null;
+        k.successCount++;
+        k.totalRequests++;
+
+        this.saveState();
+    }
+
+    /**
+     * FEEDBACK LOOP: Failure
+     * Enhanced with error classification
+     */
+    public markFailed(key: string, classification: ErrorClassification) {
+        const k = this.keys.find(x => x.key === key);
+        if (!k) return;
+
+        // Don't modify DEAD keys
+        if (k.circuitState === 'DEAD') return;
+
+        // If this error shouldn't mark the key as failed, skip
+        if (!classification.markKeyFailed) return;
+
+        k.failedAt = Date.now();
+        k.failCount++;
+        k.totalRequests++;
+        k.isQuotaError = classification.type === 'QUOTA';
+        k.customCooldown = classification.cooldownMs || null;
+
+        // Permanent death for auth errors
+        if (classification.markKeyDead) {
+            k.circuitState = 'DEAD';
+            console.error(`[Key DEAD] ...${key.slice(-4)} - Permanently removed from rotation`);
+            this.saveState();
+            return;
+        }
+
+        // State Transitions
+        if (k.circuitState === 'HALF_OPEN') {
+            k.circuitState = 'OPEN';
+            k.halfOpenTestTime = Date.now() + CONFIG.HALF_OPEN_TEST_DELAY;
+        } else if (k.failCount >= CONFIG.MAX_CONSECUTIVE_FAILURES || classification.type === 'QUOTA') {
+            k.circuitState = 'OPEN';
+            k.halfOpenTestTime = Date.now() + (classification.cooldownMs || CONFIG.HALF_OPEN_TEST_DELAY);
+        }
+
+        this.saveState();
+    }
+
+    /**
+     * Legacy markFailed for backward compatibility
+     */
+    public markFailedLegacy(key: string, isQuota: boolean = false) {
+        this.markFailed(key, {
+            type: isQuota ? 'QUOTA' : 'TRANSIENT',
+            retryable: true,
+            cooldownMs: isQuota ? CONFIG.COOLDOWN_QUOTA : CONFIG.COOLDOWN_TRANSIENT,
+            markKeyFailed: true,
+            markKeyDead: false,
+        });
+    }
+
+    /**
+     * Calculate backoff delay with jitter
+     */
+    public calculateBackoff(attempt: number): number {
+        const exponential = CONFIG.BASE_BACKOFF * Math.pow(2, attempt);
+        const capped = Math.min(exponential, CONFIG.MAX_BACKOFF);
+        const jitter = Math.random() * 1000;
+        return capped + jitter;
+    }
+
+
+
+    // ... inside class ...
+
+    /**
+     * Get health statistics
+     */
+    public getStats(): ApiKeyManagerStats {
+        const total = this.keys.length;
+        const dead = this.keys.filter(k => k.circuitState === 'DEAD').length;
+        const cooling = this.keys.filter(k => k.circuitState === 'OPEN' || k.circuitState === 'HALF_OPEN').length;
+        const healthy = total - dead - cooling;
+        return { total, healthy, cooling, dead };
+    }
+
+    // Helper for testing
+    public _getKeys(): KeyState[] {
+        return this.keys;
+    }
+
+    private saveState() {
+        if (!this.storage) return;
+        const state = this.keys.reduce((acc, k) => ({
+            ...acc,
+            [k.key]: {
+                failCount: k.failCount,
+                failedAt: k.failedAt,
+                isQuotaError: k.isQuotaError,
+                circuitState: k.circuitState,
+                lastUsed: k.lastUsed,
+                successCount: k.successCount,
+                totalRequests: k.totalRequests,
+                customCooldown: k.customCooldown,
+            }
+        }), {});
+        this.storage.setItem(this.storageKey, JSON.stringify(state));
+    }
+
+    private loadState() {
+        if (!this.storage) return;
+        try {
+            const raw = this.storage.getItem(this.storageKey);
+            if (!raw) return;
+            const data = JSON.parse(raw);
+            this.keys.forEach(k => {
+                if (data[k.key]) Object.assign(k, data[k.key]);
+            });
+        } catch (e) {
+            console.error("Failed to load key state");
+        }
+    }
+}