@splashcodex/api-key-manager 5.0.2 → 5.3.0

package/src/index.ts

@@ -6,9 +6,19 @@
  * NEW in v5.0: Provider Presets (GeminiManager, OpenAIManager, MultiManager),
  *              Built-in Persistence (FileStorage, MemoryStorage)
  * Gemini-Specific: finishReason handling, Safety blocks, RECITATION detection
+ * Infrastructure: cockatiel (Bulkhead queueing + ExponentialBackoff w/ decorrelated jitter)
  */
 
 import { EventEmitter } from 'events';
+import {
+    bulkhead as cockatielBulkhead,
+    BulkheadRejectedError as CockatielBulkheadRejectedError,
+    ExponentialBackoff,
+    decorrelatedJitterGenerator,
+    type BulkheadPolicy,
+    type IBackoff,
+} from 'cockatiel';
+import { z } from 'zod';
 
 // ─── Re-exports: Persistence ─────────────────────────────────────────────────
 // Persistence adapters can be imported from root or via subpath
@@ -80,17 +90,30 @@ export interface ExecuteOptions {
     provider?: string;       // Filter keys by provider (e.g. 'openai')
 }
 
-export interface ApiKeyManagerOptions {
-    storage?: any;
-    strategy?: LoadBalancingStrategy;
-    fallbackFn?: () => any;
-    concurrency?: number;    // Max concurrent execute() calls
-    semanticCache?: {
-        threshold?: number;  // Similarity threshold (0.0 - 1.0, default 0.95)
-        ttlMs?: number;      // Cache TTL
-        getEmbedding: (text: string) => Promise<number[]>;
-    };
-}
+export const ApiKeyManagerOptionsSchema = z.object({
+    storage: z.any().optional(),
+    strategy: z.any().optional(), // Expected: LoadBalancingStrategy
+    fallbackFn: z.custom<() => any>((val) => typeof val === 'function', 'Must be a function').optional(),
+    /** Max concurrent execute() calls. When limit is reached, excess requests queue (up to concurrencyQueueSize) then reject. */
+    concurrency: z.number().int().positive().optional(),
+    /**
+     * Maximum number of requests to hold in the bulkhead queue when all concurrency slots are busy.
+     * - Default: `0` — excess requests are rejected immediately (preserves v3 behavior).
+     * - Set to a positive number to allow requests to wait for a free slot.
+     *
+     * @example
+     * // Queue up to 10 waiting requests before rejecting
+     * new ApiKeyManager(keys, { concurrency: 5, concurrencyQueueSize: 10 })
+     */
+    concurrencyQueueSize: z.number().int().nonnegative().optional(),
+    semanticCache: z.object({
+        threshold: z.number().min(0).max(1).optional(),  // Similarity threshold (0.0 - 1.0, default 0.95)
+        ttlMs: z.number().int().positive().optional(),      // Cache TTL
+        getEmbedding: z.custom<(text: string) => Promise<number[]>>((val) => typeof val === 'function', 'Must be a function'),
+    }).optional(),
+}).passthrough();
+
+export type ApiKeyManagerOptions = z.infer<typeof ApiKeyManagerOptionsSchema>;
 
 export interface CacheEntry {
     vector: number[];
@@ -126,6 +149,7 @@ const CONFIG = {
     HALF_OPEN_TEST_DELAY: 60 * 1000,      // 1 minute after open
     MAX_BACKOFF: 64 * 1000,               // 64 seconds max
     BASE_BACKOFF: 1000,                   // 1 second base
+    DEAD_KEY_TTL: 60 * 60 * 1000,        // 1 hour — DEAD keys get retested after this
 };
 
 // Error classification patterns
@@ -298,14 +322,18 @@ export class ApiKeyManager extends EventEmitter {
     private strategy: LoadBalancingStrategy;
     private fallbackFn?: () => any;
 
-    // Bulkhead state
-    private maxConcurrency: number;
-    private activeCalls: number = 0;
+    // Bulkhead state — managed by cockatiel BulkheadPolicy
+    // Provides FIFO queueing (requests wait for a slot) instead of immediate rejection
+    private bulkheadPolicy: BulkheadPolicy | null = null;
 
     // Health check state
     private healthCheckFn?: (key: string) => Promise<boolean>;
     private healthCheckInterval?: ReturnType<typeof setInterval>;
 
+    // Debounced persistence — avoids writeFileSync on every single call
+    private _saveTimer?: ReturnType<typeof setTimeout>;
+    private _saveDirty: boolean = false;
+
     // Semantic Cache v4
     private semanticCache?: SemanticCache;
     private getEmbeddingFn?: (text: string) => Promise<number[]>;
@@ -329,7 +357,7 @@ export class ApiKeyManager extends EventEmitter {
 
         // Detect if second arg is options object or legacy storage
         let options: ApiKeyManagerOptions = {};
-        if (storageOrOptions && typeof storageOrOptions === 'object' && ('storage' in storageOrOptions || 'strategy' in storageOrOptions || 'fallbackFn' in storageOrOptions || 'concurrency' in storageOrOptions || 'semanticCache' in storageOrOptions)) {
+        if (storageOrOptions && typeof storageOrOptions === 'object' && ('storage' in storageOrOptions || 'strategy' in storageOrOptions || 'fallbackFn' in storageOrOptions || 'concurrency' in storageOrOptions || 'concurrencyQueueSize' in storageOrOptions || 'semanticCache' in storageOrOptions)) {
             // New v3 options object
             options = storageOrOptions as ApiKeyManagerOptions;
         } else {
@@ -340,13 +368,27 @@ export class ApiKeyManager extends EventEmitter {
             };
         }
 
+        // Validate options with Zod (will throw meaningful errors on invalid configs)
+        options = ApiKeyManagerOptionsSchema.parse(options);
+
         this.storage = options.storage || {
             getItem: () => null,
             setItem: () => { },
         };
         this.strategy = options.strategy || new StandardStrategy();
         this.fallbackFn = options.fallbackFn;
-        this.maxConcurrency = options.concurrency || Infinity;
+
+        // Build cockatiel bulkhead.
+        // queueSize defaults to 0 (immediate rejection — preserves existing API contract).
+        // Set concurrencyQueueSize > 0 to opt-in to queuing instead of rejection.
+        const maxConcurrency = options.concurrency ?? Infinity;
+        const queueSize = options.concurrencyQueueSize ?? 0;
+        if (maxConcurrency !== Infinity) {
+            this.bulkheadPolicy = cockatielBulkhead(maxConcurrency, queueSize);
+            this.bulkheadPolicy.onReject(() => {
+                this.emit('bulkheadRejected');
+            });
+        }
 
         // Init Semantic Cache if provided
         if (options.semanticCache) {
@@ -603,11 +645,31 @@ export class ApiKeyManager extends EventEmitter {
 
     // ─── Backoff ─────────────────────────────────────────────────────────────
 
+    /**
+     * Calculate exponential backoff with decorrelated jitter using cockatiel.
+     * Decorrelated jitter avoids thundering-herd by randomizing retry intervals
+     * independent of the previous delay, which is statistically superior to
+     * simple `random * exponential` jitter.
+     *
+     * @param attempt - Zero-indexed attempt number
+     */
+    private readonly _backoffFactory = new ExponentialBackoff({
+        initialDelay: CONFIG.BASE_BACKOFF,
+        maxDelay: CONFIG.MAX_BACKOFF,
+        generator: decorrelatedJitterGenerator,
+    });
+
     public calculateBackoff(attempt: number): number {
-        const exponential = CONFIG.BASE_BACKOFF * Math.pow(2, attempt);
-        const capped = Math.min(exponential, CONFIG.MAX_BACKOFF);
-        const jitter = Math.random() * 1000;
-        return capped + jitter;
+        // ExponentialBackoff is a linked-list: _backoffFactory.next() returns an
+        // IBackoff node with { duration, next() }. Walk `attempt` steps to get
+        // the correctly scaled delay. Uses decorrelated jitter (superior to random*exp).
+        // We cast via `any` to bypass the interface vs concrete class arg-count conflict.
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        let node: any = (this._backoffFactory as any).next();
+        for (let i = 0; i < attempt; i++) {
+            node = node.next();
+        }
+        return (node as IBackoff<unknown>).duration;
     }
 
     // ─── Stats ───────────────────────────────────────────────────────────────
@@ -664,25 +726,24 @@ export class ApiKeyManager extends EventEmitter {
             }
         }
 
-        // 2. Bulkhead check
-        if (this.activeCalls >= this.maxConcurrency) {
-            this.emit('bulkheadRejected');
-            throw new BulkheadRejectionError();
-        }
-
-        this.activeCalls++;
-        try {
-            const result = await this._executeWithRetry(fn, maxRetries, timeoutMs, finishReason, provider);
-
-            // 3. Store in Semantic Cache on success
-            if (this.semanticCache && prompt && currentPromptVector) {
-                this.semanticCache.set(prompt, currentPromptVector, result);
+        // 2. Bulkhead check — if policy exists, cockatiel queues excess requests
+        // instead of immediately rejecting them (queue drains as slots free up)
+        if (this.bulkheadPolicy) {
+            try {
+                const result = await this.bulkheadPolicy.execute(async () => {
+                    return await this._executeWithSemanticAndRetry<T>(fn, maxRetries, timeoutMs, finishReason, provider, prompt, currentPromptVector);
+                });
+                return result;
+            } catch (err) {
+                if (err instanceof CockatielBulkheadRejectedError) {
+                    throw new BulkheadRejectionError();
+                }
+                throw err;
             }
-
-            return result;
-        } finally {
-            this.activeCalls--;
         }
+
+        // No concurrency limit configured — run directly
+        return this._executeWithSemanticAndRetry<T>(fn, maxRetries, timeoutMs, finishReason, provider, prompt, currentPromptVector);
     }
 
     /**
@@ -732,13 +793,18 @@ export class ApiKeyManager extends EventEmitter {
             }
         }
 
-        // 2. Bulkhead check
-        if (this.activeCalls >= this.maxConcurrency) {
-            this.emit('bulkheadRejected');
-            throw new BulkheadRejectionError();
+        // 2. Bulkhead check — gate streaming with bulkhead slot management.
+        // Async generators can't be wrapped in bulkheadPolicy.execute() (which expects
+        // a Promise), so we check availability and track manually.
+        const useBulkhead = this.bulkheadPolicy !== null;
+        if (useBulkhead) {
+            const slots = (this.bulkheadPolicy as any);
+            const hasSlot = (slots.executionSlots > 0) || (slots.queueSlots > 0);
+            if (!hasSlot) {
+                this.emit('bulkheadRejected');
+                throw new BulkheadRejectionError();
+            }
         }
-
-        this.activeCalls++;
         const accumulatedChunks: T[] = [];
         let lastError: any;
 
@@ -813,12 +879,35 @@ export class ApiKeyManager extends EventEmitter {
                 }
             }
         } finally {
-            this.activeCalls--;
+            // Slot is freed automatically when the generator completes/throws
         }
 
         throw lastError;
     }
 
+    /**
+     * Helper: stores result in semantic cache then returns it.
+     * Called by the bulkhead execute() callback and the no-bulkhead path.
+     */
+    private async _executeWithSemanticAndRetry<T>(
+        fn: (key: string, signal?: AbortSignal) => Promise<T>,
+        maxRetries: number,
+        timeoutMs?: number,
+        finishReason?: string,
+        provider?: string,
+        prompt?: string,
+        currentPromptVector?: number[] | null
+    ): Promise<T> {
+        const result = await this._executeWithRetry(fn, maxRetries, timeoutMs, finishReason, provider);
+
+        // Store in Semantic Cache on success
+        if (this.semanticCache && prompt && currentPromptVector) {
+            this.semanticCache.set(prompt, currentPromptVector, result);
+        }
+
+        return result;
+    }
+
     private async _executeWithRetry<T>(
         fn: (key: string, signal?: AbortSignal) => Promise<T>,
         maxRetries: number,
@@ -939,6 +1028,8 @@ export class ApiKeyManager extends EventEmitter {
             clearInterval(this.healthCheckInterval);
             this.healthCheckInterval = undefined;
         }
+        // Flush any pending state to disk on shutdown
+        this._flushState();
     }
 
     private async _runHealthChecks() {
@@ -966,8 +1057,41 @@ export class ApiKeyManager extends EventEmitter {
 
     // ─── Persistence ─────────────────────────────────────────────────────────
 
+    /**
+     * Debounced save — marks state as dirty and flushes after 500ms of inactivity.
+     * Under heavy load (multiple getKey/markSuccess/markFailed calls), this coalesces
+     * dozens of writeFileSync calls into one.
+     */
     private saveState() {
         if (!this.storage) return;
+        this._saveDirty = true;
+
+        if (!this._saveTimer) {
+            this._saveTimer = setTimeout(() => {
+                this._flushState();
+                this._saveTimer = undefined;
+            }, 500);
+            if (this._saveTimer.unref) this._saveTimer.unref();
+        }
+    }
+
+    /**
+     * Immediately flush state to storage. Called by the debounce timer
+     * and by stopHealthChecks() to ensure clean shutdown.
+     */
+    public flushState() {
+        this._flushState();
+    }
+
+    private _flushState() {
+        if (!this._saveDirty || !this.storage) return;
+        this._saveDirty = false;
+
+        if (this._saveTimer) {
+            clearTimeout(this._saveTimer);
+            this._saveTimer = undefined;
+        }
+
         const state = this.keys.reduce((acc, k) => ({
             ...acc,
             [k.key]: {
@@ -995,8 +1119,34 @@ export class ApiKeyManager extends EventEmitter {
             const raw = this.storage.getItem(this.storageKey);
             if (!raw) return;
             const data = JSON.parse(raw);
+            const now = Date.now();
+
             this.keys.forEach(k => {
-                if (data[k.key]) Object.assign(k, data[k.key]);
+                if (!data[k.key]) return;
+                Object.assign(k, data[k.key]);
+
+                // Resurrect DEAD keys that have exceeded the TTL.
+                // This allows keys that were marked dead (e.g., temporary 403)
+                // to be retested after a cooldown period instead of staying dead forever.
+                if (k.circuitState === 'DEAD' && k.failedAt) {
+                    const deadDuration = now - k.failedAt;
+                    if (deadDuration >= CONFIG.DEAD_KEY_TTL) {
+                        k.circuitState = 'HALF_OPEN';
+                        k.halfOpenTestTime = null; // Allow immediate test
+                        k.failCount = 0;
+                    }
+                }
+
+                // Clear stale cooldowns from previous sessions.
+                // If a key was cooling down and the process restarted after the
+                // cooldown would have expired, reset it to CLOSED.
+                if (k.circuitState === 'OPEN' && k.failedAt) {
+                    const cooldown = k.customCooldown || (k.isQuotaError ? CONFIG.COOLDOWN_QUOTA : CONFIG.COOLDOWN_TRANSIENT);
+                    if (now - k.failedAt >= cooldown) {
+                        k.circuitState = 'HALF_OPEN';
+                        k.halfOpenTestTime = null;
+                    }
+                }
             });
         } catch (e) {
             console.error("Failed to load key state");

package/src/persistence/file.ts

@@ -10,7 +10,7 @@
  * @module persistence/file
  */
 
-import { writeFileSync, readFileSync, existsSync, mkdirSync, unlinkSync } from 'fs';
+import { writeFileSync, readFileSync, existsSync, mkdirSync, unlinkSync, renameSync } from 'fs';
 import { dirname, join } from 'path';
 import { tmpdir } from 'os';
 
@@ -57,13 +57,20 @@ export class FileStorage {
     }
 
     setItem(_key: string, value: string): void {
+        const tmpPath = this.filePath + '.tmp';
         try {
             const dir = dirname(this.filePath);
             if (!existsSync(dir)) {
                 mkdirSync(dir, { recursive: true });
             }
-            writeFileSync(this.filePath, value, 'utf-8');
+            // Write to a temp file first, then atomically rename to the target.
+            // renameSync() is a single syscall — the file is never in a
+            // half-written state if the process crashes mid-write.
+            writeFileSync(tmpPath, value, 'utf-8');
+            renameSync(tmpPath, this.filePath);
         } catch {
+            // Clean up temp file if rename failed
+            try { if (existsSync(tmpPath)) unlinkSync(tmpPath); } catch { /* silent */ }
             // Silently fail — state will be lost on restart
         }
     }

package/src/presets/anthropic.ts

@@ -0,0 +1,78 @@
+/**
+ * Anthropic Preset — One-line Claude API key management
+ *
+ * Pre-configured singleton with:
+ * - Reads `ANTHROPIC_API_KEY` from env
+ * - LatencyStrategy for optimal key selection
+ * - File persistence (survives restarts)
+ * - Health checks every 5 minutes
+ * - Concurrency limit of 20
+ *
+ * @module presets/anthropic
+ *
+ * @example
+ * ```ts
+ * import { AnthropicManager } from '@splashcodex/api-key-manager/presets/anthropic';
+ *
+ * const result = AnthropicManager.getInstance();
+ * if (!result.success) throw result.error;
+ * const claude = result.data;
+ *
+ * const response = await claude.execute(async (key) => {
+ *   const res = await fetch('https://api.anthropic.com/v1/messages', {
+ *     method: 'POST',
+ *     headers: {
+ *       'x-api-key': key,
+ *       'anthropic-version': '2023-06-01',
+ *       'content-type': 'application/json',
+ *     },
+ *     body: JSON.stringify({
+ *       model: 'claude-sonnet-4-20250514',
+ *       max_tokens: 1024,
+ *       messages: [{ role: 'user', content: 'Hello!' }],
+ *     }),
+ *   });
+ *   const data = await res.json();
+ *   return data.content[0].text;
+ * }, { maxRetries: 3, timeoutMs: 60000 });
+ * ```
+ */
+
+import { BasePreset, PresetOptions, Result } from './base';
+
+export class AnthropicManager extends BasePreset {
+    private static readonly PROVIDER = 'anthropic';
+
+    private constructor(keys: string[], options: PresetOptions) {
+        super(keys, options);
+    }
+
+    private static getDefaultOptions(): Partial<PresetOptions> {
+        return {
+            envKeys: ['ANTHROPIC_API_KEY'],
+            provider: AnthropicManager.PROVIDER,
+            concurrency: 20,
+            healthCheckIntervalMs: 300_000,
+        };
+    }
+
+    /**
+     * Get or create the singleton AnthropicManager instance.
+     *
+     * @param overrides - Optional configuration overrides
+     * @returns Result<AnthropicManager>
+     */
+    static getInstance(overrides?: Partial<PresetOptions>): Result<AnthropicManager> {
+        return BasePreset.createInstance(
+            AnthropicManager as unknown as new (keys: string[], options: PresetOptions) => AnthropicManager,
+            AnthropicManager.getDefaultOptions(),
+            overrides
+        ) as Result<AnthropicManager>;
+    }
+
+    static reset(): void {
+        BasePreset.resetInstance(AnthropicManager.PROVIDER);
+    }
+}
+
+export { Result } from './base';

package/src/presets/base.ts

@@ -20,8 +20,10 @@ import {
     LoadBalancingStrategy,
 } from '../index';
 import { FileStorage } from '../persistence/file';
-import { join } from 'path';
-import { tmpdir } from 'os';
+import { join, basename } from 'path';
+import { tmpdir, homedir } from 'os';
+import { readFileSync, existsSync } from 'fs';
+import { createHash } from 'crypto';
 
 // ─── Result Type ─────────────────────────────────────────────────────────────
 
@@ -64,6 +66,24 @@ export interface PresetLogger {
     error(message: string, ...args: any[]): void;
 }
 
+// ─── Project Identifier ─────────────────────────────────────────────────────
+
+/**
+ * Derive a short, stable identifier for the current project.
+ * Uses the directory name + a hash of the full CWD to avoid collisions
+ * when multiple projects run simultaneously.
+ *
+ * Examples:
+ *   /home/codedex/projects/WhatsDeX  → "whatsdex_a3f2"
+ *   /home/codedex/projects/DeXdo     → "dexdo_b7c1"
+ */
+function getProjectId(): string {
+    const cwd = process.cwd();
+    const dirName = basename(cwd).toLowerCase().replace(/[^a-z0-9]/g, '');
+    const hash = createHash('md5').update(cwd).digest('hex').slice(0, 4);
+    return `${dirName}_${hash}`;
+}
+
 // ─── Base Preset Class ──────────────────────────────────────────────────────
 
 /**
@@ -101,12 +121,13 @@ export abstract class BasePreset {
             ...options,
         };
 
+        const projectId = getProjectId();
         const stateFile = this.options.stateFilePath ||
-            join(tmpdir(), `codedex_${this.options.provider}_state.json`);
+            join(tmpdir(), `codedex_${this.options.provider}_${projectId}_state.json`);
 
         const storage = new FileStorage({
             filePath: stateFile,
-            clearOnInit: true, // Fresh start each session to clear stale DEAD keys
+            clearOnInit: false, // Preserve circuit breaker state across restarts
         });
 
         this.manager = new ApiKeyManager(apiKeys, {
@@ -253,6 +274,40 @@ export abstract class BasePreset {
         return [...new Set(keys)]; // Deduplicate
     }
 
+    protected static getConfigPath(): string {
+        return join(homedir(), '.codedex', 'api_keys.json');
+    }
+
+    /**
+     * Parse API keys from ~/.codedex/api_keys.json
+     * Format should be: { "ENV_VAR_NAME": "key1,key2" } or { "ENV_VAR_NAME": ["key1", "key2"] }
+     */
+    protected static parseKeysFromHomeDir(envKeys: string[]): string[] {
+        const configPath = this.getConfigPath();
+        
+        if (!existsSync(configPath)) return [];
+
+        try {
+            const fileContent = readFileSync(configPath, 'utf-8');
+            const parsed = JSON.parse(fileContent);
+            const keys: string[] = [];
+
+            for (const envName of envKeys) {
+                const value = parsed[envName];
+                if (!value) continue;
+                
+                if (Array.isArray(value)) {
+                    keys.push(...value.filter((k: any) => typeof k === 'string' && k.trim()));
+                } else if (typeof value === 'string' && value.trim()) {
+                    keys.push(...value.split(',').map(k => k.trim()).filter(k => k.length > 0));
+                }
+            }
+            return [...new Set(keys)];
+        } catch (err) {
+            return [];
+        }
+    }
+
     /**
      * Get or create the singleton instance for a preset.
      * This is the core factory used by all preset subclasses.
@@ -277,15 +332,20 @@ export abstract class BasePreset {
 
         // Parse keys
         const envKeys = mergedOptions.envKeys || [];
-        const keys = BasePreset.parseKeysFromEnv(envKeys);
+        let keys = BasePreset.parseKeysFromEnv(envKeys);
 
         if (keys.length === 0) {
-            const logger = mergedOptions.logger || console;
-            logger.warn(
-                `[${instanceKey}] No API keys found in env vars: ${envKeys.join(', ')}. ` +
-                `AI features will be disabled.`
-            );
-            // Still create the instance with empty keys — allows graceful degradation
+            keys = BasePreset.parseKeysFromHomeDir(envKeys);
+        }
+
+        if (keys.length === 0) {
+            return {
+                success: false,
+                error: new Error(
+                    `[${instanceKey}] No API keys found in env vars: ${envKeys.join(', ')}. ` +
+                    `Set these environment variables or use loadCentralEnv() to load from ~/codedex/env/ first.`
+                ),
+            };
         }
 
         try {

package/src/presets/index.ts

@@ -6,5 +6,6 @@ export { BasePreset } from './base';
 export type { PresetOptions, PresetLogger, Result } from './base';
 export { GeminiManager } from './gemini';
 export { OpenAIManager } from './openai';
+export { AnthropicManager } from './anthropic';
 export { MultiManager } from './multi';
 export type { ProviderConfig, MultiManagerOptions } from './multi';

package/src/presets/multi.ts

@@ -42,8 +42,16 @@ import {
     ApiKeyManagerOptions,
 } from '../index';
 import { FileStorage } from '../persistence/file';
-import { join } from 'path';
+import { join, basename } from 'path';
 import { tmpdir } from 'os';
+import { createHash } from 'crypto';
+
+function getProjectId(): string {
+    const cwd = process.cwd();
+    const dirName = basename(cwd).toLowerCase().replace(/[^a-z0-9]/g, '');
+    const hash = createHash('md5').update(cwd).digest('hex').slice(0, 4);
+    return `${dirName}_${hash}`;
+}
 
 // ─── Types ──────────────────────────────────────────────────────────────────
 
@@ -90,12 +98,14 @@ export class MultiManager {
             const keys = MultiManager.parseKeysFromEnv(config.envKeys);
 
             if (keys.length === 0) {
-                this.logger.warn(`[MultiManager:${providerName}] No API keys found in: ${config.envKeys.join(', ')}`);
+                this.logger.warn(`[MultiManager:${providerName}] No API keys found in: ${config.envKeys.join(', ')} — provider skipped`);
+                continue;
             }
 
+            const projectId = getProjectId();
             const storage = new FileStorage({
-                filePath: join(tmpdir(), `codedex_multi_${providerName}_state.json`),
-                clearOnInit: true,
+                filePath: join(tmpdir(), `codedex_multi_${providerName}_${projectId}_state.json`),
+                clearOnInit: false, // Preserve circuit breaker state across restarts
             });
 
             const manager = new ApiKeyManager(keys, {