mcp-agent-foundry 1.0.1 → 1.2.0

package/dist/failover/health-tracker.js

@@ -0,0 +1,350 @@
+/**
+ * Provider Health Tracker
+ *
+ * Tracks the health status of LLM providers to enable intelligent failover.
+ * Monitors success/failure rates, latency, and manages cooldown periods.
+ */
+// ============================================================================
+// Constants
+// ============================================================================
+const DEFAULT_CONFIG = {
+    failureThreshold: 3,
+    degradedThreshold: 1,
+    cooldownMs: 300000, // 5 minutes
+    healthCheckIntervalMs: 60000, // 1 minute
+    latencyWindowSize: 10,
+};
+// ============================================================================
+// ProviderHealthTracker Class
+// ============================================================================
+/**
+ * Tracks provider health for intelligent failover decisions.
+ *
+ * Features:
+ * - Tracks success/failure rates per provider
+ * - Manages cooldown periods for unhealthy providers
+ * - Provides rolling average latency metrics
+ * - Supports periodic health check callbacks
+ *
+ * @example
+ * ```typescript
+ * const tracker = new ProviderHealthTracker(logger);
+ *
+ * // Record outcomes
+ * tracker.markSuccess('openai', 150);
+ * tracker.markFailure('deepseek', new Error('Rate limited'), 429);
+ *
+ * // Check before using
+ * if (tracker.isHealthy('openai')) {
+ *   // Safe to use
+ * }
+ *
+ * // Get cooldown info
+ * const remaining = tracker.getCooldownRemaining('deepseek');
+ * ```
+ */
+export class ProviderHealthTracker {
+    health = new Map();
+    latencyHistory = new Map();
+    config;
+    logger;
+    healthCheckInterval = null;
+    healthCheckCallback = null;
+    constructor(logger, config) {
+        this.logger = logger;
+        this.config = { ...DEFAULT_CONFIG, ...config };
+    }
+    /**
+     * Record a successful request for a provider.
+     */
+    markSuccess(provider, latencyMs) {
+        const health = this.getOrCreateHealth(provider);
+        const now = new Date();
+        health.lastSuccess = now;
+        health.lastCheck = now;
+        health.consecutiveFailures = 0;
+        health.successCount++;
+        // Update rolling latency average
+        this.updateLatency(provider, latencyMs);
+        health.latencyMs = this.getAverageLatency(provider);
+        // Recover from unhealthy/degraded if in cooldown
+        if (health.cooldownUntil && now >= health.cooldownUntil) {
+            health.cooldownUntil = null;
+        }
+        // Update status based on recent performance
+        health.status = 'healthy';
+        this.logger.debug('Provider health: success', {
+            provider,
+            latencyMs,
+            status: health.status,
+            consecutiveFailures: health.consecutiveFailures,
+        });
+    }
+    /**
+     * Record a failed request for a provider.
+     */
+    markFailure(provider, error, statusCode) {
+        const health = this.getOrCreateHealth(provider);
+        const now = new Date();
+        health.lastError = {
+            code: statusCode,
+            message: error.message,
+            timestamp: now,
+        };
+        health.lastCheck = now;
+        health.consecutiveFailures++;
+        health.failureCount++;
+        // Update status based on consecutive failures
+        if (health.consecutiveFailures >= this.config.failureThreshold) {
+            health.status = 'unhealthy';
+            health.cooldownUntil = new Date(now.getTime() + this.config.cooldownMs);
+            this.logger.warn('Provider marked unhealthy', {
+                provider,
+                consecutiveFailures: health.consecutiveFailures,
+                cooldownUntil: health.cooldownUntil.toISOString(),
+                lastError: error.message,
+            });
+        }
+        else if (health.consecutiveFailures >= this.config.degradedThreshold) {
+            health.status = 'degraded';
+            this.logger.info('Provider marked degraded', {
+                provider,
+                consecutiveFailures: health.consecutiveFailures,
+                lastError: error.message,
+            });
+        }
+    }
+    /**
+     * Check if a provider is healthy and not in cooldown.
+     */
+    isHealthy(provider) {
+        const health = this.health.get(provider);
+        if (!health) {
+            return true; // Assume healthy if never tracked
+        }
+        // Check cooldown
+        if (health.cooldownUntil && new Date() < health.cooldownUntil) {
+            return false;
+        }
+        return health.status === 'healthy';
+    }
+    /**
+     * Check if a provider is available (healthy or degraded, not in cooldown).
+     */
+    isAvailable(provider) {
+        const health = this.health.get(provider);
+        if (!health) {
+            return true;
+        }
+        // Check cooldown
+        if (health.cooldownUntil && new Date() < health.cooldownUntil) {
+            return false;
+        }
+        return health.status !== 'unhealthy';
+    }
+    /**
+     * Get remaining cooldown time in ms (0 if not in cooldown).
+     */
+    getCooldownRemaining(provider) {
+        const health = this.health.get(provider);
+        if (!health?.cooldownUntil) {
+            return 0;
+        }
+        const remaining = health.cooldownUntil.getTime() - Date.now();
+        return Math.max(0, remaining);
+    }
+    /**
+     * Get health info for a provider.
+     */
+    getHealth(provider) {
+        return this.health.get(provider);
+    }
+    /**
+     * Get health info for all tracked providers.
+     */
+    getAllHealth() {
+        return new Map(this.health);
+    }
+    /**
+     * Get providers sorted by health and latency (best first).
+     */
+    getHealthyProviders(providerNames) {
+        return providerNames
+            .filter((p) => this.isAvailable(p))
+            .sort((a, b) => {
+            const healthA = this.health.get(a);
+            const healthB = this.health.get(b);
+            // Healthy before degraded
+            if (healthA?.status === 'healthy' && healthB?.status !== 'healthy')
+                return -1;
+            if (healthA?.status !== 'healthy' && healthB?.status === 'healthy')
+                return 1;
+            // Lower latency first
+            const latencyA = healthA?.latencyMs ?? 0;
+            const latencyB = healthB?.latencyMs ?? 0;
+            return latencyA - latencyB;
+        });
+    }
+    /**
+     * Reset health for a provider (e.g., after config change).
+     */
+    resetHealth(provider) {
+        this.health.delete(provider);
+        this.latencyHistory.delete(provider);
+        this.logger.info('Provider health reset', { provider });
+    }
+    /**
+     * Clear all health tracking data.
+     */
+    clear() {
+        this.health.clear();
+        this.latencyHistory.clear();
+        this.logger.info('All provider health data cleared');
+    }
+    /**
+     * Start periodic health checks.
+     */
+    startHealthCheckLoop(callback) {
+        if (this.healthCheckInterval) {
+            return; // Already running
+        }
+        this.healthCheckCallback = callback;
+        this.healthCheckInterval = setInterval(() => void this.runHealthChecks(), this.config.healthCheckIntervalMs);
+        this.logger.info('Health check loop started', {
+            intervalMs: this.config.healthCheckIntervalMs,
+        });
+    }
+    /**
+     * Stop periodic health checks.
+     */
+    stopHealthCheckLoop() {
+        if (this.healthCheckInterval) {
+            clearInterval(this.healthCheckInterval);
+            this.healthCheckInterval = null;
+            this.healthCheckCallback = null;
+            this.logger.info('Health check loop stopped');
+        }
+    }
+    /**
+     * Serialize health state for persistence.
+     */
+    serialize() {
+        const result = [];
+        for (const [provider, health] of this.health) {
+            result.push({
+                provider,
+                status: health.status,
+                consecutiveFailures: health.consecutiveFailures,
+                cooldownUntil: health.cooldownUntil?.getTime() ?? null,
+                latencyMs: health.latencyMs,
+                successCount: health.successCount,
+                failureCount: health.failureCount,
+                lastSuccessAt: health.lastSuccess?.getTime() ?? null,
+                lastErrorAt: health.lastError?.timestamp.getTime() ?? null,
+                lastErrorMessage: health.lastError?.message ?? null,
+                lastErrorCode: health.lastError?.code ?? null,
+            });
+        }
+        return result;
+    }
+    /**
+     * Restore health state from persistence.
+     */
+    restore(data) {
+        for (const item of data) {
+            const health = {
+                provider: item.provider,
+                status: item.status,
+                lastCheck: new Date(),
+                lastSuccess: item.lastSuccessAt ? new Date(item.lastSuccessAt) : null,
+                lastError: item.lastErrorAt
+                    ? {
+                        code: item.lastErrorCode ?? undefined,
+                        message: item.lastErrorMessage ?? 'Unknown error',
+                        timestamp: new Date(item.lastErrorAt),
+                    }
+                    : null,
+                consecutiveFailures: item.consecutiveFailures,
+                cooldownUntil: item.cooldownUntil ? new Date(item.cooldownUntil) : null,
+                latencyMs: item.latencyMs,
+                successCount: item.successCount,
+                failureCount: item.failureCount,
+            };
+            this.health.set(item.provider, health);
+        }
+        this.logger.info('Provider health restored', {
+            providerCount: data.length,
+        });
+    }
+    // ==========================================================================
+    // Private Methods
+    // ==========================================================================
+    getOrCreateHealth(provider) {
+        let health = this.health.get(provider);
+        if (!health) {
+            health = {
+                provider,
+                status: 'healthy',
+                lastCheck: new Date(),
+                lastSuccess: null,
+                lastError: null,
+                consecutiveFailures: 0,
+                cooldownUntil: null,
+                latencyMs: 0,
+                successCount: 0,
+                failureCount: 0,
+            };
+            this.health.set(provider, health);
+        }
+        return health;
+    }
+    updateLatency(provider, latencyMs) {
+        let history = this.latencyHistory.get(provider);
+        if (!history) {
+            history = [];
+            this.latencyHistory.set(provider, history);
+        }
+        history.push(latencyMs);
+        // Keep only the last N samples
+        if (history.length > this.config.latencyWindowSize) {
+            history.shift();
+        }
+    }
+    getAverageLatency(provider) {
+        const history = this.latencyHistory.get(provider);
+        if (!history || history.length === 0) {
+            return 0;
+        }
+        const sum = history.reduce((a, b) => a + b, 0);
+        return Math.round(sum / history.length);
+    }
+    async runHealthChecks() {
+        if (!this.healthCheckCallback) {
+            return;
+        }
+        // Check providers that are in cooldown and might be ready to recover
+        const now = Date.now();
+        for (const [provider, health] of this.health) {
+            if (health.status === 'unhealthy' && health.cooldownUntil) {
+                if (now >= health.cooldownUntil.getTime()) {
+                    this.logger.debug('Running health check for provider exiting cooldown', {
+                        provider,
+                    });
+                    try {
+                        await this.healthCheckCallback(provider);
+                        // If the callback doesn't throw, consider it a success
+                        this.logger.info('Provider recovered after cooldown', { provider });
+                    }
+                    catch (error) {
+                        // Failed, will stay unhealthy and cooldown will be extended
+                        this.logger.debug('Provider health check failed', {
+                            provider,
+                            error: error instanceof Error ? error.message : String(error),
+                        });
+                    }
+                }
+            }
+        }
+    }
+}
+//# sourceMappingURL=health-tracker.js.map

package/dist/failover/health-tracker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"health-tracker.js","sourceRoot":"","sources":["../../src/failover/health-tracker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA2DH,+EAA+E;AAC/E,YAAY;AACZ,+EAA+E;AAE/E,MAAM,cAAc,GAAkC;IACpD,gBAAgB,EAAE,CAAC;IACnB,iBAAiB,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,EAAE,YAAY;IAChC,qBAAqB,EAAE,KAAK,EAAE,WAAW;IACzC,iBAAiB,EAAE,EAAE;CACtB,CAAC;AAEF,+EAA+E;AAC/E,8BAA8B;AAC9B,+EAA+E;AAE/E;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,OAAO,qBAAqB;IACf,MAAM,GAAgC,IAAI,GAAG,EAAE,CAAC;IAChD,cAAc,GAA0B,IAAI,GAAG,EAAE,CAAC;IAClD,MAAM,CAAgC;IACtC,MAAM,CAAS;IACxB,mBAAmB,GAA0C,IAAI,CAAC;IAClE,mBAAmB,GAAiD,IAAI,CAAC;IAEjF,YAAY,MAAc,EAAE,MAA4B;QACtD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB,EAAE,SAAiB;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC;QACzB,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC;QACvB,MAAM,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC/B,MAAM,CAAC,YAAY,EAAE,CAAC;QAEtB,iCAAiC;QACjC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACxC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAEpD,iDAAiD;QACjD,IAAI,MAAM,CAAC,aAAa,IAAI,GAAG,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACxD,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC;QAC9B,CAAC;QAED,4CAA4C;QAC5C,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;QAE1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE;YAC5C,QAAQ;YACR,SAAS;YACT,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;SAChD,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB,EAAE,KAAY,EAAE,UAAmB;QAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,MAAM,CAAC,SAAS,GAAG;YACjB,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,GAAG;SACf,CAAC;QACF,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC;QACvB,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC7B,MAAM,CAAC,YAAY,EAAE,CAAC;QAEtB,8CAA8C;QAC9C,IAAI,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC/D,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;YAC5B,MAAM,CAAC,aAAa,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAExE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBAC5C,QAAQ;gBACR,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE;gBACjD,SAAS,EAAE,KAAK,CAAC,OAAO;aACzB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACvE,MAAM,CAAC,MAAM,GAAG,UAAU,CAAC;YAE3B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;gBAC3C,QAAQ;gBACR,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,SAAS,EAAE,KAAK,CAAC,OAAO;aACzB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,QAAgB;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,CAAC,kCAAkC;QACjD,CAAC;QAED,iBAAiB;QACjB,IAAI,MAAM,CAAC,aAAa,IAAI,IAAI,IAAI,EAAE,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;YAC9D,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC;QACd,CAAC;QAED,iBAAiB;QACjB,IAAI,MAAM,CAAC,aAAa,IAAI,IAAI,IAAI,EAAE,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;YAC9D,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,QAAgB;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;YAC3B,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9D,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,QAAgB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,aAAuB;QACzC,OAAO,aAAa;aACjB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;aAClC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACb,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAEnC,0BAA0B;YAC1B,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS;gBAAE,OAAO,CAAC,CAAC,CAAC;YAC9E,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS;gBAAE,OAAO,CAAC,CAAC;YAE7E,sBAAsB;YACtB,MAAM,QAAQ,GAAG,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC;YACzC,MAAM,QAAQ,GAAG,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC;YACzC,OAAO,QAAQ,GAAG,QAAQ,CAAC;QAC7B,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,QAAgB;QAC1B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAErC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,QAA6C;QAChE,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,OAAO,CAAC,kBAAkB;QAC5B,CAAC;QAED,IAAI,CAAC,mBAAmB,GAAG,QAAQ,CAAC;QACpC,IAAI,CAAC,mBAAmB,GAAG,WAAW,CACpC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,eAAe,EAAE,EACjC,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAClC,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;YAC5C,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB;SAC9C,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACxC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAChC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAEhC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED;;OAEG;IACH,SAAS;QAaP,MAAM,MAAM,GAYP,EAAE,CAAC;QAER,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ;gBACR,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,OAAO,EAAE,IAAI,IAAI;gBACtD,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,aAAa,EAAE,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,IAAI,IAAI;gBACpD,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,OAAO,EAAE,IAAI,IAAI;gBAC1D,gBAAgB,EAAE,MAAM,CAAC,SAAS,EAAE,OAAO,IAAI,IAAI;gBACnD,aAAa,EAAE,MAAM,CAAC,SAAS,EAAE,IAAI,IAAI,IAAI;aAC9C,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,IAYN;QACA,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,MAAM,GAAmB;gBAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,WAAW,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI;gBACrE,SAAS,EAAE,IAAI,CAAC,WAAW;oBACzB,CAAC,CAAC;wBACE,IAAI,EAAE,IAAI,CAAC,aAAa,IAAI,SAAS;wBACrC,OAAO,EAAE,IAAI,CAAC,gBAAgB,IAAI,eAAe;wBACjD,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC;qBACtC;oBACH,CAAC,CAAC,IAAI;gBACR,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;gBAC7C,aAAa,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI;gBACvE,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC;YAEF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;YAC3C,aAAa,EAAE,IAAI,CAAC,MAAM;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,6EAA6E;IAC7E,kBAAkB;IAClB,6EAA6E;IAErE,iBAAiB,CAAC,QAAgB;QACxC,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,GAAG;gBACP,QAAQ;gBACR,MAAM,EAAE,SAAS;gBACjB,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,WAAW,EAAE,IAAI;gBACjB,SAAS,EAAE,IAAI;gBACf,mBAAmB,EAAE,CAAC;gBACtB,aAAa,EAAE,IAAI;gBACnB,SAAS,EAAE,CAAC;gBACZ,YAAY,EAAE,CAAC;gBACf,YAAY,EAAE,CAAC;aAChB,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,aAAa,CAAC,QAAgB,EAAE,SAAiB;QACvD,IAAI,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAExB,+BAA+B;QAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACnD,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,CAAC;IACH,CAAC;IAEO,iBAAiB,CAAC,QAAgB;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAEO,KAAK,CAAC,eAAe;QAC3B,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QAED,qEAAqE;QACrE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;gBAC1D,IAAI,GAAG,IAAI,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC1C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oDAAoD,EAAE;wBACtE,QAAQ;qBACT,CAAC,CAAC;oBAEH,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;wBACzC,uDAAuD;wBACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;oBACtE,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,4DAA4D;wBAC5D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE;4BAChD,QAAQ;4BACR,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;yBAC9D,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/failover/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Failover Module
+ *
+ * Provides intelligent failover capabilities for provider operations.
+ */
+export * from './health-tracker.js';
+export * from './pricing.js';
+export * from './orchestrator.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/failover/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/failover/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC"}

package/dist/failover/index.js

@@ -0,0 +1,9 @@
+/**
+ * Failover Module
+ *
+ * Provides intelligent failover capabilities for provider operations.
+ */
+export * from './health-tracker.js';
+export * from './pricing.js';
+export * from './orchestrator.js';
+//# sourceMappingURL=index.js.map

package/dist/failover/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/failover/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC"}

package/dist/failover/orchestrator.d.ts

@@ -0,0 +1,189 @@
+/**
+ * Failover Orchestrator
+ *
+ * Wraps provider operations with intelligent retry and failover logic.
+ * Integrates health tracking, circuit breakers, and cost-aware routing.
+ */
+import type { Logger } from '../observability/logger.js';
+import type { ProviderManager } from '../providers/manager.js';
+import type { Config, CompletionResponse, Message } from '../types.js';
+import { type CircuitBreakerOptions } from '../utils/circuit-breaker.js';
+import { ProviderHealthTracker, type ProviderHealth } from './health-tracker.js';
+import { PricingService } from './pricing.js';
+import type { PersistedFailoverEvent, PersistedProviderHealth } from '../persistence/state-schema.js';
+/**
+ * Options for a single completion request with failover.
+ */
+export interface FailoverCompletionOptions {
+    /** Request temperature */
+    temperature?: number;
+    /** Maximum tokens to generate */
+    max_tokens?: number;
+    /** Request timeout in ms */
+    timeout_ms?: number;
+}
+/**
+ * Result of a completion with failover metadata.
+ */
+export interface FailoverCompletionResult {
+    /** The completion response */
+    response: CompletionResponse;
+    /** Provider that handled the request */
+    provider: string;
+    /** Model used */
+    model: string;
+    /** Number of retry attempts */
+    retryCount: number;
+    /** Number of providers tried */
+    failoverCount: number;
+    /** Total latency in ms */
+    latencyMs: number;
+    /** Whether failover was triggered */
+    usedFailover: boolean;
+}
+/**
+ * Configuration for the failover orchestrator.
+ */
+export interface FailoverOrchestratorConfig {
+    /** Enable failover (default: true) */
+    enabled?: boolean;
+    /** Maximum total retries across all providers (default: 6) */
+    maxTotalRetries?: number;
+    /** Health check interval in ms (default: 60000) */
+    healthCheckIntervalMs?: number;
+    /** Cooldown period in ms (default: 300000) */
+    cooldownMs?: number;
+    /** Use cheapest healthy provider when no preference (default: false) */
+    preferCostEfficient?: boolean;
+    /** Circuit breaker options per provider */
+    circuitBreaker?: Partial<CircuitBreakerOptions>;
+}
+/**
+ * Failover event for logging and persistence.
+ */
+export interface FailoverEvent {
+    timestamp: Date;
+    role: string;
+    fromProvider: string;
+    toProvider: string;
+    reason: string;
+    errorCode?: number;
+    errorMessage?: string;
+}
+/**
+ * Orchestrates provider requests with intelligent failover.
+ *
+ * Features:
+ * - Automatic retry with exponential backoff
+ * - Multi-provider failover chain
+ * - Health tracking and cooldown management
+ * - Circuit breaker per provider
+ * - Cost-aware provider selection
+ * - Failover event logging for diagnostics
+ *
+ * @example
+ * ```typescript
+ * const orchestrator = new FailoverOrchestrator(
+ *   providers,
+ *   config,
+ *   logger
+ * );
+ *
+ * // Execute with automatic failover
+ * const result = await orchestrator.executeWithFailover(
+ *   'coder',
+ *   messages,
+ *   { temperature: 0.7 }
+ * );
+ * ```
+ */
+export declare class FailoverOrchestrator {
+    private readonly providers;
+    private readonly appConfig;
+    private readonly logger;
+    private readonly config;
+    private readonly healthTracker;
+    private readonly pricingService;
+    private readonly circuitBreakers;
+    private readonly failoverEvents;
+    private readonly maxEventHistory;
+    constructor(providers: ProviderManager, appConfig: Config, logger: Logger, config?: FailoverOrchestratorConfig);
+    /**
+     * Execute a completion request with automatic failover.
+     */
+    executeWithFailover(role: string, messages: Message[], options: FailoverCompletionOptions): Promise<FailoverCompletionResult>;
+    /**
+     * Get available providers for a role, sorted by health and optionally cost.
+     */
+    getAvailableProviders(role: string): Array<{
+        provider: string;
+        model: string;
+    }>;
+    /**
+     * Get health status for all tracked providers.
+     */
+    getProviderHealth(): Map<string, ProviderHealth>;
+    /**
+     * Get recent failover events.
+     */
+    getFailoverEvents(): FailoverEvent[];
+    /**
+     * Initialize the orchestrator (fetch pricing, start health checks).
+     */
+    initialize(): Promise<void>;
+    /**
+     * Shutdown the orchestrator (stop health checks, persist state).
+     */
+    shutdown(): void;
+    /**
+     * Serialize state for persistence.
+     */
+    serializeState(): {
+        providerHealth: PersistedProviderHealth[];
+        failoverEvents: PersistedFailoverEvent[];
+    };
+    /**
+     * Restore state from persistence.
+     */
+    restoreState(state: {
+        providerHealth?: PersistedProviderHealth[];
+        failoverEvents?: PersistedFailoverEvent[];
+    }): void;
+    /**
+     * Get the pricing service for external use.
+     */
+    getPricingService(): PricingService;
+    /**
+     * Get the health tracker for external use.
+     */
+    getHealthTracker(): ProviderHealthTracker;
+    /**
+     * Build the ordered chain of providers to try for a role.
+     */
+    private buildProviderChain;
+    /**
+     * Execute a single provider request with retry logic.
+     */
+    private executeWithRetry;
+    /**
+     * Execute without failover (single provider only).
+     */
+    private executeSingleProvider;
+    /**
+     * Check if an error should trigger failover.
+     */
+    private shouldTriggerFailover;
+    /**
+     * Get the next provider in the chain that hasn't been tried.
+     */
+    private getNextProvider;
+    /**
+     * Get or create a circuit breaker for a provider.
+     */
+    private getOrCreateCircuitBreaker;
+    /**
+     * Record a failover event for diagnostics.
+     */
+    private recordFailoverEvent;
+}
+//# sourceMappingURL=orchestrator.d.ts.map

package/dist/failover/orchestrator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/failover/orchestrator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,KAAK,EAAE,MAAM,EAAqB,kBAAkB,EAAE,OAAO,EAAmC,MAAM,aAAa,CAAC;AAG3H,OAAO,EAAoC,KAAK,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAC3G,OAAO,EAAE,qBAAqB,EAAE,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAMtG;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iCAAiC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,8BAA8B;IAC9B,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,wCAAwC;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,0BAA0B;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,YAAY,EAAE,OAAO,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,sCAAsC;IACtC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,8DAA8D;IAC9D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mDAAmD;IACnD,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,8CAA8C;IAC9C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,2CAA2C;IAC3C,cAAc,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,CAAC;CACjD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,IAAI,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AA4BD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAkB;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuC;IAC9D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAwB;IACtD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA0C;IAC1E,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAuB;IACtD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAO;gBAGrC,SAAS,EAAE,eAAe,EAC1B,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,MAAM,CAAC,EAAE,0BAA0B;IAiBrC;;OAEG;IACG,mBAAmB,CACvB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,wBAAwB,CAAC;IAgHpC;;OAEG;IACH,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAuB/E;;OAEG;IACH,iBAAiB,IAAI,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC;IAIhD;;OAEG;IACH,iBAAiB,IAAI,aAAa,EAAE;IAIpC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAajC;;OAEG;IACH,QAAQ,IAAI,IAAI;IAKhB;;OAEG;IACH,cAAc,IAAI;QAChB,cAAc,EAAE,uBAAuB,EAAE,CAAC;QAC1C,cAAc,EAAE,sBAAsB,EAAE,CAAC;KAC1C;IAeD;;OAEG;IACH,YAAY,CAAC,KAAK,EAAE;QAClB,cAAc,CAAC,EAAE,uBAAuB,EAAE,CAAC;QAC3C,cAAc,CAAC,EAAE,sBAAsB,EAAE,CAAC;KAC3C,GAAG,IAAI;IAuBR;;OAEG;IACH,iBAAiB,IAAI,cAAc;IAInC;;OAEG;IACH,gBAAgB,IAAI,qBAAqB;IAQzC;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAuC1B;;OAEG;YACW,gBAAgB;IAiF9B;;OAEG;YACW,qBAAqB;IAoCnC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAwB7B;;OAEG;IACH,OAAO,CAAC,eAAe;IAYvB;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAoBjC;;OAEG;IACH,OAAO,CAAC,mBAAmB;CA4B5B"}