mcp-agent-foundry 1.1.0 → 1.3.0

package/dist/failover/orchestrator.d.ts

@@ -0,0 +1,189 @@
+/**
+ * Failover Orchestrator
+ *
+ * Wraps provider operations with intelligent retry and failover logic.
+ * Integrates health tracking, circuit breakers, and cost-aware routing.
+ */
+import type { Logger } from '../observability/logger.js';
+import type { ProviderManager } from '../providers/manager.js';
+import type { Config, CompletionResponse, Message } from '../types.js';
+import { type CircuitBreakerOptions } from '../utils/circuit-breaker.js';
+import { ProviderHealthTracker, type ProviderHealth } from './health-tracker.js';
+import { PricingService } from './pricing.js';
+import type { PersistedFailoverEvent, PersistedProviderHealth } from '../persistence/state-schema.js';
+/**
+ * Options for a single completion request with failover.
+ */
+export interface FailoverCompletionOptions {
+    /** Request temperature */
+    temperature?: number;
+    /** Maximum tokens to generate */
+    max_tokens?: number;
+    /** Request timeout in ms */
+    timeout_ms?: number;
+}
+/**
+ * Result of a completion with failover metadata.
+ */
+export interface FailoverCompletionResult {
+    /** The completion response */
+    response: CompletionResponse;
+    /** Provider that handled the request */
+    provider: string;
+    /** Model used */
+    model: string;
+    /** Number of retry attempts */
+    retryCount: number;
+    /** Number of providers tried */
+    failoverCount: number;
+    /** Total latency in ms */
+    latencyMs: number;
+    /** Whether failover was triggered */
+    usedFailover: boolean;
+}
+/**
+ * Configuration for the failover orchestrator.
+ */
+export interface FailoverOrchestratorConfig {
+    /** Enable failover (default: true) */
+    enabled?: boolean;
+    /** Maximum total retries across all providers (default: 6) */
+    maxTotalRetries?: number;
+    /** Health check interval in ms (default: 60000) */
+    healthCheckIntervalMs?: number;
+    /** Cooldown period in ms (default: 300000) */
+    cooldownMs?: number;
+    /** Use cheapest healthy provider when no preference (default: false) */
+    preferCostEfficient?: boolean;
+    /** Circuit breaker options per provider */
+    circuitBreaker?: Partial<CircuitBreakerOptions>;
+}
+/**
+ * Failover event for logging and persistence.
+ */
+export interface FailoverEvent {
+    timestamp: Date;
+    role: string;
+    fromProvider: string;
+    toProvider: string;
+    reason: string;
+    errorCode?: number;
+    errorMessage?: string;
+}
+/**
+ * Orchestrates provider requests with intelligent failover.
+ *
+ * Features:
+ * - Automatic retry with exponential backoff
+ * - Multi-provider failover chain
+ * - Health tracking and cooldown management
+ * - Circuit breaker per provider
+ * - Cost-aware provider selection
+ * - Failover event logging for diagnostics
+ *
+ * @example
+ * ```typescript
+ * const orchestrator = new FailoverOrchestrator(
+ *   providers,
+ *   config,
+ *   logger
+ * );
+ *
+ * // Execute with automatic failover
+ * const result = await orchestrator.executeWithFailover(
+ *   'coder',
+ *   messages,
+ *   { temperature: 0.7 }
+ * );
+ * ```
+ */
+export declare class FailoverOrchestrator {
+    private readonly providers;
+    private readonly appConfig;
+    private readonly logger;
+    private readonly config;
+    private readonly healthTracker;
+    private readonly pricingService;
+    private readonly circuitBreakers;
+    private readonly failoverEvents;
+    private readonly maxEventHistory;
+    constructor(providers: ProviderManager, appConfig: Config, logger: Logger, config?: FailoverOrchestratorConfig);
+    /**
+     * Execute a completion request with automatic failover.
+     */
+    executeWithFailover(role: string, messages: Message[], options: FailoverCompletionOptions): Promise<FailoverCompletionResult>;
+    /**
+     * Get available providers for a role, sorted by health and optionally cost.
+     */
+    getAvailableProviders(role: string): Array<{
+        provider: string;
+        model: string;
+    }>;
+    /**
+     * Get health status for all tracked providers.
+     */
+    getProviderHealth(): Map<string, ProviderHealth>;
+    /**
+     * Get recent failover events.
+     */
+    getFailoverEvents(): FailoverEvent[];
+    /**
+     * Initialize the orchestrator (fetch pricing, start health checks).
+     */
+    initialize(): Promise<void>;
+    /**
+     * Shutdown the orchestrator (stop health checks, persist state).
+     */
+    shutdown(): void;
+    /**
+     * Serialize state for persistence.
+     */
+    serializeState(): {
+        providerHealth: PersistedProviderHealth[];
+        failoverEvents: PersistedFailoverEvent[];
+    };
+    /**
+     * Restore state from persistence.
+     */
+    restoreState(state: {
+        providerHealth?: PersistedProviderHealth[];
+        failoverEvents?: PersistedFailoverEvent[];
+    }): void;
+    /**
+     * Get the pricing service for external use.
+     */
+    getPricingService(): PricingService;
+    /**
+     * Get the health tracker for external use.
+     */
+    getHealthTracker(): ProviderHealthTracker;
+    /**
+     * Build the ordered chain of providers to try for a role.
+     */
+    private buildProviderChain;
+    /**
+     * Execute a single provider request with retry logic.
+     */
+    private executeWithRetry;
+    /**
+     * Execute without failover (single provider only).
+     */
+    private executeSingleProvider;
+    /**
+     * Check if an error should trigger failover.
+     */
+    private shouldTriggerFailover;
+    /**
+     * Get the next provider in the chain that hasn't been tried.
+     */
+    private getNextProvider;
+    /**
+     * Get or create a circuit breaker for a provider.
+     */
+    private getOrCreateCircuitBreaker;
+    /**
+     * Record a failover event for diagnostics.
+     */
+    private recordFailoverEvent;
+}
+//# sourceMappingURL=orchestrator.d.ts.map

package/dist/failover/orchestrator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/failover/orchestrator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,KAAK,EAAE,MAAM,EAAqB,kBAAkB,EAAE,OAAO,EAAmC,MAAM,aAAa,CAAC;AAG3H,OAAO,EAAoC,KAAK,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAC3G,OAAO,EAAE,qBAAqB,EAAE,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAMtG;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iCAAiC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,8BAA8B;IAC9B,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,wCAAwC;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,0BAA0B;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,YAAY,EAAE,OAAO,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,sCAAsC;IACtC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,8DAA8D;IAC9D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mDAAmD;IACnD,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,8CAA8C;IAC9C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,2CAA2C;IAC3C,cAAc,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,CAAC;CACjD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,IAAI,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AA4BD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAkB;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuC;IAC9D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAwB;IACtD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA0C;IAC1E,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAuB;IACtD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAO;gBAGrC,SAAS,EAAE,eAAe,EAC1B,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,MAAM,CAAC,EAAE,0BAA0B;IAiBrC;;OAEG;IACG,mBAAmB,CACvB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,wBAAwB,CAAC;IAgHpC;;OAEG;IACH,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAuB/E;;OAEG;IACH,iBAAiB,IAAI,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC;IAIhD;;OAEG;IACH,iBAAiB,IAAI,aAAa,EAAE;IAIpC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAajC;;OAEG;IACH,QAAQ,IAAI,IAAI;IAKhB;;OAEG;IACH,cAAc,IAAI;QAChB,cAAc,EAAE,uBAAuB,EAAE,CAAC;QAC1C,cAAc,EAAE,sBAAsB,EAAE,CAAC;KAC1C;IAeD;;OAEG;IACH,YAAY,CAAC,KAAK,EAAE;QAClB,cAAc,CAAC,EAAE,uBAAuB,EAAE,CAAC;QAC3C,cAAc,CAAC,EAAE,sBAAsB,EAAE,CAAC;KAC3C,GAAG,IAAI;IAuBR;;OAEG;IACH,iBAAiB,IAAI,cAAc;IAInC;;OAEG;IACH,gBAAgB,IAAI,qBAAqB;IAQzC;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAuC1B;;OAEG;YACW,gBAAgB;IAiF9B;;OAEG;YACW,qBAAqB;IAoCnC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAwB7B;;OAEG;IACH,OAAO,CAAC,eAAe;IAYvB;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAoBjC;;OAEG;IACH,OAAO,CAAC,mBAAmB;CA4B5B"}

package/dist/failover/orchestrator.js

@@ -0,0 +1,488 @@
+/**
+ * Failover Orchestrator
+ *
+ * Wraps provider operations with intelligent retry and failover logic.
+ * Integrates health tracking, circuit breakers, and cost-aware routing.
+ */
+import { ProviderError, RateLimitError, TimeoutError, FailoverExhaustedError } from '../types.js';
+import { retry, isRetryableError } from '../utils/retry.js';
+import { CircuitBreaker, CircuitOpenError } from '../utils/circuit-breaker.js';
+import { ProviderHealthTracker } from './health-tracker.js';
+import { PricingService } from './pricing.js';
+// ============================================================================
+// Constants
+// ============================================================================
+const DEFAULT_CONFIG = {
+    enabled: true,
+    maxTotalRetries: 6,
+    healthCheckIntervalMs: 60000,
+    cooldownMs: 300000,
+    preferCostEfficient: false,
+    circuitBreaker: {
+        failureThreshold: 5,
+        successThreshold: 2,
+        timeout: 30000,
+    },
+};
+/**
+ * Default error codes that trigger failover.
+ */
+const DEFAULT_FAILOVER_ERRORS = [429, 500, 502, 503, 504];
+// ============================================================================
+// FailoverOrchestrator Class
+// ============================================================================
+/**
+ * Orchestrates provider requests with intelligent failover.
+ *
+ * Features:
+ * - Automatic retry with exponential backoff
+ * - Multi-provider failover chain
+ * - Health tracking and cooldown management
+ * - Circuit breaker per provider
+ * - Cost-aware provider selection
+ * - Failover event logging for diagnostics
+ *
+ * @example
+ * ```typescript
+ * const orchestrator = new FailoverOrchestrator(
+ *   providers,
+ *   config,
+ *   logger
+ * );
+ *
+ * // Execute with automatic failover
+ * const result = await orchestrator.executeWithFailover(
+ *   'coder',
+ *   messages,
+ *   { temperature: 0.7 }
+ * );
+ * ```
+ */
+export class FailoverOrchestrator {
+    providers;
+    appConfig;
+    logger;
+    config;
+    healthTracker;
+    pricingService;
+    circuitBreakers = new Map();
+    failoverEvents = [];
+    maxEventHistory = 100;
+    constructor(providers, appConfig, logger, config) {
+        this.providers = providers;
+        this.appConfig = appConfig;
+        this.logger = logger;
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        // Initialize health tracker
+        this.healthTracker = new ProviderHealthTracker(logger, {
+            cooldownMs: this.config.cooldownMs,
+            healthCheckIntervalMs: this.config.healthCheckIntervalMs,
+        });
+        // Initialize pricing service
+        this.pricingService = new PricingService(logger);
+    }
+    /**
+     * Execute a completion request with automatic failover.
+     */
+    async executeWithFailover(role, messages, options) {
+        if (!this.config.enabled) {
+            // Failover disabled, use primary provider only
+            return this.executeSingleProvider(role, messages, options);
+        }
+        const startTime = Date.now();
+        const roleConfig = this.appConfig.roles[role];
+        if (!roleConfig) {
+            throw new Error(`Unknown role: ${role}`);
+        }
+        // Build the chain of providers to try
+        const providerChain = this.buildProviderChain(role, roleConfig);
+        const errors = new Map();
+        const attemptedProviders = [];
+        let totalRetries = 0;
+        let failoverCount = 0;
+        // Get failover error codes from config or use defaults
+        const failoverErrors = roleConfig.fallback_chain?.on_errors ?? DEFAULT_FAILOVER_ERRORS;
+        // Get retry config from role or use defaults
+        const retryConfig = roleConfig.fallback_chain?.retry ?? {};
+        for (const { provider, model } of providerChain) {
+            if (totalRetries >= this.config.maxTotalRetries) {
+                this.logger.warn('Max total retries reached', {
+                    role,
+                    totalRetries,
+                    attemptedProviders,
+                });
+                break;
+            }
+            // Check if provider is available
+            if (!this.healthTracker.isAvailable(provider)) {
+                const cooldown = this.healthTracker.getCooldownRemaining(provider);
+                this.logger.debug('Skipping provider in cooldown', {
+                    provider,
+                    cooldownRemainingMs: cooldown,
+                });
+                continue;
+            }
+            // Check circuit breaker
+            const breaker = this.getOrCreateCircuitBreaker(provider);
+            if (breaker.getState() === 'OPEN') {
+                this.logger.debug('Skipping provider with open circuit', { provider });
+                continue;
+            }
+            attemptedProviders.push(provider);
+            try {
+                const result = await this.executeWithRetry(provider, model, messages, options, retryConfig, failoverErrors);
+                const latencyMs = Date.now() - startTime;
+                return {
+                    response: result.response,
+                    provider,
+                    model,
+                    retryCount: result.retryCount,
+                    failoverCount,
+                    latencyMs,
+                    usedFailover: failoverCount > 0,
+                };
+            }
+            catch (error) {
+                const err = error instanceof Error ? error : new Error(String(error));
+                errors.set(provider, err);
+                totalRetries++;
+                // Check if this error should trigger failover
+                const shouldFailover = this.shouldTriggerFailover(err, failoverErrors);
+                if (shouldFailover && providerChain.length > attemptedProviders.length) {
+                    const nextProvider = this.getNextProvider(providerChain, attemptedProviders);
+                    if (nextProvider) {
+                        this.recordFailoverEvent(role, provider, nextProvider.provider, err);
+                        failoverCount++;
+                        this.logger.info('Failing over to next provider', {
+                            role,
+                            fromProvider: provider,
+                            toProvider: nextProvider.provider,
+                            errorMessage: err.message,
+                        });
+                        continue;
+                    }
+                }
+                // No more providers or not a failover-triggering error
+                throw err;
+            }
+        }
+        // All providers exhausted
+        throw new FailoverExhaustedError(role, attemptedProviders, errors);
+    }
+    /**
+     * Get available providers for a role, sorted by health and optionally cost.
+     */
+    getAvailableProviders(role) {
+        const roleConfig = this.appConfig.roles[role];
+        if (!roleConfig) {
+            return [];
+        }
+        const chain = this.buildProviderChain(role, roleConfig);
+        // Filter to only configured and available providers
+        const available = chain.filter(({ provider }) => {
+            const isConfigured = this.providers.isConfigured(provider);
+            const isAvailable = this.healthTracker.isAvailable(provider);
+            return isConfigured && isAvailable;
+        });
+        // Optionally sort by cost
+        if (this.config.preferCostEfficient) {
+            return this.pricingService.sortByCost(available);
+        }
+        return available;
+    }
+    /**
+     * Get health status for all tracked providers.
+     */
+    getProviderHealth() {
+        return this.healthTracker.getAllHealth();
+    }
+    /**
+     * Get recent failover events.
+     */
+    getFailoverEvents() {
+        return [...this.failoverEvents];
+    }
+    /**
+     * Initialize the orchestrator (fetch pricing, start health checks).
+     */
+    async initialize() {
+        // Fetch pricing data
+        await this.pricingService.refresh();
+        // Start health check loop
+        this.healthTracker.startHealthCheckLoop(async (provider) => {
+            const providerInstance = this.providers.get(provider);
+            await providerInstance.healthCheck();
+        });
+        this.logger.info('Failover orchestrator initialized');
+    }
+    /**
+     * Shutdown the orchestrator (stop health checks, persist state).
+     */
+    shutdown() {
+        this.healthTracker.stopHealthCheckLoop();
+        this.logger.info('Failover orchestrator shut down');
+    }
+    /**
+     * Serialize state for persistence.
+     */
+    serializeState() {
+        return {
+            providerHealth: this.healthTracker.serialize(),
+            failoverEvents: this.failoverEvents.map((e) => ({
+                timestamp: e.timestamp.getTime(),
+                role: e.role,
+                fromProvider: e.fromProvider,
+                toProvider: e.toProvider,
+                reason: e.reason,
+                errorCode: e.errorCode,
+                errorMessage: e.errorMessage,
+            })),
+        };
+    }
+    /**
+     * Restore state from persistence.
+     */
+    restoreState(state) {
+        if (state.providerHealth) {
+            this.healthTracker.restore(state.providerHealth);
+        }
+        if (state.failoverEvents) {
+            this.failoverEvents.length = 0;
+            for (const e of state.failoverEvents) {
+                this.failoverEvents.push({
+                    timestamp: new Date(e.timestamp),
+                    role: e.role,
+                    fromProvider: e.fromProvider,
+                    toProvider: e.toProvider,
+                    reason: e.reason,
+                    errorCode: e.errorCode,
+                    errorMessage: e.errorMessage,
+                });
+            }
+        }
+        this.logger.info('Failover orchestrator state restored');
+    }
+    /**
+     * Get the pricing service for external use.
+     */
+    getPricingService() {
+        return this.pricingService;
+    }
+    /**
+     * Get the health tracker for external use.
+     */
+    getHealthTracker() {
+        return this.healthTracker;
+    }
+    // ==========================================================================
+    // Private Methods
+    // ==========================================================================
+    /**
+     * Build the ordered chain of providers to try for a role.
+     */
+    buildProviderChain(role, roleConfig) {
+        const chain = [];
+        // Primary provider
+        chain.push({
+            provider: roleConfig.provider,
+            model: roleConfig.model,
+        });
+        // Fallback chain (extended)
+        if (roleConfig.fallback_chain?.providers) {
+            for (const fb of roleConfig.fallback_chain.providers) {
+                chain.push({
+                    provider: fb.provider,
+                    model: fb.model,
+                });
+            }
+        }
+        // Legacy single fallback
+        if (roleConfig.fallback) {
+            // Only add if not already in chain
+            const exists = chain.some((p) => p.provider === roleConfig.fallback.provider && p.model === roleConfig.fallback.model);
+            if (!exists) {
+                chain.push({
+                    provider: roleConfig.fallback.provider,
+                    model: roleConfig.fallback.model,
+                });
+            }
+        }
+        return chain;
+    }
+    /**
+     * Execute a single provider request with retry logic.
+     */
+    async executeWithRetry(provider, model, messages, options, retryConfig, _failoverErrors) {
+        const breaker = this.getOrCreateCircuitBreaker(provider);
+        let retryCount = 0;
+        const retryOptions = {
+            maxAttempts: retryConfig.max_attempts ?? 2,
+            initialDelayMs: retryConfig.initial_delay_ms ?? 1000,
+            maxDelayMs: retryConfig.max_delay_ms ?? 30000,
+            shouldRetry: (error) => isRetryableError(error),
+            onRetry: (error, attempt, delayMs) => {
+                retryCount++;
+                this.logger.debug('Retrying provider request', {
+                    provider,
+                    attempt,
+                    delayMs,
+                    error: error.message,
+                });
+            },
+        };
+        try {
+            const response = await retry(async () => {
+                const startTime = Date.now();
+                try {
+                    // Execute through circuit breaker
+                    const result = await breaker.execute(async () => {
+                        const providerInstance = this.providers.get(provider);
+                        const request = {
+                            model,
+                            messages,
+                            temperature: options.temperature,
+                            max_tokens: options.max_tokens,
+                            timeout_ms: options.timeout_ms,
+                        };
+                        return providerInstance.complete(request);
+                    });
+                    // Record success
+                    const latencyMs = Date.now() - startTime;
+                    this.healthTracker.markSuccess(provider, latencyMs);
+                    return result;
+                }
+                catch (error) {
+                    const latencyMs = Date.now() - startTime;
+                    const err = error instanceof Error ? error : new Error(String(error));
+                    // Extract status code if available
+                    let statusCode;
+                    if (error instanceof ProviderError) {
+                        statusCode = error.statusCode;
+                    }
+                    else if (error instanceof RateLimitError) {
+                        statusCode = 429;
+                    }
+                    else if (error instanceof CircuitOpenError) {
+                        // Circuit is open, don't record as failure
+                        throw error;
+                    }
+                    // Record failure
+                    this.healthTracker.markFailure(provider, err, statusCode);
+                    throw error;
+                }
+            }, retryOptions);
+            return { response, retryCount };
+        }
+        catch (error) {
+            // Final failure after retries
+            throw error;
+        }
+    }
+    /**
+     * Execute without failover (single provider only).
+     */
+    async executeSingleProvider(role, messages, options) {
+        const startTime = Date.now();
+        const roleConfig = this.appConfig.roles[role];
+        if (!roleConfig) {
+            throw new Error(`Unknown role: ${role}`);
+        }
+        const providerInstance = this.providers.get(roleConfig.provider);
+        const request = {
+            model: roleConfig.model,
+            messages,
+            temperature: options.temperature,
+            max_tokens: options.max_tokens,
+            timeout_ms: options.timeout_ms,
+        };
+        const response = await providerInstance.complete(request);
+        const latencyMs = Date.now() - startTime;
+        return {
+            response,
+            provider: roleConfig.provider,
+            model: roleConfig.model,
+            retryCount: 0,
+            failoverCount: 0,
+            latencyMs,
+            usedFailover: false,
+        };
+    }
+    /**
+     * Check if an error should trigger failover.
+     */
+    shouldTriggerFailover(error, failoverErrors) {
+        // Rate limit always triggers failover
+        if (error instanceof RateLimitError) {
+            return true;
+        }
+        // Timeout triggers failover
+        if (error instanceof TimeoutError) {
+            return true;
+        }
+        // Circuit open triggers failover
+        if (error instanceof CircuitOpenError) {
+            return true;
+        }
+        // Check status code
+        if (error instanceof ProviderError && error.statusCode !== undefined) {
+            return failoverErrors.includes(error.statusCode);
+        }
+        return false;
+    }
+    /**
+     * Get the next provider in the chain that hasn't been tried.
+     */
+    getNextProvider(chain, attempted) {
+        for (const p of chain) {
+            if (!attempted.includes(p.provider)) {
+                return p;
+            }
+        }
+        return undefined;
+    }
+    /**
+     * Get or create a circuit breaker for a provider.
+     */
+    getOrCreateCircuitBreaker(provider) {
+        let breaker = this.circuitBreakers.get(provider);
+        if (!breaker) {
+            breaker = new CircuitBreaker(this.config.circuitBreaker);
+            // Log state changes
+            breaker.onStateChange((prev, next, meta) => {
+                this.logger.info('Circuit breaker state change', {
+                    provider,
+                    previousState: prev,
+                    newState: next,
+                    failureCount: meta.failureCount,
+                });
+            });
+            this.circuitBreakers.set(provider, breaker);
+        }
+        return breaker;
+    }
+    /**
+     * Record a failover event for diagnostics.
+     */
+    recordFailoverEvent(role, fromProvider, toProvider, error) {
+        const event = {
+            timestamp: new Date(),
+            role,
+            fromProvider,
+            toProvider,
+            reason: error.message,
+        };
+        if (error instanceof ProviderError) {
+            event.errorCode = error.statusCode;
+        }
+        else if (error instanceof RateLimitError) {
+            event.errorCode = 429;
+        }
+        event.errorMessage = error.message;
+        this.failoverEvents.push(event);
+        // Trim history
+        while (this.failoverEvents.length > this.maxEventHistory) {
+            this.failoverEvents.shift();
+        }
+    }
+}
+//# sourceMappingURL=orchestrator.js.map