claude-flow-novice 2.0.6 → 2.0.8

package/dist/src/swarm/error-recovery/resilience-architecture.js

@@ -0,0 +1,714 @@
+/**
+ * Resilience Architecture
+ * Circuit breaker patterns, failover mechanisms, and disaster recovery procedures
+ */ function _define_property(obj, key, value) {
+    if (key in obj) {
+        Object.defineProperty(obj, key, {
+            value: value,
+            enumerable: true,
+            configurable: true,
+            writable: true
+        });
+    } else {
+        obj[key] = value;
+    }
+    return obj;
+}
+import { EventEmitter } from "node:events";
+import { createClient } from "redis";
+export var CircuitState = /*#__PURE__*/ function(CircuitState) {
+    CircuitState["CLOSED"] = "closed";
+    CircuitState["OPEN"] = "open";
+    CircuitState["HALF_OPEN"] = "half_open"; // Testing if service has recovered
+    return CircuitState;
+}({});
+export class ResilienceArchitecture extends EventEmitter {
+    async start() {
+        if (this.isRunning) {
+            return;
+        }
+        try {
+            await this.redis.connect();
+            // Initialize circuit breakers
+            for (const cbConfig of this.config.circuitBreakers){
+                const circuitBreaker = new CircuitBreaker(cbConfig, this.logger);
+                this.circuitBreakers.set(cbConfig.name, circuitBreaker);
+                circuitBreaker.on('stateChange', (state)=>{
+                    this.emit('circuitBreakerStateChange', {
+                        name: cbConfig.name,
+                        state
+                    });
+                });
+            }
+            // Initialize failover managers
+            for (const foConfig of this.config.failover){
+                const failoverManager = new FailoverManager(foConfig, this.logger, this.redis);
+                this.failoverManagers.set(foConfig.primary.id, failoverManager);
+                failoverManager.on('failover', (state)=>{
+                    this.emit('failover', state);
+                });
+                failoverManager.on('failback', (state)=>{
+                    this.emit('failback', state);
+                });
+            }
+            // Start subsystems
+            await this.healthMonitor.start();
+            await this.disasterRecovery.start();
+            this.isRunning = true;
+            this.logger.info('Resilience architecture started', {
+                circuitBreakers: this.circuitBreakers.size,
+                failoverManagers: this.failoverManagers.size
+            });
+            this.emit('started');
+        } catch (error) {
+            this.logger.error('Failed to start resilience architecture', {
+                error
+            });
+            throw error;
+        }
+    }
+    async stop() {
+        if (!this.isRunning) {
+            return;
+        }
+        this.isRunning = false;
+        await this.healthMonitor.stop();
+        await this.disasterRecovery.stop();
+        for (const circuitBreaker of this.circuitBreakers.values()){
+            circuitBreaker.destroy();
+        }
+        for (const failoverManager of this.failoverManagers.values()){
+            await failoverManager.stop();
+        }
+        await this.redis.disconnect();
+        this.emit('stopped');
+        this.logger.info('Resilience architecture stopped');
+    }
+    async executeWithResilience(serviceName, operation, options = {}) {
+        const circuitBreaker = this.circuitBreakers.get(serviceName);
+        const failoverManager = this.failoverManagers.get(serviceName);
+        if (!circuitBreaker && !failoverManager) {
+            // No resilience mechanisms configured, execute directly
+            return await this.executeWithTimeout(operation, options.timeout);
+        }
+        // Apply rate limiting
+        await this.rateLimiter.acquire();
+        // Apply bulkhead pattern
+        return await this.bulkheadManager.execute(async ()=>{
+            // Try circuit breaker first
+            if (circuitBreaker) {
+                try {
+                    return await circuitBreaker.execute(operation, options.timeout);
+                } catch (error) {
+                    this.logger.warn('Circuit breaker rejected call', {
+                        serviceName,
+                        error: error instanceof Error ? error.message : String(error)
+                    });
+                    if (options.fallback) {
+                        return await options.fallback();
+                    }
+                    throw error;
+                }
+            }
+            // Try failover manager
+            if (failoverManager) {
+                return await failoverManager.execute(operation, options);
+            }
+            return await this.executeWithTimeout(operation, options.timeout);
+        });
+    }
+    async executeWithTimeout(operation, timeoutMs = 30000) {
+        return new Promise((resolve, reject)=>{
+            const timer = setTimeout(()=>{
+                reject(new Error(`Operation timed out after ${timeoutMs}ms`));
+            }, timeoutMs);
+            operation().then((result)=>{
+                clearTimeout(timer);
+                resolve(result);
+            }).catch((error)=>{
+                clearTimeout(timer);
+                reject(error);
+            });
+        });
+    }
+    async getCircuitBreakerStatus(name) {
+        const circuitBreaker = this.circuitBreakers.get(name);
+        return circuitBreaker ? circuitBreaker.getMetrics() : null;
+    }
+    async getFailoverStatus(serviceId) {
+        const failoverManager = this.failoverManagers.get(serviceId);
+        return failoverManager ? failoverManager.getState() : null;
+    }
+    async triggerEmergencyProcedure(procedureId, context = {}) {
+        return await this.disasterRecovery.executeProcedure(procedureId, context);
+    }
+    async manualFailover(serviceId, targetEndpointId) {
+        const failoverManager = this.failoverManagers.get(serviceId);
+        if (!failoverManager) {
+            throw new Error(`No failover manager found for service: ${serviceId}`);
+        }
+        return await failoverManager.manualFailover(targetEndpointId);
+    }
+    async manualFailback(serviceId) {
+        const failoverManager = this.failoverManagers.get(serviceId);
+        if (!failoverManager) {
+            throw new Error(`No failover manager found for service: ${serviceId}`);
+        }
+        return await failoverManager.manualFailback();
+    }
+    async getHealthStatus() {
+        const statuses = [];
+        for (const [serviceId, failoverManager] of this.failoverManagers){
+            const status = await failoverManager.getHealthStatus();
+            statuses.push({
+                service: serviceId,
+                status
+            });
+        }
+        return statuses;
+    }
+    constructor(logger, config){
+        super(), _define_property(this, "redis", void 0), _define_property(this, "logger", void 0), _define_property(this, "config", void 0), _define_property(this, "isRunning", false), _define_property(this, "circuitBreakers", new Map()), _define_property(this, "failoverManagers", new Map()), _define_property(this, "healthMonitor", void 0), _define_property(this, "disasterRecovery", void 0), _define_property(this, "bulkheadManager", void 0), _define_property(this, "rateLimiter", void 0);
+        this.logger = logger;
+        this.config = config;
+        this.redis = createClient(config.redis);
+        this.healthMonitor = new HealthMonitor(logger, config);
+        this.disasterRecovery = new DisasterRecoveryManager(logger, config.disasterRecovery);
+        this.bulkheadManager = new BulkheadManager(config.bulkheads);
+        this.rateLimiter = new RateLimiter(config.rateLimiting);
+    }
+}
+let CircuitBreaker = class CircuitBreaker extends EventEmitter {
+    async execute(operation, timeoutMs) {
+        const startTime = Date.now();
+        try {
+            if (this.state === "open") {
+                if (this.shouldAttemptReset()) {
+                    this.setState("half_open");
+                    this.halfOpenCalls = 0;
+                } else {
+                    throw new Error(`Circuit breaker is OPEN for ${this.config.name}`);
+                }
+            }
+            if (this.state === "half_open" && this.halfOpenCalls >= this.config.halfOpenMaxCalls) {
+                throw new Error(`Circuit breaker is HALF_OPEN and has exceeded max calls for ${this.config.name}`);
+            }
+            if (this.state === "half_open") {
+                this.halfOpenCalls++;
+            }
+            this.totalCalls++;
+            const result = await this.executeWithTimeout(operation, timeoutMs);
+            const responseTime = Date.now() - startTime;
+            this.recordSuccess(responseTime);
+            return result;
+        } catch (error) {
+            this.recordFailure();
+            throw error;
+        }
+    }
+    async executeWithTimeout(operation, timeoutMs) {
+        const timeout = timeoutMs || 30000;
+        return new Promise((resolve, reject)=>{
+            const timer = setTimeout(()=>{
+                this.recordTimeout();
+                reject(new Error(`Operation timed out after ${timeout}ms`));
+            }, timeout);
+            operation().then((result)=>{
+                clearTimeout(timer);
+                resolve(result);
+            }).catch((error)=>{
+                clearTimeout(timer);
+                reject(error);
+            });
+        });
+    }
+    recordSuccess(responseTime) {
+        this.successes++;
+        this.lastSuccessTime = new Date();
+        this.responseTimes.push(responseTime);
+        // Keep only last 100 response times
+        if (this.responseTimes.length > 100) {
+            this.responseTimes = this.responseTimes.slice(-100);
+        }
+        if (this.state === "half_open") {
+            this.setState("closed");
+            this.failures = 0;
+        }
+        this.logger.debug('Circuit breaker success', {
+            name: this.config.name,
+            state: this.state,
+            successes: this.successes,
+            failures: this.failures
+        });
+    }
+    recordFailure() {
+        this.failures++;
+        this.lastFailureTime = new Date();
+        if (this.state === "half_open") {
+            this.setState("open");
+        } else if (this.shouldOpenCircuit()) {
+            this.setState("open");
+        }
+        this.logger.debug('Circuit breaker failure', {
+            name: this.config.name,
+            state: this.state,
+            failures: this.failures,
+            threshold: this.config.failureThreshold
+        });
+    }
+    recordTimeout() {
+        this.timeouts++;
+        this.lastFailureTime = new Date();
+        this.recordFailure();
+    }
+    shouldOpenCircuit() {
+        return this.failures >= this.config.failureThreshold;
+    }
+    shouldAttemptReset() {
+        return Date.now() - this.stateChangedAt.getTime() >= this.config.recoveryTimeout;
+    }
+    setState(newState) {
+        const oldState = this.state;
+        this.state = newState;
+        this.stateChangedAt = new Date();
+        this.logger.info('Circuit breaker state changed', {
+            name: this.config.name,
+            from: oldState,
+            to: newState
+        });
+        this.emit('stateChange', newState);
+        if (this.config.stateChangeHandler) {
+            this.config.stateChangeHandler(newState);
+        }
+    }
+    getMetrics() {
+        return {
+            state: this.state,
+            failures: this.failures,
+            successes: this.successes,
+            timeouts: this.timeouts,
+            totalCalls: this.totalCalls,
+            failureRate: this.totalCalls > 0 ? this.failures / this.totalCalls : 0,
+            lastFailureTime: this.lastFailureTime,
+            lastSuccessTime: this.lastSuccessTime,
+            stateChangedAt: this.stateChangedAt,
+            averageResponseTime: this.responseTimes.length > 0 ? this.responseTimes.reduce((sum, time)=>sum + time, 0) / this.responseTimes.length : 0
+        };
+    }
+    destroy() {
+        this.removeAllListeners();
+    }
+    constructor(config, logger){
+        super(), _define_property(this, "config", void 0), _define_property(this, "logger", void 0), _define_property(this, "state", "closed"), _define_property(this, "failures", 0), _define_property(this, "successes", 0), _define_property(this, "timeouts", 0), _define_property(this, "totalCalls", 0), _define_property(this, "lastFailureTime", void 0), _define_property(this, "lastSuccessTime", void 0), _define_property(this, "stateChangedAt", new Date()), _define_property(this, "halfOpenCalls", 0), _define_property(this, "responseTimes", []);
+        this.config = config;
+        this.logger = logger;
+    }
+};
+let FailoverManager = class FailoverManager extends EventEmitter {
+    async start() {
+        this.startHealthChecks();
+        this.logger.info('Failover manager started', {
+            primary: this.config.primary.id,
+            secondaryCount: this.config.secondary.length
+        });
+    }
+    async stop() {
+        if (this.healthCheckTimer) {
+            clearInterval(this.healthCheckTimer);
+        }
+        this.logger.info('Failover manager stopped');
+    }
+    async execute(operation, options = {}) {
+        const endpoint = this.state.active;
+        try {
+            const result = await operation();
+            this.recordSuccess(endpoint);
+            return result;
+        } catch (error) {
+            this.recordFailure(endpoint, error);
+            if (this.shouldFailover()) {
+                await this.performFailover();
+                // Retry with new endpoint
+                return await operation();
+            }
+            throw error;
+        }
+    }
+    async manualFailover(targetEndpointId) {
+        let targetEndpoint;
+        if (targetEndpointId) {
+            targetEndpoint = this.config.secondary.find((ep)=>ep.id === targetEndpointId) || this.config.secondary.find((ep)=>ep.id === targetEndpointId);
+            if (!targetEndpoint) {
+                throw new Error(`Secondary endpoint not found: ${targetEndpointId}`);
+            }
+        } else {
+            targetEndpoint = this.selectBestSecondary();
+        }
+        await this.performFailoverTo(targetEndpoint);
+        return true;
+    }
+    async manualFailback() {
+        if (!this.state.inFailover) {
+            return false;
+        }
+        const primaryHealthy = this.healthStatuses.get(this.config.primary.id)?.healthy ?? false;
+        if (!primaryHealthy) {
+            this.logger.warn('Cannot failback to unhealthy primary', {
+                primaryId: this.config.primary.id
+            });
+            return false;
+        }
+        await this.performFailback();
+        return true;
+    }
+    startHealthChecks() {
+        this.healthCheckTimer = setInterval(async ()=>{
+            await this.performHealthChecks();
+        }, this.config.healthCheckInterval);
+        // Initial health check
+        this.performHealthChecks();
+    }
+    async performHealthChecks() {
+        const endpoints = [
+            this.config.primary,
+            ...this.config.secondary
+        ];
+        for (const endpoint of endpoints){
+            try {
+                const startTime = Date.now();
+                const healthy = await this.checkEndpointHealth(endpoint);
+                const responseTime = Date.now() - startTime;
+                this.updateHealthStatus(endpoint, healthy, responseTime);
+            } catch (error) {
+                this.updateHealthStatus(endpoint, false, 0, error instanceof Error ? error.message : String(error));
+            }
+        }
+        await this.evaluateFailoverConditions();
+        await this.evaluateFailbackConditions();
+    }
+    async checkEndpointHealth(endpoint) {
+        // Implementation would perform actual health check
+        // For now, simulate with random success
+        return Math.random() > 0.1; // 90% success rate
+    }
+    updateHealthStatus(endpoint, healthy, responseTime, error) {
+        const current = this.healthStatuses.get(endpoint.id) || {
+            endpoint,
+            healthy: true,
+            responseTime: 0,
+            lastCheck: new Date(),
+            consecutiveFailures: 0,
+            consecutiveSuccesses: 0
+        };
+        if (healthy) {
+            current.consecutiveSuccesses++;
+            current.consecutiveFailures = 0;
+        } else {
+            current.consecutiveFailures++;
+            current.consecutiveSuccesses = 0;
+        }
+        current.healthy = healthy;
+        current.responseTime = responseTime;
+        current.lastCheck = new Date();
+        current.error = error;
+        this.healthStatuses.set(endpoint.id, current);
+    }
+    async evaluateFailoverConditions() {
+        if (this.state.inFailover) {
+            return; // Already in failover mode
+        }
+        const primaryStatus = this.healthStatuses.get(this.config.primary.id);
+        if (!primaryStatus || primaryStatus.healthy) {
+            return; // Primary is healthy
+        }
+        const consecutiveFailures = primaryStatus.consecutiveFailures;
+        if (consecutiveFailures >= this.config.failoverThreshold) {
+            await this.performFailover();
+        }
+    }
+    async evaluateFailbackConditions() {
+        if (!this.state.inFailover || !this.config.automaticFailback) {
+            return;
+        }
+        const primaryStatus = this.healthStatuses.get(this.config.primary.id);
+        if (!primaryStatus || !primaryStatus.healthy) {
+            return; // Primary is not healthy
+        }
+        const consecutiveSuccesses = primaryStatus.consecutiveSuccesses;
+        if (consecutiveSuccesses >= this.config.failbackThreshold) {
+            await this.performFailback();
+        }
+    }
+    shouldFailover() {
+        const activeStatus = this.healthStatuses.get(this.state.active.id);
+        return activeStatus ? !activeStatus.healthy : false;
+    }
+    async performFailover() {
+        const targetEndpoint = this.selectBestSecondary();
+        await this.performFailoverTo(targetEndpoint);
+    }
+    async performFailoverTo(targetEndpoint) {
+        const previousActive = this.state.active;
+        this.state.active = targetEndpoint;
+        this.state.inFailover = true;
+        this.state.failoverCount++;
+        this.state.lastFailover = new Date();
+        this.logger.warn('Failover performed', {
+            from: previousActive.id,
+            to: targetEndpoint.id,
+            failoverCount: this.state.failoverCount
+        });
+        this.emit('failover', {
+            previous: previousActive,
+            current: targetEndpoint,
+            state: this.state
+        });
+        // Store failover state in Redis
+        await this.redis.setEx(`swarm:error-recovery-final:failover:${this.config.primary.id}`, 3600, JSON.stringify(this.state));
+    }
+    async performFailback() {
+        const previousActive = this.state.active;
+        this.state.active = this.config.primary;
+        this.state.inFailover = false;
+        this.state.lastFailback = new Date();
+        this.logger.info('Failback performed', {
+            from: previousActive.id,
+            to: this.config.primary.id,
+            duration: Date.now() - this.state.lastFailover.getTime()
+        });
+        this.emit('failback', {
+            previous: previousActive,
+            current: this.config.primary,
+            state: this.state
+        });
+        // Clear failover state from Redis
+        await this.redis.del(`swarm:error-recovery-final:failover:${this.config.primary.id}`);
+    }
+    selectBestSecondary() {
+        const healthySecondaries = this.config.secondary.filter((ep)=>{
+            const status = this.healthStatuses.get(ep.id);
+            return status && status.healthy;
+        });
+        if (healthySecondaries.length === 0) {
+            // No healthy secondaries, select by priority
+            return this.config.secondary.reduce((best, current)=>current.priority < best.priority ? current : best);
+        }
+        // Select healthy secondary with best response time
+        return healthySecondaries.reduce((best, current)=>{
+            const bestStatus = this.healthStatuses.get(best.id);
+            const currentStatus = this.healthStatuses.get(current.id);
+            return currentStatus.responseTime < bestStatus.responseTime ? current : best;
+        });
+    }
+    recordSuccess(endpoint) {
+        const status = this.healthStatuses.get(endpoint.id);
+        if (status) {
+            status.consecutiveSuccesses++;
+            status.consecutiveFailures = 0;
+        }
+    }
+    recordFailure(endpoint, error) {
+        const status = this.healthStatuses.get(endpoint.id);
+        if (status) {
+            status.consecutiveFailures++;
+            status.consecutiveSuccesses = 0;
+            status.error = error instanceof Error ? error.message : String(error);
+        }
+    }
+    getState() {
+        return {
+            ...this.state
+        };
+    }
+    async getHealthStatus() {
+        return this.healthStatuses.get(this.state.active.id) || {
+            endpoint: this.state.active,
+            healthy: false,
+            responseTime: 0,
+            lastCheck: new Date(),
+            consecutiveFailures: 0,
+            consecutiveSuccesses: 0,
+            error: 'No health status available'
+        };
+    }
+    constructor(config, logger, redis){
+        super(), _define_property(this, "config", void 0), _define_property(this, "logger", void 0), _define_property(this, "redis", void 0), _define_property(this, "state", void 0), _define_property(this, "healthStatuses", new Map()), _define_property(this, "healthCheckTimer", void 0);
+        this.config = config;
+        this.logger = logger;
+        this.redis = redis;
+        this.state = {
+            primary: config.primary,
+            active: config.primary,
+            secondary: [
+                ...config.secondary
+            ],
+            failoverCount: 0,
+            lastFailover: new Date(),
+            inFailover: false,
+            automaticFailbackEnabled: config.automaticFailback
+        };
+        // Initialize health statuses
+        this.updateHealthStatus(config.primary, true, 0);
+        config.secondary.forEach((endpoint)=>{
+            this.updateHealthStatus(endpoint, true, 0);
+        });
+    }
+};
+let HealthMonitor = class HealthMonitor {
+    async start() {
+        this.isRunning = true;
+        this.logger.info('Health monitor started');
+    }
+    async stop() {
+        this.isRunning = false;
+        this.logger.info('Health monitor stopped');
+    }
+    constructor(logger, config){
+        _define_property(this, "logger", void 0);
+        _define_property(this, "config", void 0);
+        _define_property(this, "isRunning", false);
+        this.logger = logger;
+        this.config = config;
+    }
+};
+let DisasterRecoveryManager = class DisasterRecoveryManager {
+    async start() {
+        if (!this.config.enabled) {
+            return;
+        }
+        this.isRunning = true;
+        this.logger.info('Disaster recovery manager started');
+    }
+    async stop() {
+        this.isRunning = false;
+        this.logger.info('Disaster recovery manager stopped');
+    }
+    async executeProcedure(procedureId, context = {}) {
+        const procedure = this.config.emergencyProcedures.find((p)=>p.id === procedureId);
+        if (!procedure) {
+            throw new Error(`Emergency procedure not found: ${procedureId}`);
+        }
+        this.logger.warn('Executing emergency procedure', {
+            id: procedureId,
+            name: procedure.name,
+            severity: procedure.severity
+        });
+        try {
+            for (const step of procedure.steps){
+                await this.executeStep(step, context);
+            }
+            this.logger.info('Emergency procedure completed successfully', {
+                id: procedureId
+            });
+            return true;
+        } catch (error) {
+            this.logger.error('Emergency procedure failed', {
+                id: procedureId,
+                error: error instanceof Error ? error.message : String(error)
+            });
+            return false;
+        }
+    }
+    async executeStep(step, context) {
+        this.logger.debug('Executing emergency step', {
+            id: step.id,
+            name: step.name,
+            action: step.action
+        });
+        // Implementation would execute the actual step
+        // For now, simulate execution
+        await new Promise((resolve)=>setTimeout(resolve, 1000));
+    }
+    constructor(logger, config){
+        _define_property(this, "logger", void 0);
+        _define_property(this, "config", void 0);
+        _define_property(this, "isRunning", false);
+        this.logger = logger;
+        this.config = config;
+    }
+};
+let BulkheadManager = class BulkheadManager {
+    async execute(operation) {
+        if (!this.config.enabled) {
+            return await operation();
+        }
+        return new Promise((resolve, reject)=>{
+            if (this.runningCalls < this.config.maxConcurrentCalls) {
+                this.executeNow(operation, resolve, reject);
+            } else if (this.queue.length < this.config.maxQueueSize) {
+                this.queue.push({
+                    resolve,
+                    reject,
+                    operation
+                });
+            } else {
+                reject(new Error('Bulkhead queue is full'));
+            }
+        });
+    }
+    async executeNow(operation, resolve, reject) {
+        this.runningCalls++;
+        const timeout = setTimeout(()=>{
+            this.runningCalls--;
+            reject(new Error('Bulkhead operation timed out'));
+            this.processQueue();
+        }, this.config.timeout);
+        try {
+            const result = await operation();
+            clearTimeout(timeout);
+            this.runningCalls--;
+            resolve(result);
+            this.processQueue();
+        } catch (error) {
+            clearTimeout(timeout);
+            this.runningCalls--;
+            reject(error);
+            this.processQueue();
+        }
+    }
+    processQueue() {
+        if (this.queue.length === 0 || this.runningCalls >= this.config.maxConcurrentCalls) {
+            return;
+        }
+        const { resolve, reject, operation } = this.queue.shift();
+        this.executeNow(operation, resolve, reject);
+    }
+    constructor(config){
+        _define_property(this, "config", void 0);
+        _define_property(this, "runningCalls", 0);
+        _define_property(this, "queue", []);
+        this.config = config;
+    }
+};
+let RateLimiter = class RateLimiter {
+    async acquire() {
+        if (!this.config.enabled) {
+            return;
+        }
+        this.refillTokens();
+        if (this.tokens >= 1) {
+            this.tokens--;
+            return;
+        }
+        // Wait for token refill
+        const waitTime = Math.ceil(1000 / this.config.requestsPerSecond);
+        await new Promise((resolve)=>setTimeout(resolve, waitTime));
+        await this.acquire();
+    }
+    refillTokens() {
+        const now = Date.now();
+        const timePassed = now - this.lastRefill;
+        const tokensToAdd = Math.floor(timePassed * this.config.requestsPerSecond / 1000);
+        if (tokensToAdd > 0) {
+            this.tokens = Math.min(this.tokens + tokensToAdd, this.config.burstSize);
+            this.lastRefill = now;
+        }
+    }
+    constructor(config){
+        _define_property(this, "config", void 0);
+        _define_property(this, "tokens", 0);
+        _define_property(this, "lastRefill", Date.now());
+        this.config = config;
+        this.tokens = config.requestsPerSecond;
+    }
+};