claude-flow-novice 2.0.3 → 2.0.4

package/dist/src/swarm/error-recovery/automated-recovery-workflows.js

@@ -0,0 +1,998 @@
+/**
+ * Automated Recovery Workflows
+ * Multi-strategy recovery approaches with context-aware decision making
+ */ function _define_property(obj, key, value) {
+    if (key in obj) {
+        Object.defineProperty(obj, key, {
+            value: value,
+            enumerable: true,
+            configurable: true,
+            writable: true
+        });
+    } else {
+        obj[key] = value;
+    }
+    return obj;
+}
+import { EventEmitter } from "node:events";
+import { createClient } from "redis";
+export class AutomatedRecoveryWorkflows extends EventEmitter {
+    async start() {
+        if (this.isRunning) {
+            return;
+        }
+        try {
+            await this.redis.connect();
+            await this.loadHistoricalData();
+            this.logger.info('Automated recovery workflows started', {
+                strategies: this.strategies.size,
+                maxConcurrent: this.config.global.maxConcurrentRecoveries
+            });
+            this.isRunning = true;
+            this.startMonitoring();
+            this.emit('started');
+        } catch (error) {
+            this.logger.error('Failed to start automated recovery workflows', {
+                error
+            });
+            throw error;
+        }
+    }
+    async stop() {
+        if (!this.isRunning) {
+            return;
+        }
+        this.isRunning = false;
+        // Wait for active workflows to complete or timeout
+        const timeoutMs = this.config.global.defaultTimeoutMs;
+        const startTime = Date.now();
+        while(this.activeWorkflows.size > 0 && Date.now() - startTime < timeoutMs){
+            await new Promise((resolve)=>setTimeout(resolve, 1000));
+        }
+        // Force stop any remaining workflows
+        for (const workflow of this.activeWorkflows.values()){
+            await this.stopWorkflow(workflow.id, 'System shutdown');
+        }
+        await this.saveHistoricalData();
+        await this.redis.disconnect();
+        this.emit('stopped');
+        this.logger.info('Automated recovery workflows stopped');
+    }
+    initializeStrategies() {
+        // Load configured strategies
+        this.config.strategies.forEach((strategy)=>{
+            this.strategies.set(strategy.id, strategy);
+        });
+        // Add default strategies if none provided
+        if (this.strategies.size === 0) {
+            this.addDefaultStrategies();
+        }
+    }
+    addDefaultStrategies() {
+        const defaultStrategies = [
+            {
+                id: 'agent-restart',
+                name: 'Agent Restart Strategy',
+                description: 'Restarts failed or unhealthy agents',
+                type: 'standard',
+                conditions: {
+                    errorTypes: [
+                        'agent_timeout',
+                        'agent_failure',
+                        'agent_unresponsive'
+                    ],
+                    severity: [
+                        'low',
+                        'medium'
+                    ],
+                    categories: [
+                        'agent'
+                    ]
+                },
+                actions: [
+                    {
+                        id: 'stop-agent',
+                        type: 'restart',
+                        target: 'agent',
+                        parameters: {
+                            graceful: true
+                        },
+                        timeoutMs: 30000,
+                        rollbackAction: {
+                            id: 'keep-agent-stopped',
+                            type: 'custom',
+                            target: 'agent',
+                            parameters: {
+                                action: 'no_op'
+                            },
+                            timeoutMs: 1000
+                        }
+                    },
+                    {
+                        id: 'verify-agent-health',
+                        type: 'custom',
+                        target: 'agent',
+                        parameters: {
+                            action: 'health_check'
+                        },
+                        timeoutMs: 10000
+                    }
+                ],
+                priority: 1,
+                maxRetries: 3,
+                cooldownMs: 60000,
+                successThreshold: 0.8
+            },
+            {
+                id: 'task-retry',
+                name: 'Task Retry Strategy',
+                description: 'Retries failed tasks with exponential backoff',
+                type: 'standard',
+                conditions: {
+                    errorTypes: [
+                        'task_failure',
+                        'task_timeout'
+                    ],
+                    severity: [
+                        'low',
+                        'medium'
+                    ],
+                    categories: [
+                        'task'
+                    ]
+                },
+                actions: [
+                    {
+                        id: 'retry-task',
+                        type: 'retry',
+                        target: 'task',
+                        parameters: {
+                            backoffMultiplier: 2,
+                            maxDelay: 30000,
+                            jitter: true
+                        },
+                        timeoutMs: 60000
+                    }
+                ],
+                priority: 2,
+                maxRetries: 5,
+                cooldownMs: 5000,
+                successThreshold: 0.7
+            },
+            {
+                id: 'memory-cleanup',
+                name: 'Memory Cleanup Strategy',
+                description: 'Performs memory cleanup and optimization',
+                type: 'conservative',
+                conditions: {
+                    errorTypes: [
+                        'memory_exhaustion',
+                        'memory_leak'
+                    ],
+                    severity: [
+                        'medium',
+                        'high'
+                    ],
+                    categories: [
+                        'memory',
+                        'system'
+                    ]
+                },
+                actions: [
+                    {
+                        id: 'garbage-collect',
+                        type: 'custom',
+                        target: 'system',
+                        parameters: {
+                            action: 'force_gc'
+                        },
+                        timeoutMs: 10000
+                    },
+                    {
+                        id: 'clear-cache',
+                        type: 'custom',
+                        target: 'system',
+                        parameters: {
+                            action: 'clear_caches'
+                        },
+                        timeoutMs: 5000
+                    },
+                    {
+                        id: 'restart-memory-intensive-agents',
+                        type: 'restart',
+                        target: 'memory_intensive_agents',
+                        parameters: {
+                            graceful: false
+                        },
+                        timeoutMs: 30000
+                    }
+                ],
+                priority: 3,
+                maxRetries: 2,
+                cooldownMs: 120000,
+                successThreshold: 0.9
+            },
+            {
+                id: 'circuit-breaker',
+                name: 'Circuit Breaker Strategy',
+                description: 'Activates circuit breaker to prevent cascading failures',
+                type: 'aggressive',
+                conditions: {
+                    errorTypes: [
+                        'cascading_failure',
+                        'network_partition'
+                    ],
+                    severity: [
+                        'high',
+                        'critical'
+                    ],
+                    categories: [
+                        'network',
+                        'system'
+                    ]
+                },
+                actions: [
+                    {
+                        id: 'activate-circuit-breaker',
+                        type: 'failover',
+                        target: 'system',
+                        parameters: {
+                            mode: 'open',
+                            timeout: 60000,
+                            halfOpenRetries: 3
+                        },
+                        timeoutMs: 5000
+                    },
+                    {
+                        id: 'enable-fallback-mode',
+                        type: 'failover',
+                        target: 'system',
+                        parameters: {
+                            mode: 'degraded'
+                        },
+                        timeoutMs: 10000
+                    }
+                ],
+                priority: 4,
+                maxRetries: 1,
+                cooldownMs: 300000,
+                successThreshold: 0.95
+            },
+            {
+                id: 'emergency-shutdown',
+                name: 'Emergency Shutdown Strategy',
+                description: 'Performs emergency shutdown to prevent data loss',
+                type: 'emergency',
+                conditions: {
+                    errorTypes: [
+                        'critical_system_failure',
+                        'data_corruption'
+                    ],
+                    severity: [
+                        'critical'
+                    ],
+                    categories: [
+                        'system'
+                    ]
+                },
+                actions: [
+                    {
+                        id: 'graceful-shutdown',
+                        type: 'custom',
+                        target: 'system',
+                        parameters: {
+                            saveState: true,
+                            timeout: 30000,
+                            forceAfterTimeout: true
+                        },
+                        timeoutMs: 45000
+                    },
+                    {
+                        id: 'activate-emergency-procedures',
+                        type: 'custom',
+                        target: 'system',
+                        parameters: {
+                            notifyAdmin: true,
+                            createBackup: true,
+                            lockSystem: true
+                        },
+                        timeoutMs: 60000
+                    }
+                ],
+                priority: 5,
+                maxRetries: 0,
+                cooldownMs: 0,
+                successThreshold: 1.0
+            }
+        ];
+        defaultStrategies.forEach((strategy)=>{
+            this.strategies.set(strategy.id, strategy);
+        });
+    }
+    async handleError(error) {
+        const context = await this.buildRecoveryContext(error);
+        const strategy = await this.selectBestStrategy(context);
+        if (!strategy) {
+            this.logger.warn('No recovery strategy found for error', {
+                errorId: error.id,
+                type: error.type
+            });
+            throw new Error('No suitable recovery strategy available');
+        }
+        const workflow = await this.createRecoveryWorkflow(strategy, context);
+        await this.executeWorkflow(workflow);
+        return workflow.id;
+    }
+    async buildRecoveryContext(error) {
+        const previousAttempts = await this.getPreviousAttempts(error);
+        const systemLoad = await this.getSystemLoad();
+        const swarmState = await this.getSwarmState();
+        return {
+            errorId: error.id,
+            errorType: error.type,
+            severity: error.severity,
+            category: error.category,
+            source: error.source,
+            affectedComponents: this.extractAffectedComponents(error),
+            swarmState,
+            previousAttempts,
+            systemLoad,
+            timeConstraints: await this.getTimeConstraints()
+        };
+    }
+    async selectBestStrategy(context) {
+        const candidates = [];
+        for (const strategy of this.strategies.values()){
+            const score = await this.scoreStrategy(strategy, context);
+            if (score > 0) {
+                candidates.push({
+                    strategy,
+                    score
+                });
+            }
+        }
+        if (candidates.length === 0) {
+            return null;
+        }
+        // Sort by score and apply safety checks
+        candidates.sort((a, b)=>b.score - a.score);
+        for (const { strategy } of candidates){
+            if (await this.safetyMonitor.isSafeToExecute(strategy, context)) {
+                return strategy;
+            }
+        }
+        return null; // No strategy is safe to execute
+    }
+    async scoreStrategy(strategy, context) {
+        let score = 0;
+        // Check basic conditions
+        if (!this.matchesConditions(strategy, context)) {
+            return 0;
+        }
+        // Base score from conditions match
+        score += 50;
+        // Adjust for severity appropriateness
+        const severityScore = this.getSeverityScore(strategy.type, context.severity);
+        score += severityScore;
+        // Adjust for previous success rate
+        const successRate = await this.learningEngine.getStrategySuccessRate(strategy.id);
+        score += successRate * 20;
+        // Adjust for system load
+        const loadScore = this.getLoadScore(strategy, context.systemLoad);
+        score += loadScore;
+        // Adjust for timing constraints
+        const timingScore = this.getTimingScore(strategy, context.timeConstraints);
+        score += timingScore;
+        // Apply confidence factors
+        score *= this.getConfidenceFactor(strategy, context);
+        return Math.min(score, 100);
+    }
+    matchesConditions(strategy, context) {
+        const { conditions } = strategy;
+        // Check error types
+        if (conditions.errorTypes.length > 0 && !conditions.errorTypes.includes(context.errorType)) {
+            return false;
+        }
+        // Check severity
+        if (conditions.severity.length > 0 && !conditions.severity.includes(context.severity)) {
+            return false;
+        }
+        // Check categories
+        if (conditions.categories.length > 0 && !conditions.categories.includes(context.category)) {
+            return false;
+        }
+        return true;
+    }
+    getSeverityScore(strategyType, severity) {
+        const mapping = {
+            conservative: {
+                low: 20,
+                medium: 15,
+                high: 5,
+                critical: 0
+            },
+            standard: {
+                low: 15,
+                medium: 20,
+                high: 15,
+                critical: 5
+            },
+            aggressive: {
+                low: 5,
+                medium: 15,
+                high: 20,
+                critical: 15
+            },
+            emergency: {
+                low: 0,
+                medium: 5,
+                high: 15,
+                critical: 25
+            }
+        };
+        return mapping[strategyType]?.[severity] || 0;
+    }
+    getLoadScore(strategy, systemLoad) {
+        // Heavy strategies get lower scores under high load
+        if (strategy.type === 'aggressive' || strategy.type === 'emergency') {
+            if (systemLoad.cpu > 80 || systemLoad.memory > 80) {
+                return -10;
+            }
+        }
+        // Conservative strategies preferred under high load
+        if (strategy.type === 'conservative' && (systemLoad.cpu > 70 || systemLoad.memory > 70)) {
+            return 10;
+        }
+        return 0;
+    }
+    getTimingScore(strategy, timeConstraints) {
+        // Emergency strategies get higher score during critical times
+        if (strategy.type === 'emergency' && timeConstraints.maxDowntime < 60000) {
+            return 15;
+        }
+        // Conservative strategies preferred during business hours
+        if (strategy.type === 'conservative' && timeConstraints.businessHours) {
+            return 10;
+        }
+        return 0;
+    }
+    getConfidenceFactor(strategy, context) {
+        let factor = 1.0;
+        // Reduce confidence for previous failed attempts
+        const recentFailures = context.previousAttempts.filter((attempt)=>attempt.strategyId === strategy.id && Date.now() - attempt.timestamp.getTime() < 300000);
+        factor *= Math.pow(0.5, recentFailures.length);
+        // Adjust for cooldown period
+        const lastAttempt = context.previousAttempts.filter((attempt)=>attempt.strategyId === strategy.id).sort((a, b)=>b.timestamp.getTime() - a.timestamp.getTime())[0];
+        if (lastAttempt && Date.now() - lastAttempt.timestamp.getTime() < strategy.cooldownMs) {
+            factor *= 0.3;
+        }
+        return Math.max(factor, 0.1);
+    }
+    async createRecoveryWorkflow(strategy, context) {
+        const workflow = {
+            id: `workflow-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+            name: strategy.name,
+            description: strategy.description,
+            strategy,
+            context,
+            attempts: [],
+            status: 'created',
+            createdAt: new Date(),
+            updatedAt: new Date(),
+            estimatedDuration: this.estimateWorkflowDuration(strategy),
+            success: false
+        };
+        this.activeWorkflows.set(workflow.id, workflow);
+        // Store in Redis
+        await this.redis.setEx(`swarm:error-recovery-final:workflows:${workflow.id}`, 3600, JSON.stringify(workflow));
+        this.emit('workflowCreated', workflow);
+        return workflow;
+    }
+    async executeWorkflow(workflow) {
+        workflow.status = 'running';
+        workflow.updatedAt = new Date();
+        this.logger.info('Executing recovery workflow', {
+            workflowId: workflow.id,
+            strategy: workflow.strategy.id,
+            errorId: workflow.context.errorId
+        });
+        this.emit('workflowStarted', workflow);
+        try {
+            const attempt = await this.createAttempt(workflow);
+            const success = await this.executeAttempt(workflow, attempt);
+            workflow.attempts.push(attempt);
+            workflow.success = success;
+            workflow.status = success ? 'completed' : 'failed';
+            workflow.actualDuration = Date.now() - workflow.createdAt.getTime();
+            workflow.updatedAt = new Date();
+            // Update in Redis
+            await this.redis.setEx(`swarm:error-recovery-final:workflows:${workflow.id}`, 3600, JSON.stringify(workflow));
+            // Publish result
+            await this.redis.publish('swarm:error-recovery-final', JSON.stringify({
+                type: success ? 'RECOVERY_SUCCESS' : 'RECOVERY_FAILED',
+                workflowId: workflow.id,
+                timestamp: new Date().toISOString()
+            }));
+            this.emit(success ? 'workflowCompleted' : 'workflowFailed', workflow);
+            // Move to completed workflows
+            this.activeWorkflows.delete(workflow.id);
+            this.completedWorkflows.push(workflow);
+            // Learn from the result
+            await this.learningEngine.recordWorkflowResult(workflow);
+        } catch (error) {
+            workflow.status = 'failed';
+            workflow.updatedAt = new Date();
+            this.logger.error('Workflow execution failed', {
+                workflowId: workflow.id,
+                error: error instanceof Error ? error.message : String(error)
+            });
+            this.emit('workflowFailed', workflow);
+        }
+    }
+    async createAttempt(workflow) {
+        const attempt = {
+            id: `attempt-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+            strategyId: workflow.strategy.id,
+            timestamp: new Date(),
+            actions: [
+                ...workflow.strategy.actions
+            ],
+            status: 'pending',
+            duration: 0,
+            confidence: await this.calculateAttemptConfidence(workflow)
+        };
+        return attempt;
+    }
+    async executeAttempt(workflow, attempt) {
+        attempt.status = 'running';
+        const startTime = Date.now();
+        this.logger.info('Executing recovery attempt', {
+            workflowId: workflow.id,
+            attemptId: attempt.id,
+            actions: attempt.actions.length
+        });
+        try {
+            for (const action of attempt.actions){
+                const actionSuccess = await this.executeAction(workflow, action);
+                if (!actionSuccess) {
+                    await this.rollbackAction(workflow, action);
+                    attempt.status = 'failed';
+                    attempt.error = `Action ${action.id} failed`;
+                    return false;
+                }
+            }
+            attempt.status = 'completed';
+            attempt.duration = Date.now() - startTime;
+            // Verify recovery success
+            const verificationResult = await this.verifyRecovery(workflow);
+            attempt.result = verificationResult;
+            return verificationResult.success;
+        } catch (error) {
+            attempt.status = 'failed';
+            attempt.error = error instanceof Error ? error.message : String(error);
+            attempt.duration = Date.now() - startTime;
+            return false;
+        }
+    }
+    async executeAction(workflow, action) {
+        this.logger.info('Executing recovery action', {
+            workflowId: workflow.id,
+            actionId: action.id,
+            type: action.type,
+            target: action.target
+        });
+        try {
+            switch(action.type){
+                case 'restart':
+                    return await this.executeRestartAction(workflow, action);
+                case 'scale':
+                    return await this.executeScaleAction(workflow, action);
+                case 'rollback':
+                    return await this.executeRollbackAction(workflow, action);
+                case 'retry':
+                    return await this.executeRetryAction(workflow, action);
+                case 'failover':
+                    return await this.executeFailoverAction(workflow, action);
+                case 'mitigate':
+                    return await this.executeMitigateAction(workflow, action);
+                case 'custom':
+                    return await this.executeCustomAction(workflow, action);
+                default:
+                    this.logger.warn('Unknown action type', {
+                        type: action.type
+                    });
+                    return false;
+            }
+        } catch (error) {
+            this.logger.error('Action execution failed', {
+                workflowId: workflow.id,
+                actionId: action.id,
+                error: error instanceof Error ? error.message : String(error)
+            });
+            return false;
+        }
+    }
+    async executeRestartAction(workflow, action) {
+        const { target, parameters } = action;
+        if (target === 'agent') {
+            return await this.restartAgent(parameters.agentId, parameters.graceful);
+        } else if (target === 'service') {
+            return await this.restartService(parameters.serviceId, parameters.graceful);
+        }
+        return false;
+    }
+    async executeScaleAction(workflow, action) {
+        const { target, parameters } = action;
+        if (target === 'agents') {
+            return await this.scaleAgents(parameters.direction, parameters.count);
+        } else if (target === 'resources') {
+            return await this.scaleResources(parameters.resource, parameters.amount);
+        }
+        return false;
+    }
+    async executeRollbackAction(workflow, action) {
+        const { target, parameters } = action;
+        // Implementation would rollback changes
+        this.logger.info('Rolling back changes', {
+            target,
+            parameters
+        });
+        return true;
+    }
+    async executeRetryAction(workflow, action) {
+        const { target, parameters } = action;
+        if (target === 'task') {
+            return await this.retryTask(parameters.taskId, parameters);
+        }
+        return false;
+    }
+    async executeFailoverAction(workflow, action) {
+        const { target, parameters } = action;
+        if (target === 'system') {
+            return await this.activateFailover(parameters.mode);
+        }
+        return false;
+    }
+    async executeMitigateAction(workflow, action) {
+        const { target, parameters } = action;
+        // Implementation would apply mitigation measures
+        this.logger.info('Applying mitigation', {
+            target,
+            parameters
+        });
+        return true;
+    }
+    async executeCustomAction(workflow, action) {
+        const { target, parameters } = action;
+        // Implementation would execute custom action
+        this.logger.info('Executing custom action', {
+            target,
+            parameters
+        });
+        return true;
+    }
+    async rollbackAction(workflow, action) {
+        if (action.rollbackAction) {
+            this.logger.info('Rolling back action', {
+                workflowId: workflow.id,
+                actionId: action.id
+            });
+            await this.executeAction(workflow, action.rollbackAction);
+        }
+    }
+    async verifyRecovery(workflow) {
+        // Check if the original error is resolved
+        const errorResolved = await this.checkErrorResolution(workflow.context.errorId);
+        // Check system health
+        const systemHealth = await this.checkSystemHealth();
+        // Check affected components
+        const componentHealth = await this.checkComponentHealth(workflow.context.affectedComponents);
+        const success = errorResolved && systemHealth.overall > 0.8 && componentHealth.every((c)=>c.health > 0.7);
+        return {
+            success,
+            details: {
+                errorResolved,
+                systemHealth,
+                componentHealth
+            }
+        };
+    }
+    // Action implementations (simplified)
+    async restartAgent(agentId, graceful) {
+        this.logger.info('Restarting agent', {
+            agentId,
+            graceful
+        });
+        // Implementation would restart the agent
+        return true;
+    }
+    async restartService(serviceId, graceful) {
+        this.logger.info('Restarting service', {
+            serviceId,
+            graceful
+        });
+        // Implementation would restart the service
+        return true;
+    }
+    async scaleAgents(direction, count) {
+        this.logger.info('Scaling agents', {
+            direction,
+            count
+        });
+        // Implementation would scale agents
+        return true;
+    }
+    async scaleResources(resource, amount) {
+        this.logger.info('Scaling resources', {
+            resource,
+            amount
+        });
+        // Implementation would scale resources
+        return true;
+    }
+    async retryTask(taskId, parameters) {
+        this.logger.info('Retrying task', {
+            taskId,
+            parameters
+        });
+        // Implementation would retry the task
+        return true;
+    }
+    async activateFailover(mode) {
+        this.logger.info('Activating failover', {
+            mode
+        });
+        // Implementation would activate failover mode
+        return true;
+    }
+    // Helper methods
+    async calculateAttemptConfidence(workflow) {
+        const strategySuccessRate = await this.learningEngine.getStrategySuccessRate(workflow.strategy.id);
+        const contextConfidence = this.getContextConfidence(workflow.context);
+        return (strategySuccessRate + contextConfidence) / 2;
+    }
+    getContextConfidence(context) {
+        let confidence = 0.5;
+        // Adjust based on system load
+        if (context.systemLoad.cpu < 50 && context.systemLoad.memory < 50) {
+            confidence += 0.2;
+        } else if (context.systemLoad.cpu > 80 || context.systemLoad.memory > 80) {
+            confidence -= 0.2;
+        }
+        // Adjust based on previous attempts
+        const recentFailures = context.previousAttempts.filter((attempt)=>Date.now() - attempt.timestamp.getTime() < 300000);
+        confidence -= recentFailures.length * 0.1;
+        return Math.max(Math.min(confidence, 1.0), 0.1);
+    }
+    estimateWorkflowDuration(strategy) {
+        return strategy.actions.reduce((total, action)=>total + action.timeoutMs, 0);
+    }
+    extractAffectedComponents(error) {
+        if ('affectedComponents' in error) {
+            return error.affectedComponents;
+        }
+        return [
+            error.source
+        ];
+    }
+    async getPreviousAttempts(error) {
+        // Implementation would load previous attempts from Redis
+        return [];
+    }
+    async getSystemLoad() {
+        return {
+            cpu: Math.random() * 100,
+            memory: Math.random() * 100,
+            activeTasks: Math.floor(Math.random() * 100)
+        };
+    }
+    async getSwarmState() {
+        // Implementation would get current swarm state
+        return {};
+    }
+    async getTimeConstraints() {
+        const now = new Date();
+        const businessHours = now.getHours() >= 9 && now.getHours() <= 17;
+        return {
+            maxDowntime: 300000,
+            businessHours
+        };
+    }
+    async checkErrorResolution(errorId) {
+        // Implementation would check if the original error is resolved
+        return true;
+    }
+    async checkSystemHealth() {
+        return {
+            overall: Math.random(),
+            details: {}
+        };
+    }
+    async checkComponentHealth(components) {
+        return components.map((component)=>({
+                component,
+                health: Math.random()
+            }));
+    }
+    startMonitoring() {
+        setInterval(async ()=>{
+            try {
+                await this.cleanupOldWorkflows();
+                await this.updateMetrics();
+            } catch (error) {
+                this.logger.error('Error in recovery monitoring', {
+                    error
+                });
+            }
+        }, this.config.global.monitoringIntervalMs);
+    }
+    async cleanupOldWorkflows() {
+        const cutoff = Date.now() - this.config.learning.historyRetentionDays * 24 * 60 * 60 * 1000;
+        this.completedWorkflows = this.completedWorkflows.filter((workflow)=>workflow.createdAt.getTime() > cutoff);
+        // Keep only last 1000 workflows in memory
+        if (this.completedWorkflows.length > 1000) {
+            this.completedWorkflows = this.completedWorkflows.slice(-1000);
+        }
+    }
+    async updateMetrics() {
+        const metrics = {
+            activeWorkflows: this.activeWorkflows.size,
+            completedWorkflows: this.completedWorkflows.length,
+            strategies: this.strategies.size,
+            timestamp: new Date().toISOString()
+        };
+        await this.redis.setEx('swarm:error-recovery-final:recovery-metrics', 300, JSON.stringify(metrics));
+    }
+    async loadHistoricalData() {
+    // Implementation would load historical workflow data from Redis
+    }
+    async saveHistoricalData() {
+    // Implementation would save workflow history to Redis
+    }
+    async stopWorkflow(workflowId, reason) {
+        const workflow = this.activeWorkflows.get(workflowId);
+        if (!workflow) return;
+        workflow.status = 'failed';
+        workflow.updatedAt = new Date();
+        this.logger.info('Stopping workflow', {
+            workflowId,
+            reason
+        });
+        this.emit('workflowStopped', {
+            workflowId,
+            reason
+        });
+    }
+    // Public API methods
+    async getWorkflow(workflowId) {
+        const workflow = this.activeWorkflows.get(workflowId);
+        if (workflow) return workflow;
+        const completed = this.completedWorkflows.find((w)=>w.id === workflowId);
+        if (completed) return completed;
+        // Try loading from Redis
+        const data = await this.redis.get(`swarm:error-recovery-final:workflows:${workflowId}`);
+        return data ? JSON.parse(data) : null;
+    }
+    async getActiveWorkflows() {
+        return Array.from(this.activeWorkflows.values());
+    }
+    async getCompletedWorkflows(limit = 50) {
+        return this.completedWorkflows.sort((a, b)=>b.createdAt.getTime() - a.createdAt.getTime()).slice(0, limit);
+    }
+    async addStrategy(strategy) {
+        this.strategies.set(strategy.id, strategy);
+        this.logger.info('Recovery strategy added', {
+            id: strategy.id,
+            name: strategy.name
+        });
+    }
+    async removeStrategy(strategyId) {
+        this.strategies.delete(strategyId);
+        this.logger.info('Recovery strategy removed', {
+            id: strategyId
+        });
+    }
+    async getStrategySuccessRate(strategyId) {
+        return await this.learningEngine.getStrategySuccessRate(strategyId);
+    }
+    constructor(logger, config){
+        super(), _define_property(this, "redis", void 0), _define_property(this, "logger", void 0), _define_property(this, "config", void 0), _define_property(this, "isRunning", false), _define_property(this, "activeWorkflows", new Map()), _define_property(this, "completedWorkflows", []), _define_property(this, "strategies", new Map()), _define_property(this, "learningEngine", void 0), _define_property(this, "safetyMonitor", void 0);
+        this.logger = logger;
+        this.config = config;
+        this.redis = createClient(config.redis);
+        this.learningEngine = new RecoveryLearningEngine(logger, config.learning);
+        this.safetyMonitor = new RecoverySafetyMonitor(logger, config.safety);
+        this.initializeStrategies();
+    }
+}
+let RecoveryLearningEngine = class RecoveryLearningEngine {
+    async recordWorkflowResult(workflow) {
+        if (!this.config.enabled) return;
+        const strategyId = workflow.strategy.id;
+        const stats = this.strategyStats.get(strategyId) || {
+            attempts: 0,
+            successes: 0
+        };
+        stats.attempts++;
+        if (workflow.success) {
+            stats.successes++;
+        }
+        this.strategyStats.set(strategyId, stats);
+        // Adapt strategies if enabled
+        if (this.config.adaptStrategies) {
+            await this.adaptStrategy(strategyId, workflow);
+        }
+        this.logger.debug('Recorded workflow result', {
+            strategyId,
+            success: workflow.success,
+            successRate: stats.successes / stats.attempts
+        });
+    }
+    async getStrategySuccessRate(strategyId) {
+        const stats = this.strategyStats.get(strategyId);
+        return stats ? stats.successes / stats.attempts : 0.5; // Default 50% for new strategies
+    }
+    async adaptStrategy(strategyId, workflow) {
+        // Implementation would adapt strategy parameters based on results
+        this.logger.info('Adapting strategy based on results', {
+            strategyId
+        });
+    }
+    constructor(logger, config){
+        _define_property(this, "logger", void 0);
+        _define_property(this, "config", void 0);
+        _define_property(this, "strategyStats", new Map());
+        this.logger = logger;
+        this.config = config;
+    }
+};
+let RecoverySafetyMonitor = class RecoverySafetyMonitor {
+    async isSafeToExecute(strategy, context) {
+        // Check for cascading failures
+        if (this.cascadingFailures >= this.config.maxCascadingFailures) {
+            this.logger.warn('Cascading failure limit reached', {
+                count: this.cascadingFailures,
+                limit: this.config.maxCascadingFailures
+            });
+            return false;
+        }
+        // Check emergency mode triggers
+        if (this.config.emergencyMode.enabled) {
+            const shouldTriggerEmergency = this.config.emergencyMode.triggers.includes(context.errorType);
+            if (shouldTriggerEmergency && strategy.type !== 'emergency') {
+                return false; // Only emergency strategies allowed
+            }
+        }
+        // Check quarantining
+        if (this.config.quarantining.enabled) {
+            if (await this.isComponentQuarantined(context.source)) {
+                this.logger.warn('Component is quarantined', {
+                    component: context.source
+                });
+                return false;
+            }
+        }
+        return true;
+    }
+    async recordCascadingFailure() {
+        const now = Date.now();
+        if (now - this.lastFailureTime < 60000) {
+            this.cascadingFailures++;
+        } else {
+            this.cascadingFailures = 1;
+        }
+        this.lastFailureTime = now;
+    }
+    async isComponentQuarantined(component) {
+        // Implementation would check Redis for quarantined components
+        return false;
+    }
+    constructor(logger, config){
+        _define_property(this, "logger", void 0);
+        _define_property(this, "config", void 0);
+        _define_property(this, "cascadingFailures", 0);
+        _define_property(this, "lastFailureTime", 0);
+        this.logger = logger;
+        this.config = config;
+    }
+};