@fsai-flow/core 0.0.2 → 0.0.4

package/src/lib/RedisLeaderElectionManager.ts

@@ -0,0 +1,334 @@
+import Redis, { RedisOptions } from 'ioredis';
+import { LoggerProxy as Logger } from '@fsai-flow/workflow';
+
+export interface RedisLeaderElectionCallbacks {
+	onStartedLeading: () => void;
+	onStoppedLeading: () => void;
+	onNewLeader?: (identity: string) => void;
+}
+
+export class RedisLeaderElectionManager {
+	private redis: Redis;
+	private isLeader = false;
+	private lockKey: string;
+	private nodeId: string;
+	private lockTTL: number = 30000; // 30 seconds in milliseconds
+	private renewalInterval: number = 10000; // 10 seconds in milliseconds
+	private renewalTimer?: NodeJS.Timeout;
+	private callbacks: RedisLeaderElectionCallbacks;
+
+	constructor(
+		lockKey: string,
+		redisConfig: string | RedisOptions,
+		callbacks: RedisLeaderElectionCallbacks
+	) {
+		this.lockKey = `leader:${lockKey}`;
+		this.nodeId = this.generateNodeId();
+		this.callbacks = callbacks;
+		
+		// Initialize Redis connection
+		if (typeof redisConfig === 'string') {
+			this.redis = new Redis(redisConfig);
+		} else {
+			this.redis = new Redis(redisConfig);
+		}
+
+		// Handle Redis connection events
+		this.redis.on('connect', () => {
+			Logger.info(`Redis connected for leader election: ${this.lockKey}`);
+		});
+
+		this.redis.on('ready', () => {
+			Logger.info(`Redis ready for leader election: ${this.lockKey}`);
+		});
+
+		this.redis.on('error', (error: any) => {
+			Logger.error(`Redis connection error for leader election: ${error.message}`);
+		});
+	}
+
+	/**
+	 * Wait for Redis connection to be ready
+	 */
+	private async waitForRedisConnection(): Promise<void> {
+		return new Promise((resolve, reject) => {
+			if (this.redis.status === 'ready') {
+				Logger.debug(`Redis already ready for ${this.lockKey}`);
+				resolve();
+				return;
+			}
+
+			let isSettled = false;
+			let timeoutId: NodeJS.Timeout | null = null;
+
+			// Cleanup function to prevent memory leaks
+			const cleanup = () => {
+				if (isSettled) return; // Already cleaned up
+				isSettled = true;
+
+				// Clear timeout safely
+				if (timeoutId !== null) {
+					clearTimeout(timeoutId);
+					timeoutId = null;
+				}
+
+				// Remove event listeners safely
+				try {
+					this.redis.off('ready', onReady);
+					this.redis.off('error', onError);
+				} catch (error) {
+					// Ignore cleanup errors - Redis connection might be destroyed
+					Logger.debug(`Cleanup warning for ${this.lockKey}: ${error}`);
+				}
+			};
+
+			const onReady = () => {
+				cleanup();
+				Logger.debug(`Redis connection established for ${this.lockKey}`);
+				resolve();
+			};
+
+			const onError = (error: Error) => {
+				cleanup();
+				Logger.error(`Redis connection failed for ${this.lockKey}: ${error.message}`);
+				reject(error);
+			};
+
+			const onTimeout = () => {
+				cleanup();
+				reject(new Error(`Redis connection timeout after 10 seconds for ${this.lockKey}`));
+			};
+
+			// Set up timeout
+			timeoutId = setTimeout(onTimeout, 10000); // 10 second timeout
+
+			// Set up event listeners
+			this.redis.once('ready', onReady);
+			this.redis.once('error', onError);
+		});
+	}
+
+	/**
+	 * Start the leader election process
+	 */
+	async start(): Promise<void> {
+		Logger.info(`🚀 Starting Redis leader election for ${this.nodeId} on key ${this.lockKey}`);
+		Logger.info(`🔧 Leader election config: TTL=${this.lockTTL}ms, RenewalInterval=${this.renewalInterval}ms`);
+		
+		// Wait for Redis connection to be ready
+		Logger.info(`⏳ ${this.nodeId} waiting for Redis connection...`);
+		await this.waitForRedisConnection();
+		Logger.info(`✅ Redis connection ready for ${this.nodeId}`);
+		
+		// Try to acquire leadership immediately
+		Logger.info(`🎯 ${this.nodeId} making initial leadership attempt...`);
+		await this.tryAcquireLeadership();
+		
+		// Start the renewal/retry loop
+		this.startRenewalLoop();
+		
+		Logger.info(`✅ Leader election process started for ${this.nodeId}`);
+	}
+
+	/**
+	 * Stop the leader election process
+	 */
+	async stop(): Promise<void> {
+		Logger.info(`🛑 Stopping Redis leader election for ${this.nodeId}`);
+		
+		// Stop renewal timer
+		if (this.renewalTimer) {
+			Logger.debug(`⏹️  Stopping leadership monitoring loop for ${this.nodeId}`);
+			clearInterval(this.renewalTimer as NodeJS.Timeout);
+			this.renewalTimer = undefined;
+		}
+
+		// Release leadership if we have it
+		if (this.isLeader) {
+			Logger.info(`👋 ${this.nodeId} releasing leadership voluntarily for ${this.lockKey}`);
+			await this.releaseLock();
+		}
+
+		// Close Redis connection
+		Logger.debug(`🔌 Disconnecting Redis for ${this.nodeId}`);
+		this.redis.disconnect();
+		
+		Logger.info(`✅ Leader election stopped for ${this.nodeId}`);
+	}
+
+	/**
+	 * Check if this node is currently the leader
+	 */
+	getIsLeader(): boolean {
+		return this.isLeader;
+	}
+
+	/**
+	 * Try to acquire leadership
+	 */
+	private async tryAcquireLeadership(): Promise<boolean> {
+		try {
+			// Check if Redis connection is ready before attempting operations
+			if (this.redis.status !== 'ready') {
+				Logger.warn(`Redis not ready for ${this.nodeId}, current status: ${this.redis.status}`);
+				return false;
+			}
+
+			Logger.debug(`Node ${this.nodeId} attempting to acquire leadership for ${this.lockKey}`);
+			
+			// Use SET with NX (not exists) and PX (expire in milliseconds)
+			const result = await this.redis.set(
+				this.lockKey,
+				this.nodeId,
+				'PX', this.lockTTL,
+				'NX'
+			);
+
+			if (result === 'OK') {
+				// Successfully acquired leadership
+				if (!this.isLeader) {
+					this.isLeader = true;
+					Logger.info(`🏆 Node ${this.nodeId} ACQUIRED LEADERSHIP for ${this.lockKey} (TTL: ${this.lockTTL}ms)`);
+					this.callbacks.onStartedLeading();
+				} else {
+					Logger.debug(`Node ${this.nodeId} renewed leadership lock for ${this.lockKey}`);
+				}
+				return true;
+			} else {
+				// Failed to acquire leadership - check who is the current leader
+				const currentLeader = await this.redis.get(this.lockKey);
+				const lockTTL = await this.redis.pttl(this.lockKey);
+				
+				if (currentLeader) {
+					if (currentLeader !== this.nodeId) {
+						Logger.debug(`👑 Node ${this.nodeId} sees ${currentLeader} is the current leader for ${this.lockKey} (TTL: ${lockTTL}ms)`);
+						if (this.callbacks.onNewLeader) {
+							this.callbacks.onNewLeader(currentLeader);
+						}
+					}
+				} else {
+					Logger.debug(`Node ${this.nodeId} found no current leader for ${this.lockKey}, but failed to acquire lock`);
+				}
+				
+				// Lost leadership if we previously had it
+				if (this.isLeader) {
+					this.isLeader = false;
+					Logger.info(`📉 Node ${this.nodeId} LOST LEADERSHIP for ${this.lockKey}`);
+					this.callbacks.onStoppedLeading();
+				}
+				return false;
+			}
+		} catch (error: any) {
+			Logger.error(`❌ Error during leader election for ${this.nodeId}: ${error.message}`);
+			
+			// If we had leadership and there's an error, assume we lost it
+			if (this.isLeader) {
+				this.isLeader = false;
+				Logger.info(`💥 Node ${this.nodeId} LOST LEADERSHIP due to error: ${this.lockKey}`);
+				this.callbacks.onStoppedLeading();
+			}
+			return false;
+		}
+	}
+
+	/**
+	 * Renew the leadership lock
+	 */
+	private async renewLock(): Promise<boolean> {
+		try {
+			// Check if Redis connection is ready before attempting operations
+			if (this.redis.status !== 'ready') {
+				Logger.warn(`Redis not ready for ${this.nodeId} renewal, current status: ${this.redis.status}`);
+				return false;
+			}
+
+			Logger.debug(`🔄 Node ${this.nodeId} attempting to renew leadership for ${this.lockKey}`);
+			
+			// Use Lua script to atomically check ownership and renew
+			const script = `
+				if redis.call("get", KEYS[1]) == ARGV[1] then
+					return redis.call("pexpire", KEYS[1], ARGV[2])
+				else
+					return 0
+				end
+			`;
+			
+			const result = await this.redis.eval(
+				script,
+				1,
+				this.lockKey,
+				this.nodeId,
+				this.lockTTL.toString()
+			);
+
+			if (result === 1) {
+				Logger.debug(`✅ Node ${this.nodeId} successfully renewed leadership for ${this.lockKey} (TTL: ${this.lockTTL}ms)`);
+				return true;
+			} else {
+				Logger.warn(`⚠️  Node ${this.nodeId} failed to renew leadership for ${this.lockKey} - lock no longer owned by this node`);
+				return false;
+			}
+		} catch (error: any) {
+			Logger.error(`❌ Error renewing leadership lock for ${this.nodeId}: ${error.message}`);
+			return false;
+		}
+	}
+
+	/**
+	 * Release the leadership lock
+	 */
+	private async releaseLock(): Promise<void> {
+		try {
+			// Use Lua script to atomically check ownership and delete
+			const script = `
+				if redis.call("get", KEYS[1]) == ARGV[1] then
+					return redis.call("del", KEYS[1])
+				else
+					return 0
+				end
+			`;
+			
+			await this.redis.eval(script, 1, this.lockKey, this.nodeId);
+			Logger.info(`Node ${this.nodeId} released leadership lock for ${this.lockKey}`);
+		} catch (error: any) {
+			Logger.error(`Error releasing lock for ${this.nodeId}: ${error.message}`);
+		}
+	}
+
+	/**
+	 * Start the renewal/retry loop
+	 */
+	private startRenewalLoop(): void {
+		Logger.info(`Starting leadership monitoring loop for ${this.nodeId} (check every ${this.renewalInterval}ms)`);
+		
+		this.renewalTimer = setInterval(async () => {
+			if (this.isLeader) {
+				// Leader: Try to renew the lock
+				Logger.debug(`⏰ [LEADER] Node ${this.nodeId} periodic leadership renewal check`);
+				const renewed = await this.renewLock();
+				if (!renewed) {
+					// Failed to renew - we lost leadership
+					this.isLeader = false;
+					Logger.info(`💔 [LEADER] Node ${this.nodeId} LOST LEADERSHIP (failed to renew) for ${this.lockKey}`);
+					this.callbacks.onStoppedLeading();
+				} else {
+					// Successfully renewed - keeping leadership
+					//Logger.info(`🔒 [LEADER] Node ${this.nodeId} RENEWED LEADERSHIP - staying active for ${this.lockKey}`);
+				}
+			} else {
+				// Follower: Try to acquire leadership
+				Logger.debug(`⏰ [FOLLOWER] Node ${this.nodeId} checking for available leadership`);
+				await this.tryAcquireLeadership();
+			}
+		}, this.renewalInterval);
+	}
+
+	/**
+	 * Generate a unique node identifier
+	 */
+	private generateNodeId(): string {
+		return process.env['POD_NAME'] || 
+			   process.env['HOSTNAME'] || 
+			   `node-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+	}
+}

package/src/lib/UserSettings.ts

@@ -13,7 +13,7 @@ import {
 	USER_FOLDER_ENV_OVERWRITE,
 	USER_SETTINGS_FILE_NAME,
 	USER_SETTINGS_SUBFOLDER,
-} from '../../src';
+} from '..';
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const { promisify } = require('util');

package/src/lib/WorkflowExecute.ts

@@ -1,4 +1,3 @@
-
 import PCancelable = require('p-cancelable');
 
 import {
@@ -24,7 +23,7 @@ import {
 	WorkflowOperationError,
 } from '@fsai-flow/workflow';
 import { get } from 'lodash';
-import { NodeExecuteFunctions } from '../../src';
+import { NodeExecuteFunctions } from '..';
 
 export class WorkflowExecute {
 	runExecutionData: IRunExecutionData;
@@ -837,7 +836,7 @@ export class WorkflowExecute {
 							}
 
 							break;
-						} catch (error) {
+						} catch (error: any) {
 							this.runExecutionData.resultData.lastNodeExecuted = executionData.node.name;
 
 							executionError = {
@@ -846,6 +845,27 @@ export class WorkflowExecute {
 								stack: (error as NodeOperationError | NodeApiError).stack,
 							};
 
+							// Check if it's a critical database connection error that should fail immediately
+							const isCriticalConnectionError = error.message?.includes('too many clients') ||
+															error.message?.includes('Connection terminated') ||
+															error.message?.includes('connection is closed') ||
+															error.message?.includes('ECONNRESET') ||
+															error.message?.includes('ENOTFOUND') ||
+															error.code === 'ECONNRESET' ||
+															error.code === 'ENOTFOUND';
+
+							if (isCriticalConnectionError) {
+								Logger.error(`Critical database connection error in node "${executionNode.name}": ${error.message}`, {
+									node: executionNode.name,
+									workflowId: workflow.id,
+									error: error.message
+								});
+								
+								// For critical connection errors, break out of retry loop immediately
+								// This will cause the workflow to fail on this node
+								break;
+							}
+
 							Logger.debug(`Running node "${executionNode.name}" finished with error`, {
 								node: executionNode.name,
 								workflowId: workflow.id,

package/tsconfig.json

@@ -1,5 +1,4 @@
 {
-  "extends": "../../tsconfig.base.json",
   "compilerOptions": {
 		"baseUrl": ".",
     "paths": {