@fsai-flow/core 0.0.1 → 0.0.3

package/README.md

@@ -1,6 +1,14 @@
-# core
+# @fsai-flow/core
 
-This library was generated with [Nx](https://nx.dev).
+This library is a dependency for [FSAI-Flow](https://github.com/your-org/flowx) that contains base classes and types used throughout the application.
+
+## Installation
+
+This project is published to the npm registry. To use it in your project:
+
+```bash
+npm install @fsai-flow/core
+```
 
 ## Building
 
@@ -9,3 +17,15 @@ Run `nx build core` to build the library.
 ## Running unit tests
 
 Run `nx test core` to execute the unit tests via [Jest](https://jestjs.io).
+
+## Contributing
+
+To contribute to this project:
+
+1. Clone this repository
+2. Make your changes
+3. Open a Pull Request
+
+## Publishing
+
+Maintainers can publish a new version to the npm package registry.

package/package.json

@@ -1,15 +1,15 @@
 {
   "name": "@fsai-flow/core",
-  "version": "0.0.1",
-  "license": "PolyForm Noncommercial License 1.0.0",
+  "version": "0.0.3",
   "dependencies": {
-    "@fsai-flow/workflow": "workspace:^",
+    "@fsai-flow/workflow": "0.0.2",
     "client-oauth2": "^4.3.3",
     "cron": "~3.3.0",
     "crypto-js": "~4.2.0",
     "file-type": "^16.0.0",
-    "form-data": "^4.0.0",
+    "form-data": "4.0.4",
     "https-proxy-agent": "^7.0.6",
+    "ioredis": "^5.3.2",
     "lodash": "^4.17.21",
     "mime-types": "^2.1.27",
     "oauth-1.0a": "^2.2.6",
@@ -20,7 +20,7 @@
     "simple-oauth2": "^5.1.0",
     "tslib": "^2.3.0",
     "uuid": "^11.0.3",
-    "fast-glob": "catalog:"
+    "fast-glob": "3.2.12"
   },
   "devDependencies": {
     "@types/crypto-js": "^4.0.1",
@@ -35,10 +35,20 @@
     "@types/simple-oauth2": "^5.0.7",
     "@types/uuid": "^10.0.0",
     "axios": "^1.7.9",
+    "ioredis": "^5.3.2",
     "jsonc-eslint-parser": "^2.4.0",
     "typescript": "~5.7.2"
   },
   "type": "commonjs",
   "main": "dist/src/index",
-  "types": "dist/src/index.d.ts"
+  "types": "dist/src/index.d.ts",
+  "overrides": {
+    "request": {
+      "form-data": "2.5.4",
+      "tough-cookie": "4.1.3"
+    },
+    "request-promise-native": {
+      "tough-cookie": "4.1.3"
+    }
+  }
 }

package/src/index.ts

@@ -15,6 +15,7 @@ export * from './lib/BinaryDataManager';
 export * from './lib/Constants';
 export * from './lib/Credentials';
 export * from './lib/Interfaces';
+export * from './lib/RedisLeaderElectionManager';
 export * from './lib/LoadNodeParameterOptions';
 export * from './lib/NodeExecuteFunctions';
 export * from './lib/WorkflowExecute';

package/src/lib/ActiveWorkflows.ts

@@ -18,14 +18,49 @@ import {
 
 import { secureRandomNumber } from '../utils/crypto';
 
+import { RedisLeaderElectionManager } from './RedisLeaderElectionManager';
+
+import type Redis from 'ioredis';
+
 // eslint-disable-next-line import/no-cycle
 import { ITriggerTime, IWorkflowData } from '../../src';
+import { RedisOptions } from 'ioredis';
+
+interface IPollingWorkflow {
+	workflow: Workflow;
+	node: INode;
+	cronTimes: string[];
+	timezone: string;
+	executeTrigger: () => Promise<void>;
+	activeCronJobs: CronJob[];
+}
 
 export class ActiveWorkflows {
 	private workflowData: {
 		[key: string]: IWorkflowData;
 	} = {};
 
+	private redisConfig: string | RedisOptions | undefined;
+	private instanceLeaderElection: RedisLeaderElectionManager | null = null;
+	private pollingWorkflows: Map<string, IPollingWorkflow> = new Map();
+	private isLeader = false;
+	private isInitializingLeaderElection = false;
+	private leaderElectionInitialized = false;
+	private leaderElectionEnabled = false;
+
+	constructor(redisConfig?: string | RedisOptions) {
+		this.redisConfig = redisConfig;
+		this.leaderElectionEnabled = redisConfig !== undefined;
+		
+		if (!this.leaderElectionEnabled) {
+			Logger.info('📴 Redis leader election disabled - workflows will run on all instances');
+			this.isLeader = true; // In non-queue mode, every instance is a "leader"
+			this.leaderElectionInitialized = true;
+		}
+	}
+
+
+
 	/**
 	 * Returns if the workflow is active
 	 *
@@ -118,7 +153,138 @@ export class ActiveWorkflows {
 	}
 
 	/**
-	 * Activates polling for the given node
+	 * Initialize global leader election (called once per instance)
+	 */
+	private async initializeLeaderElection(): Promise<void> {
+		// Skip leader election if disabled
+		if (!this.leaderElectionEnabled) {
+			Logger.debug('📴 Leader election disabled - skipping initialization');
+			return;
+		}
+
+		// Prevent race condition - only one initialization at a time
+		if (this.leaderElectionInitialized || this.isInitializingLeaderElection) {
+			Logger.debug('⏳ Leader election already initialized or in progress, waiting...');
+			
+			// Wait for initialization to complete
+			while (this.isInitializingLeaderElection) {
+				await new Promise(resolve => setTimeout(resolve, 100));
+			}
+			return;
+		}
+
+		this.isInitializingLeaderElection = true;
+		
+		try {
+			const nodeId = process.env['POD_NAME'] || process.env['HOSTNAME'] || `flowx-instance-${Math.random().toString(36).substr(2, 9)}`;
+			const lockName = 'global-polling-leadership';
+
+			Logger.info(`🌟 Initializing global polling leadership for instance ${nodeId}`);
+
+			this.instanceLeaderElection = new RedisLeaderElectionManager(
+				lockName,
+				this.redisConfig!,  // We know it's not null because leaderElectionEnabled is true
+				{
+					onStartedLeading: () => {
+						Logger.info(`🏆 Instance ${nodeId} became GLOBAL POLLING LEADER - starting all registered workflows (current count: ${this.pollingWorkflows.size})`);
+						this.isLeader = true;
+						// Set initialized flag before starting workflows to prevent timing issue
+						this.leaderElectionInitialized = true;
+						this.startAllPollingWorkflows();
+					},
+					onStoppedLeading: () => {
+						Logger.info(`📉 Instance ${nodeId} lost GLOBAL POLLING LEADERSHIP - stopping all workflows`);
+						this.isLeader = false;
+						this.stopAllPollingWorkflows();
+					},
+					onNewLeader: (identity: string) => {
+						//Logger.info(`👑 New global polling leader: ${identity}`);
+					}
+				}
+			);
+
+			await this.instanceLeaderElection.start();
+			// Only set this if we didn't become leader during start() - prevent double-setting
+			if (!this.leaderElectionInitialized) {
+				this.leaderElectionInitialized = true;
+			}
+			
+			Logger.info(`✅ Global leader election initialized for ${nodeId}`);
+		} finally {
+			this.isInitializingLeaderElection = false;
+		}
+	}
+
+	/**
+	 * Start all registered polling workflows (called when becoming leader)
+	 */
+	private startAllPollingWorkflows(): void {
+		Logger.debug(`🔍 startAllPollingWorkflows called - isLeader: ${this.isLeader}, initialized: ${this.leaderElectionInitialized}, workflows: ${this.pollingWorkflows.size}`);
+		
+		// Safety check: Only start if we're truly the leader and initialization is complete
+		if (!this.isLeader || !this.leaderElectionInitialized) {
+			Logger.warn(`⚠️  Cannot start workflows - not leader (isLeader: ${this.isLeader}, initialized: ${this.leaderElectionInitialized})`);
+			return;
+		}
+
+		Logger.info(`🚀 Starting all ${this.pollingWorkflows.size} registered polling workflows`);
+		Logger.debug(`📋 Registered workflow keys: ${Array.from(this.pollingWorkflows.keys()).join(', ')}`);
+		
+		let startedCount = 0;
+		for (const [workflowKey, pollingWorkflow] of this.pollingWorkflows) {
+			try {
+				// Skip if already running (double-check safety)
+				if (pollingWorkflow.activeCronJobs.length > 0) {
+					Logger.warn(`⚠️  Workflow ${workflowKey} already has active cron jobs - skipping`);
+					continue;
+				}
+
+				Logger.info(`⏰ Starting cron jobs for workflow "${pollingWorkflow.workflow.name}", node "${pollingWorkflow.node.name}" (${pollingWorkflow.cronTimes.length} schedules)`);
+				
+				// Create and start cron jobs for this workflow
+				pollingWorkflow.activeCronJobs = pollingWorkflow.cronTimes.map(cronTime =>
+					new CronJob(cronTime, pollingWorkflow.executeTrigger, undefined, true, pollingWorkflow.timezone)
+				);
+				
+				startedCount++;
+				Logger.info(`✅ Started ${pollingWorkflow.activeCronJobs.length} cron jobs for workflow "${pollingWorkflow.workflow.name}"`);
+			} catch (error: any) {
+				Logger.error(`❌ Failed to start polling for ${workflowKey}: ${error.message}`);
+			}
+		}
+		
+		Logger.info(`🎯 Successfully started ${startedCount} of ${this.pollingWorkflows.size} polling workflows`);
+	}
+
+	/**
+	 * Stop all polling workflows (called when losing leadership)
+	 */
+	private stopAllPollingWorkflows(): void {
+		Logger.info(`🛑 Stopping all ${this.pollingWorkflows.size} polling workflows`);
+		
+		for (const [workflowKey, pollingWorkflow] of this.pollingWorkflows) {
+			try {
+				// Skip if already stopped (safety check)
+				if (pollingWorkflow.activeCronJobs.length === 0) {
+					Logger.debug(`⏭️  Workflow ${workflowKey} already stopped - skipping`);
+					continue;
+				}
+
+				Logger.info(`⏹️  Stopping cron jobs for workflow "${pollingWorkflow.workflow.name}", node "${pollingWorkflow.node.name}"`);
+				
+				// Stop all cron jobs for this workflow
+				pollingWorkflow.activeCronJobs.forEach(job => job.stop());
+				pollingWorkflow.activeCronJobs = [];
+				
+				Logger.debug(`✅ Stopped polling for ${workflowKey}`);
+			} catch (error: any) {
+				Logger.error(`❌ Failed to stop polling for ${workflowKey}: ${error.message}`);
+			}
+		}
+	}
+
+	/**
+	 * Activates polling for the given node with global Redis leader election
 	 *
 	 * @param {INode} node
 	 * @param {Workflow} workflow
@@ -137,27 +303,128 @@ export class ActiveWorkflows {
 	): Promise<IPollResponse> {
 		const pollFunctions = getPollFunctions(workflow, node, additionalData, mode, activation);
 
+		// Get polling configuration
 		const pollTimes = pollFunctions.getNodeParameter('pollTimes') as unknown as {
 			item: ITriggerTime[];
 		};
 
-		// Define the order the cron-time-parameter appear
+		// Build cron times
+		const cronTimes = this.buildCronTimes(pollTimes);
+		const timezone = pollFunctions.getTimezone();
+
+		// Validate cron times
+		this.validateCronTimes(cronTimes);
+
+		// Create unique workflow key
+		const workflowKey = `${workflow.id}-${node.name}`;
+
+		// Check if this workflow is already registered (prevents duplicate registrations)
+		if (this.pollingWorkflows.has(workflowKey)) {
+			Logger.warn(`⚠️  Workflow ${workflowKey} is already registered - returning existing closeFunction`);
+			const existingWorkflow = this.pollingWorkflows.get(workflowKey)!;
+			return {
+				closeFunction: async () => {
+					Logger.info(`🗑️  Removing duplicate registration for: "${workflow.name}" (node: ${node.name})`);
+					// Just log - don't actually remove since this is a duplicate
+				}
+			};
+		}
+
+		// Create the polling execution function
+		const executeTrigger = async () => {
+			try {
+				Logger.info(`Polling trigger initiated for workflow "${workflow.name}"`, {
+					workflowName: workflow.name,
+					workflowId: workflow.id,
+				});
+				const pollResponse = await workflow.runPoll(node, pollFunctions);
+
+				if (pollResponse !== null) {
+					// eslint-disable-next-line no-underscore-dangle
+					pollFunctions.__emit(pollResponse);
+				}
+			} catch (error: any) {
+				Logger.error(
+					`Polling trigger failed for workflow "${workflow.name}"`,
+					{
+						workflowName: workflow.name,
+						workflowId: workflow.id,
+						nodeName: node.name,
+						error: error.message,
+						statusCode: error.statusCode,
+						httpCode: error.httpCode,
+						description: error.description,
+						cause: error.cause?.message,
+					}
+				);
+			}
+		};
+
+		// Register this polling workflow
+		const pollingWorkflow: IPollingWorkflow = {
+			workflow,
+			node,
+			cronTimes,
+			timezone,
+			executeTrigger,
+			activeCronJobs: []
+		};
+
+		this.pollingWorkflows.set(workflowKey, pollingWorkflow);
+		Logger.info(`📋 Registered polling workflow: "${workflow.name}" (node: ${node.name}) [Key: ${workflowKey}] - Total workflows: ${this.pollingWorkflows.size}`);
+
+		// Initialize global leader election if enabled
+		await this.initializeLeaderElection();
+
+		// Start workflow if we're the leader (or if leader election is disabled)
+		if (this.isLeader && this.leaderElectionInitialized && pollingWorkflow.activeCronJobs.length === 0) {
+			const reason = this.leaderElectionEnabled ? 'Leader established' : 'Leader election disabled';
+			Logger.info(`🔥 Post-registration start: "${workflow.name}" [Key: ${workflowKey}] - ${reason}, starting immediately`);
+			try {
+				pollingWorkflow.activeCronJobs = pollingWorkflow.cronTimes.map(cronTime =>
+					new CronJob(cronTime, pollingWorkflow.executeTrigger, undefined, true, pollingWorkflow.timezone)
+				);
+				Logger.info(`✅ Post-registration: Started ${pollingWorkflow.activeCronJobs.length} cron jobs for "${workflow.name}"`);
+			} catch (error: any) {
+				Logger.error(`❌ Failed to start newly registered workflow ${workflowKey}: ${error.message}`);
+			}
+		} else {
+			const leaderElectionStatus = this.leaderElectionEnabled ? 'enabled' : 'disabled';
+			Logger.debug(`⏳ Post-registration wait: "${workflow.name}" [Key: ${workflowKey}] (isLeader: ${this.isLeader}, initialized: ${this.leaderElectionInitialized}, activeCronJobs: ${pollingWorkflow.activeCronJobs.length}, leaderElection: ${leaderElectionStatus})`);
+		}
+
+		// Return cleanup function
+		return {
+			closeFunction: async () => {
+				Logger.info(`🗑️  Removing polling workflow: "${workflow.name}" (node: ${node.name})`);
+				
+				// Stop cron jobs for this workflow
+				const workflow_to_remove = this.pollingWorkflows.get(workflowKey);
+				if (workflow_to_remove) {
+					workflow_to_remove.activeCronJobs.forEach(job => job.stop());
+				}
+				
+				// Remove from registered workflows
+				this.pollingWorkflows.delete(workflowKey);
+				
+				Logger.debug(`✅ Removed polling workflow ${workflowKey}`);
+			}
+		};
+	}
+
+	/**
+	 * Extract cron time building logic (same as original)
+	 */
+	private buildCronTimes(pollTimes: { item: ITriggerTime[] }): string[] {
 		const parameterOrder = [
-			'second', // 0 - 59
-			'minute', // 0 - 59
-			'hour', // 0 - 23
-			'dayOfMonth', // 1 - 31
-			'month', // 0 - 11(Jan - Dec)
-			'weekday', // 0 - 6(Sun - Sat)
+			'second', 'minute', 'hour', 'dayOfMonth', 'month', 'weekday'
 		];
 
-		// Get all the trigger times
 		const cronTimes: string[] = [];
-		let cronTime: string[];
-		let parameterName: string;
 		if (pollTimes.item !== undefined) {
 			for (const item of pollTimes.item) {
-				cronTime = [];
+				let cronTime: string[];
+				
 				if (item.mode === 'custom') {
 					cronTimes.push((item['cronExpression'] as string).trim());
 					continue;
@@ -177,51 +444,29 @@ export class ActiveWorkflows {
 					continue;
 				}
 
-				for (parameterName of parameterOrder) {
+				cronTime = [];
+				for (const parameterName of parameterOrder) {
 					if (item[parameterName] !== undefined) {
-						// Value is set so use it
 						cronTime.push(item[parameterName] as string);
 					} else if (parameterName === 'second') {
-						// For seconds we use by default a random one to make sure to
-						// balance the load a little bit over time
 						cronTime.push(secureRandomNumber(1, 60).toString());
 					} else {
-						// For all others set "any"
 						cronTime.push('*');
 					}
 				}
-
 				cronTimes.push(cronTime.join(' '));
 			}
 		}
+		return cronTimes;
+	}
 
-		// The trigger function to execute when the cron-time got reached
-		const executeTrigger = async () => {
-			// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-			Logger.debug(`Polling trigger initiated for workflow "${workflow.name}"`, {
-				workflowName: workflow.name,
-				workflowId: workflow.id,
-			});
-			const pollResponse = await workflow.runPoll(node, pollFunctions);
-
-			if (pollResponse !== null) {
-				// eslint-disable-next-line no-underscore-dangle
-				pollFunctions.__emit(pollResponse);
-			}
-		};
-
-		// Execute the trigger directly to be able to know if it works
-		await executeTrigger();
-
-		const timezone = pollFunctions.getTimezone();
-
-		// Start the cron-jobs
-		const cronJobs: CronJob[] = [];
-		// eslint-disable-next-line @typescript-eslint/no-shadow
+	/**
+	 * Validate cron times (same as original)
+	 */
+	private validateCronTimes(cronTimes: string[]): void {
 		for (const cronTime of cronTimes) {
 			const cronTimeParts = cronTime.split(' ');
 			
-			// Verificar se é polling em segundos
 			if (cronTimeParts.length === 6 && cronTimeParts[0].includes('/')) {
 				const secondsInterval = parseInt(cronTimeParts[0].replace('*/', ''), 10);
 				if (secondsInterval < 1) {
@@ -230,20 +475,7 @@ export class ActiveWorkflows {
 			} else if (cronTimeParts.length > 0 && cronTimeParts[0].includes('*') && !cronTimeParts[0].includes('/')) {
 				throw new Error('The polling interval is too short. It has to be at least a minute!');
 			}
-
-			cronJobs.push(new CronJob(cronTime, executeTrigger, undefined, true, timezone));
-		}
-
-		// Stop the cron-jobs
-		async function closeFunction() {
-			for (const cronJob of cronJobs) {
-				cronJob.stop();
-			}
 		}
-
-		return {
-			closeFunction,
-		};
 	}
 
 	/**
@@ -299,6 +531,45 @@ export class ActiveWorkflows {
 			}
 		}
 
+		// Remove polling workflows from the global map (closeFunction will handle stopping cron jobs)
+		const pollingWorkflowsToRemove: string[] = [];
+		for (const [workflowKey, pollingWorkflow] of this.pollingWorkflows) {
+			if (pollingWorkflow.workflow.id === id) {
+				pollingWorkflowsToRemove.push(workflowKey);
+			}
+		}
+
+		for (const workflowKey of pollingWorkflowsToRemove) {
+			const pollingWorkflow = this.pollingWorkflows.get(workflowKey);
+			if (pollingWorkflow) {
+				Logger.info(`🗑️  Removing polling workflow: "${pollingWorkflow.workflow.name}" (node: ${pollingWorkflow.node.name})`);
+				
+				// Stop cron jobs for this workflow
+				pollingWorkflow.activeCronJobs.forEach(job => job.stop());
+				
+				// Remove from registered workflows
+				this.pollingWorkflows.delete(workflowKey);
+				
+				Logger.debug(`✅ Removed polling workflow ${workflowKey}`);
+			}
+		}
+
+		// If no more polling workflows are registered and leader election is enabled, stop it
+		if (this.pollingWorkflows.size === 0 && this.instanceLeaderElection && this.leaderElectionEnabled) {
+			Logger.info(`📴 No more polling workflows - stopping global leader election`);
+			try {
+				await this.instanceLeaderElection.stop();
+				this.instanceLeaderElection = null;
+				this.isLeader = false;
+				this.leaderElectionInitialized = false;
+			} catch (error: any) {
+				Logger.error(
+					// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/restrict-template-expressions
+					`There was a problem stopping global leader election: "${error.message}"`
+				);
+			}
+		}
+
 		delete this.workflowData[id];
 	}
 }

package/src/lib/NodeExecuteFunctions.ts

@@ -899,7 +899,6 @@ export async function getCurrentOAuth2AccessToken(
 			client: { id: clientId, secret: clientSecret },
 			auth: { tokenHost: authUrl, tokenPath: accessTokenUrl },
 			http: { agent: httpAgent },
-			options: { authorizationMethod: 'body' },
 		};
 		
 		if (grantType === 'authorizationCode') {
@@ -999,9 +998,6 @@ export async function requestOAuth2(
 		http: {
 			agent: credentials['httpAgent'],
 		},
-		options: {
-			authorizationMethod: 'body',
-		},
 	};
 
 	const client = new AuthorizationCode(config);