@fsai-flow/core 0.0.1 → 0.0.3

package/src/lib/RedisLeaderElectionManager.ts

@@ -0,0 +1,252 @@
+import Redis, { RedisOptions } from 'ioredis';
+import { LoggerProxy as Logger } from '@fsai-flow/workflow';
+
+export interface RedisLeaderElectionCallbacks {
+	onStartedLeading: () => void;
+	onStoppedLeading: () => void;
+	onNewLeader?: (identity: string) => void;
+}
+
+export class RedisLeaderElectionManager {
+	private redis: Redis;
+	private isLeader = false;
+	private lockKey: string;
+	private nodeId: string;
+	private lockTTL: number = 30000; // 30 seconds in milliseconds
+	private renewalInterval: number = 10000; // 10 seconds in milliseconds
+	private renewalTimer?: NodeJS.Timeout;
+	private callbacks: RedisLeaderElectionCallbacks;
+
+	constructor(
+		lockKey: string,
+		redisConfig: string | RedisOptions,
+		callbacks: RedisLeaderElectionCallbacks
+	) {
+		this.lockKey = `leader:${lockKey}`;
+		this.nodeId = this.generateNodeId();
+		this.callbacks = callbacks;
+		
+		// Initialize Redis connection
+		if (typeof redisConfig === 'string') {
+			this.redis = new Redis(redisConfig);
+		} else {
+			this.redis = new Redis(redisConfig);
+		}
+
+		// Handle Redis connection events
+		this.redis.on('connect', () => {
+			Logger.info(`Redis connected for leader election: ${this.lockKey}`);
+		});
+
+		this.redis.on('error', (error: any) => {
+			Logger.error(`Redis connection error for leader election: ${error.message}`);
+		});
+	}
+
+	/**
+	 * Start the leader election process
+	 */
+	async start(): Promise<void> {
+		Logger.info(`🚀 Starting Redis leader election for ${this.nodeId} on key ${this.lockKey}`);
+		Logger.info(`🔧 Leader election config: TTL=${this.lockTTL}ms, RenewalInterval=${this.renewalInterval}ms`);
+		
+		// Try to acquire leadership immediately
+		Logger.info(`🎯 ${this.nodeId} making initial leadership attempt...`);
+		await this.tryAcquireLeadership();
+		
+		// Start the renewal/retry loop
+		this.startRenewalLoop();
+		
+		Logger.info(`✅ Leader election process started for ${this.nodeId}`);
+	}
+
+	/**
+	 * Stop the leader election process
+	 */
+	async stop(): Promise<void> {
+		Logger.info(`🛑 Stopping Redis leader election for ${this.nodeId}`);
+		
+		// Stop renewal timer
+		if (this.renewalTimer) {
+			Logger.debug(`⏹️  Stopping leadership monitoring loop for ${this.nodeId}`);
+			clearInterval(this.renewalTimer as NodeJS.Timeout);
+			this.renewalTimer = undefined;
+		}
+
+		// Release leadership if we have it
+		if (this.isLeader) {
+			Logger.info(`👋 ${this.nodeId} releasing leadership voluntarily for ${this.lockKey}`);
+			await this.releaseLock();
+		}
+
+		// Close Redis connection
+		Logger.debug(`🔌 Disconnecting Redis for ${this.nodeId}`);
+		this.redis.disconnect();
+		
+		Logger.info(`✅ Leader election stopped for ${this.nodeId}`);
+	}
+
+	/**
+	 * Check if this node is currently the leader
+	 */
+	getIsLeader(): boolean {
+		return this.isLeader;
+	}
+
+	/**
+	 * Try to acquire leadership
+	 */
+	private async tryAcquireLeadership(): Promise<boolean> {
+		try {
+			Logger.debug(`Node ${this.nodeId} attempting to acquire leadership for ${this.lockKey}`);
+			
+			// Use SET with NX (not exists) and PX (expire in milliseconds)
+			const result = await this.redis.set(
+				this.lockKey,
+				this.nodeId,
+				'PX', this.lockTTL,
+				'NX'
+			);
+
+			if (result === 'OK') {
+				// Successfully acquired leadership
+				if (!this.isLeader) {
+					this.isLeader = true;
+					Logger.info(`🏆 Node ${this.nodeId} ACQUIRED LEADERSHIP for ${this.lockKey} (TTL: ${this.lockTTL}ms)`);
+					this.callbacks.onStartedLeading();
+				} else {
+					Logger.debug(`Node ${this.nodeId} renewed leadership lock for ${this.lockKey}`);
+				}
+				return true;
+			} else {
+				// Failed to acquire leadership - check who is the current leader
+				const currentLeader = await this.redis.get(this.lockKey);
+				const lockTTL = await this.redis.pttl(this.lockKey);
+				
+				if (currentLeader) {
+					if (currentLeader !== this.nodeId) {
+						Logger.debug(`👑 Node ${this.nodeId} sees ${currentLeader} is the current leader for ${this.lockKey} (TTL: ${lockTTL}ms)`);
+						if (this.callbacks.onNewLeader) {
+							this.callbacks.onNewLeader(currentLeader);
+						}
+					}
+				} else {
+					Logger.debug(`Node ${this.nodeId} found no current leader for ${this.lockKey}, but failed to acquire lock`);
+				}
+				
+				// Lost leadership if we previously had it
+				if (this.isLeader) {
+					this.isLeader = false;
+					Logger.info(`📉 Node ${this.nodeId} LOST LEADERSHIP for ${this.lockKey}`);
+					this.callbacks.onStoppedLeading();
+				}
+				return false;
+			}
+		} catch (error: any) {
+			Logger.error(`❌ Error during leader election for ${this.nodeId}: ${error.message}`);
+			
+			// If we had leadership and there's an error, assume we lost it
+			if (this.isLeader) {
+				this.isLeader = false;
+				Logger.info(`💥 Node ${this.nodeId} LOST LEADERSHIP due to error: ${this.lockKey}`);
+				this.callbacks.onStoppedLeading();
+			}
+			return false;
+		}
+	}
+
+	/**
+	 * Renew the leadership lock
+	 */
+	private async renewLock(): Promise<boolean> {
+		try {
+			Logger.debug(`🔄 Node ${this.nodeId} attempting to renew leadership for ${this.lockKey}`);
+			
+			// Use Lua script to atomically check ownership and renew
+			const script = `
+				if redis.call("get", KEYS[1]) == ARGV[1] then
+					return redis.call("pexpire", KEYS[1], ARGV[2])
+				else
+					return 0
+				end
+			`;
+			
+			const result = await this.redis.eval(
+				script,
+				1,
+				this.lockKey,
+				this.nodeId,
+				this.lockTTL.toString()
+			);
+
+			if (result === 1) {
+				Logger.debug(`✅ Node ${this.nodeId} successfully renewed leadership for ${this.lockKey} (TTL: ${this.lockTTL}ms)`);
+				return true;
+			} else {
+				Logger.warn(`⚠️  Node ${this.nodeId} failed to renew leadership for ${this.lockKey} - lock no longer owned by this node`);
+				return false;
+			}
+		} catch (error: any) {
+			Logger.error(`❌ Error renewing leadership lock for ${this.nodeId}: ${error.message}`);
+			return false;
+		}
+	}
+
+	/**
+	 * Release the leadership lock
+	 */
+	private async releaseLock(): Promise<void> {
+		try {
+			// Use Lua script to atomically check ownership and delete
+			const script = `
+				if redis.call("get", KEYS[1]) == ARGV[1] then
+					return redis.call("del", KEYS[1])
+				else
+					return 0
+				end
+			`;
+			
+			await this.redis.eval(script, 1, this.lockKey, this.nodeId);
+			Logger.info(`Node ${this.nodeId} released leadership lock for ${this.lockKey}`);
+		} catch (error: any) {
+			Logger.error(`Error releasing lock for ${this.nodeId}: ${error.message}`);
+		}
+	}
+
+	/**
+	 * Start the renewal/retry loop
+	 */
+	private startRenewalLoop(): void {
+		Logger.info(`Starting leadership monitoring loop for ${this.nodeId} (check every ${this.renewalInterval}ms)`);
+		
+		this.renewalTimer = setInterval(async () => {
+			if (this.isLeader) {
+				// Leader: Try to renew the lock
+				Logger.debug(`⏰ [LEADER] Node ${this.nodeId} periodic leadership renewal check`);
+				const renewed = await this.renewLock();
+				if (!renewed) {
+					// Failed to renew - we lost leadership
+					this.isLeader = false;
+					Logger.info(`💔 [LEADER] Node ${this.nodeId} LOST LEADERSHIP (failed to renew) for ${this.lockKey}`);
+					this.callbacks.onStoppedLeading();
+				} else {
+					// Successfully renewed - keeping leadership
+					//Logger.info(`🔒 [LEADER] Node ${this.nodeId} RENEWED LEADERSHIP - staying active for ${this.lockKey}`);
+				}
+			} else {
+				// Follower: Try to acquire leadership
+				Logger.debug(`⏰ [FOLLOWER] Node ${this.nodeId} checking for available leadership`);
+				await this.tryAcquireLeadership();
+			}
+		}, this.renewalInterval);
+	}
+
+	/**
+	 * Generate a unique node identifier
+	 */
+	private generateNodeId(): string {
+		return process.env['POD_NAME'] || 
+			   process.env['HOSTNAME'] || 
+			   `node-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+	}
+}

package/src/lib/WorkflowExecute.ts

@@ -1,4 +1,3 @@
-
 import PCancelable = require('p-cancelable');
 
 import {
@@ -284,6 +283,10 @@ export class WorkflowExecute {
 	): void {
 		let stillDataMissing = false;
 
+		if (connectionData.node === '__proto__' || connectionData.node === 'constructor' || connectionData.node === 'prototype') {
+			throw new Error('Prototype pollution detected');
+		}
+
 		// Check if node has multiple inputs as then we have to wait for all input data
 		// to be present before we can add it to the node-execution-stack
 		if (workflow.connectionsByDestinationNode[connectionData.node]['main'].length > 1) {
@@ -837,7 +840,7 @@ export class WorkflowExecute {
 							}
 
 							break;
-						} catch (error) {
+						} catch (error: any) {
 							this.runExecutionData.resultData.lastNodeExecuted = executionData.node.name;
 
 							executionError = {
@@ -846,6 +849,27 @@ export class WorkflowExecute {
 								stack: (error as NodeOperationError | NodeApiError).stack,
 							};
 
+							// Check if it's a critical database connection error that should fail immediately
+							const isCriticalConnectionError = error.message?.includes('too many clients') ||
+															error.message?.includes('Connection terminated') ||
+															error.message?.includes('connection is closed') ||
+															error.message?.includes('ECONNRESET') ||
+															error.message?.includes('ENOTFOUND') ||
+															error.code === 'ECONNRESET' ||
+															error.code === 'ENOTFOUND';
+
+							if (isCriticalConnectionError) {
+								Logger.error(`Critical database connection error in node "${executionNode.name}": ${error.message}`, {
+									node: executionNode.name,
+									workflowId: workflow.id,
+									error: error.message
+								});
+								
+								// For critical connection errors, break out of retry loop immediately
+								// This will cause the workflow to fail on this node
+								break;
+							}
+
 							Logger.debug(`Running node "${executionNode.name}" finished with error`, {
 								node: executionNode.name,
 								workflowId: workflow.id,

package/tsconfig.base.json

@@ -0,0 +1,28 @@
+{
+    "compileOnSave": false,
+    "compilerOptions": {
+      "rootDir": ".",
+      "sourceMap": true,
+      "declaration": false,
+      "moduleResolution": "node",
+      "emitDecoratorMetadata": true,
+      "experimentalDecorators": true,
+      "importHelpers": true,
+      "target": "es2015",
+      "module": "esnext",
+      "lib": ["es2020", "dom"],
+      "skipLibCheck": true,
+      "skipDefaultLibCheck": true,
+      "baseUrl": ".",
+      "paths": {
+        "@fsai-flow/core": ["libs/core/src/index.ts"],
+        "@fsai-flow/nodes-base": ["libs/nodes-base/src/index.ts"],
+        "@fsai-flow/nodes-langchain": ["libs/nodes-langchain/src/index.ts"],
+        "@fsai-flow/workflow": ["libs/workflow/src/index.ts"],
+        "@fsai-flow/design-system": ["apps/design-system/src/main.ts"],
+        "@fsai-flow/editor-ui": ["apps/editor-ui/src/index.ts"],
+        "@fsai-flow/cli": ["apps/cli/src/index.ts"],
+      }
+    },
+    "exclude": ["node_modules", "tmp"]
+  }

package/tsconfig.json

@@ -1,5 +1,5 @@
 {
-  "extends": "../../tsconfig.base.json",
+  "extends": "./tsconfig.base.json",
   "compilerOptions": {
 		"baseUrl": ".",
     "paths": {

package/dist/README.md

@@ -1,11 +0,0 @@
-# core
-
-This library was generated with [Nx](https://nx.dev).
-
-## Building
-
-Run `nx build core` to build the library.
-
-## Running unit tests
-
-Run `nx test core` to execute the unit tests via [Jest](https://jestjs.io).

package/dist/package.json

@@ -1,44 +0,0 @@
-{
-  "name": "@fsai-flow/core",
-  "version": "0.0.1",
-  "dependencies": {
-    "@fsai-flow/workflow": "workspace:^",
-    "client-oauth2": "^4.3.3",
-    "cron": "~3.3.0",
-    "crypto-js": "~4.2.0",
-    "file-type": "^16.0.0",
-    "form-data": "^4.0.0",
-    "https-proxy-agent": "^7.0.6",
-    "lodash": "^4.17.21",
-    "mime-types": "^2.1.27",
-    "oauth-1.0a": "^2.2.6",
-    "p-cancelable": "^2.0.0",
-    "qs": "^6.10.1",
-    "request": "^2.88.2",
-    "request-promise-native": "^1.0.9",
-    "simple-oauth2": "^5.1.0",
-    "tslib": "^2.3.0",
-    "uuid": "^11.0.3",
-    "fast-glob": "catalog:"
-  },
-  "devDependencies": {
-    "@types/crypto-js": "^4.0.1",
-    "@types/express": "^5.0.0",
-    "@types/lodash": "^4.17.13",
-    "@types/lodash.get": "^4.4.9",
-    "@types/mime-types": "^2.1.0",
-    "@types/node": "^22.10.2",
-    "@types/qs": "^6.9.17",
-    "@types/request": "^2.48.12",
-    "@types/request-promise-native": "^1.0.15",
-    "@types/simple-oauth2": "^5.0.7",
-    "@types/uuid": "^10.0.0",
-    "axios": "^1.7.9",
-    "jsonc-eslint-parser": "^2.4.0",
-    "typescript": "~5.7.2"
-  },
-  "type": "commonjs",
-  "main": "dist/src/index",
-  "types": "dist/src/index.d.ts",
-  "private": true
-}

package/dist/src/index.d.ts

@@ -1,15 +0,0 @@
-import * as NodeExecuteFunctions from './lib/NodeExecuteFunctions';
-import * as UserSettings from './lib/UserSettings';
-import * as getInputConnectionDataLegacy from './lib/InputConnectionDataLegacy';
-export * from './lib/ActiveWorkflows';
-export * from './lib/ActiveWebhooks';
-export * from './lib/BinaryDataManager';
-export * from './lib/Constants';
-export * from './lib/Credentials';
-export * from './lib/Interfaces';
-export * from './lib/LoadNodeParameterOptions';
-export * from './lib/NodeExecuteFunctions';
-export * from './lib/WorkflowExecute';
-export * from './lib/ChangeCase';
-export * from './lib/NodesLoader';
-export { NodeExecuteFunctions, UserSettings, getInputConnectionDataLegacy };

package/dist/src/index.js

@@ -1,29 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getInputConnectionDataLegacy = exports.UserSettings = exports.NodeExecuteFunctions = void 0;
-const tslib_1 = require("tslib");
-/* eslint-disable import/no-cycle */
-const NodeExecuteFunctions = tslib_1.__importStar(require("./lib/NodeExecuteFunctions"));
-exports.NodeExecuteFunctions = NodeExecuteFunctions;
-const UserSettings = tslib_1.__importStar(require("./lib/UserSettings"));
-exports.UserSettings = UserSettings;
-const getInputConnectionDataLegacy = tslib_1.__importStar(require("./lib/InputConnectionDataLegacy"));
-exports.getInputConnectionDataLegacy = getInputConnectionDataLegacy;
-try {
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, import/no-extraneous-dependencies, global-require, @typescript-eslint/no-var-requires
-    require('source-map-support').install();
-    // eslint-disable-next-line no-empty
-}
-catch (error) { }
-tslib_1.__exportStar(require("./lib/ActiveWorkflows"), exports);
-tslib_1.__exportStar(require("./lib/ActiveWebhooks"), exports);
-tslib_1.__exportStar(require("./lib/BinaryDataManager"), exports);
-tslib_1.__exportStar(require("./lib/Constants"), exports);
-tslib_1.__exportStar(require("./lib/Credentials"), exports);
-tslib_1.__exportStar(require("./lib/Interfaces"), exports);
-tslib_1.__exportStar(require("./lib/LoadNodeParameterOptions"), exports);
-tslib_1.__exportStar(require("./lib/NodeExecuteFunctions"), exports);
-tslib_1.__exportStar(require("./lib/WorkflowExecute"), exports);
-tslib_1.__exportStar(require("./lib/ChangeCase"), exports);
-tslib_1.__exportStar(require("./lib/NodesLoader"), exports);
-//# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;;AAAA,oCAAoC;AACpC,yFAAmE;AAsBlE,oDAAoB;AArBrB,yEAAmD;AAsBlD,oCAAY;AArBb,sGAAgF;AAsB/E,oEAA4B;AApB7B,IAAI,CAAC;IACJ,gMAAgM;IAChM,OAAO,CAAC,oBAAoB,CAAC,CAAC,OAAO,EAAE,CAAC;IACxC,oCAAoC;AACrC,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC,CAAA,CAAC;AAElB,gEAAsC;AACtC,+DAAqC;AACrC,kEAAwC;AACxC,0DAAgC;AAChC,4DAAkC;AAClC,2DAAiC;AACjC,yEAA+C;AAC/C,qEAA2C;AAC3C,gEAAsC;AACtC,2DAAiC;AACjC,4DAAkC"}

package/dist/src/lib/ActiveWebhooks.d.ts

@@ -1,59 +0,0 @@
-import { IWebhookData, WebhookHttpMethod, Workflow, WorkflowActivateMode, WorkflowExecuteMode } from '@fsai-flow/workflow';
-export declare class ActiveWebhooks {
-    private workflowWebhooks;
-    private webhookUrls;
-    testWebhooks: boolean;
-    /**
-     * Adds a new webhook
-     *
-     * @param {IWebhookData} webhookData
-     * @param {WorkflowExecuteMode} mode
-     * @returns {Promise<void>}
-     * @memberof ActiveWebhooks
-     */
-    add(workflow: Workflow, webhookData: IWebhookData, mode: WorkflowExecuteMode, activation: WorkflowActivateMode): Promise<void>;
-    /**
-     * Returns webhookData if a webhook with matches is currently registered
-     *
-     * @param {WebhookHttpMethod} httpMethod
-     * @param {string} path
-     * @param {(string | undefined)} webhookId
-     * @returns {(IWebhookData | undefined)}
-     * @memberof ActiveWebhooks
-     */
-    get(httpMethod: WebhookHttpMethod, path: string, webhookId?: string): IWebhookData | undefined;
-    /**
-     * Gets all request methods associated with a single webhook
-     * @param path
-     */
-    getWebhookMethods(path: string): string[];
-    /**
-     * Returns the ids of all the workflows which have active webhooks
-     *
-     * @returns {string[]}
-     * @memberof ActiveWebhooks
-     */
-    getWorkflowIds(): string[];
-    /**
-     * Returns key to uniquely identify a webhook
-     *
-     * @param {WebhookHttpMethod} httpMethod
-     * @param {string} path
-     * @param {(string | undefined)} webhookId
-     * @returns {string}
-     * @memberof ActiveWebhooks
-     */
-    getWebhookKey(httpMethod: WebhookHttpMethod, path: string, webhookId?: string): string;
-    /**
-     * Removes all webhooks of a workflow
-     *
-     * @param {Workflow} workflow
-     * @returns {boolean}
-     * @memberof ActiveWebhooks
-     */
-    removeWorkflow(workflow: Workflow): Promise<boolean>;
-    /**
-     * Removes all the webhooks of the given workflows
-     */
-    removeAll(workflows: Workflow[]): Promise<void>;
-}