@fsai-flow/core 0.0.1

package/src/lib/ActiveWorkflows.ts

@@ -0,0 +1,304 @@
+/* eslint-disable no-continue */
+/* eslint-disable no-await-in-loop */
+/* eslint-disable no-restricted-syntax */
+import { CronJob } from 'cron';
+
+import {
+	IGetExecutePollFunctions,
+	IGetExecuteTriggerFunctions,
+	INode,
+	IPollResponse,
+	ITriggerResponse,
+	IWorkflowExecuteAdditionalData,
+	LoggerProxy as Logger,
+	Workflow,
+	WorkflowActivateMode,
+	WorkflowExecuteMode,
+} from '@fsai-flow/workflow';
+
+import { secureRandomNumber } from '../utils/crypto';
+
+// eslint-disable-next-line import/no-cycle
+import { ITriggerTime, IWorkflowData } from '../../src';
+
+export class ActiveWorkflows {
+	private workflowData: {
+		[key: string]: IWorkflowData;
+	} = {};
+
+	/**
+	 * Returns if the workflow is active
+	 *
+	 * @param {string} id The id of the workflow to check
+	 * @returns {boolean}
+	 * @memberof ActiveWorkflows
+	 */
+	isActive(id: string): boolean {
+		// eslint-disable-next-line no-prototype-builtins
+		return this.workflowData.hasOwnProperty(id);
+	}
+
+	/**
+	 * Returns the ids of the currently active workflows
+	 *
+	 * @returns {string[]}
+	 * @memberof ActiveWorkflows
+	 */
+	allActiveWorkflows(): string[] {
+		return Object.keys(this.workflowData);
+	}
+
+	/**
+	 * Returns the Workflow data for the workflow with
+	 * the given id if it is currently active
+	 *
+	 * @param {string} id
+	 * @returns {(WorkflowData | undefined)}
+	 * @memberof ActiveWorkflows
+	 */
+	get(id: string): IWorkflowData | undefined {
+		return this.workflowData[id];
+	}
+
+	/**
+	 * Makes a workflow active
+	 *
+	 * @param {string} id The id of the workflow to activate
+	 * @param {Workflow} workflow The workflow to activate
+	 * @param {IWorkflowExecuteAdditionalData} additionalData The additional data which is needed to run workflows
+	 * @returns {Promise<void>}
+	 * @memberof ActiveWorkflows
+	 */
+	async add(
+		id: string,
+		workflow: Workflow,
+		additionalData: IWorkflowExecuteAdditionalData,
+		mode: WorkflowExecuteMode,
+		activation: WorkflowActivateMode,
+		getTriggerFunctions: IGetExecuteTriggerFunctions,
+		getPollFunctions: IGetExecutePollFunctions,
+	): Promise<void> {
+		this.workflowData[id] = {};
+		const triggerNodes = workflow.getTriggerNodes();
+
+		let triggerResponse: ITriggerResponse | undefined;
+		this.workflowData[id].triggerResponses = [];
+		for (const triggerNode of triggerNodes) {
+			triggerResponse = await workflow.runTrigger(
+				triggerNode,
+				getTriggerFunctions,
+				additionalData,
+				mode,
+				activation,
+			);
+			if (triggerResponse !== undefined) {
+				// If a response was given save it
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				this.workflowData[id].triggerResponses!.push(triggerResponse);
+			}
+		}
+
+		const pollNodes = workflow.getPollNodes();
+		if (pollNodes.length) {
+			this.workflowData[id].pollResponses = [];
+			for (const pollNode of pollNodes) {
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				this.workflowData[id].pollResponses!.push(
+					await this.activatePolling(
+						pollNode,
+						workflow,
+						additionalData,
+						getPollFunctions,
+						mode,
+						activation,
+					),
+				);
+			}
+		}
+	}
+
+	/**
+	 * Activates polling for the given node
+	 *
+	 * @param {INode} node
+	 * @param {Workflow} workflow
+	 * @param {IWorkflowExecuteAdditionalData} additionalData
+	 * @param {IGetExecutePollFunctions} getPollFunctions
+	 * @returns {Promise<IPollResponse>}
+	 * @memberof ActiveWorkflows
+	 */
+	async activatePolling(
+		node: INode,
+		workflow: Workflow,
+		additionalData: IWorkflowExecuteAdditionalData,
+		getPollFunctions: IGetExecutePollFunctions,
+		mode: WorkflowExecuteMode,
+		activation: WorkflowActivateMode,
+	): Promise<IPollResponse> {
+		const pollFunctions = getPollFunctions(workflow, node, additionalData, mode, activation);
+
+		const pollTimes = pollFunctions.getNodeParameter('pollTimes') as unknown as {
+			item: ITriggerTime[];
+		};
+
+		// Define the order the cron-time-parameter appear
+		const parameterOrder = [
+			'second', // 0 - 59
+			'minute', // 0 - 59
+			'hour', // 0 - 23
+			'dayOfMonth', // 1 - 31
+			'month', // 0 - 11(Jan - Dec)
+			'weekday', // 0 - 6(Sun - Sat)
+		];
+
+		// Get all the trigger times
+		const cronTimes: string[] = [];
+		let cronTime: string[];
+		let parameterName: string;
+		if (pollTimes.item !== undefined) {
+			for (const item of pollTimes.item) {
+				cronTime = [];
+				if (item.mode === 'custom') {
+					cronTimes.push((item['cronExpression'] as string).trim());
+					continue;
+				}
+				if (item.mode === 'everyMinute') {
+					cronTimes.push(`${secureRandomNumber(1, 60).toString()} * * * * *`);
+					continue;
+				}
+				if (item.mode === 'everyX') {
+					if (item['unit'] === 'minutes') {
+						cronTimes.push(`${secureRandomNumber(1, 60).toString()} */${item['value']} * * * *`);
+					} else if (item['unit'] === 'hours') {
+						cronTimes.push(`${secureRandomNumber(1, 60).toString()} 0 */${item['value']} * * *`);
+					} else if (item['unit'] === 'seconds') {
+						cronTimes.push(`*/${item['value']} * * * * *`);
+					}
+					continue;
+				}
+
+				for (parameterName of parameterOrder) {
+					if (item[parameterName] !== undefined) {
+						// Value is set so use it
+						cronTime.push(item[parameterName] as string);
+					} else if (parameterName === 'second') {
+						// For seconds we use by default a random one to make sure to
+						// balance the load a little bit over time
+						cronTime.push(secureRandomNumber(1, 60).toString());
+					} else {
+						// For all others set "any"
+						cronTime.push('*');
+					}
+				}
+
+				cronTimes.push(cronTime.join(' '));
+			}
+		}
+
+		// The trigger function to execute when the cron-time got reached
+		const executeTrigger = async () => {
+			// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+			Logger.debug(`Polling trigger initiated for workflow "${workflow.name}"`, {
+				workflowName: workflow.name,
+				workflowId: workflow.id,
+			});
+			const pollResponse = await workflow.runPoll(node, pollFunctions);
+
+			if (pollResponse !== null) {
+				// eslint-disable-next-line no-underscore-dangle
+				pollFunctions.__emit(pollResponse);
+			}
+		};
+
+		// Execute the trigger directly to be able to know if it works
+		await executeTrigger();
+
+		const timezone = pollFunctions.getTimezone();
+
+		// Start the cron-jobs
+		const cronJobs: CronJob[] = [];
+		// eslint-disable-next-line @typescript-eslint/no-shadow
+		for (const cronTime of cronTimes) {
+			const cronTimeParts = cronTime.split(' ');
+			
+			// Verificar se é polling em segundos
+			if (cronTimeParts.length === 6 && cronTimeParts[0].includes('/')) {
+				const secondsInterval = parseInt(cronTimeParts[0].replace('*/', ''), 10);
+				if (secondsInterval < 1) {
+					throw new Error('The polling interval is too short. It has to be at least 1 second!');
+				}
+			} else if (cronTimeParts.length > 0 && cronTimeParts[0].includes('*') && !cronTimeParts[0].includes('/')) {
+				throw new Error('The polling interval is too short. It has to be at least a minute!');
+			}
+
+			cronJobs.push(new CronJob(cronTime, executeTrigger, undefined, true, timezone));
+		}
+
+		// Stop the cron-jobs
+		async function closeFunction() {
+			for (const cronJob of cronJobs) {
+				cronJob.stop();
+			}
+		}
+
+		return {
+			closeFunction,
+		};
+	}
+
+	/**
+	 * Makes a workflow inactive
+	 *
+	 * @param {string} id The id of the workflow to deactivate
+	 * @returns {Promise<void>}
+	 * @memberof ActiveWorkflows
+	 */
+	async remove(id: string): Promise<void> {
+		if (!this.isActive(id)) {
+			// Workflow is currently not registered
+			throw new Error(
+				`The workflow with the id "${id}" is currently not active and can so not be removed`,
+			);
+		}
+
+		const workflowData = this.workflowData[id];
+
+		if (workflowData.triggerResponses) {
+			for (const triggerResponse of workflowData.triggerResponses) {
+				if (triggerResponse.closeFunction) {
+					try {
+						await triggerResponse.closeFunction();
+					} catch (error: any) {
+						Logger.error(
+							// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/restrict-template-expressions
+							`There was a problem deactivating trigger of workflow "${id}": "${error.message}"`,
+							{
+								workflowId: id,
+							},
+						);
+					}
+				}
+			}
+		}
+
+		if (workflowData.pollResponses) {
+			for (const pollResponse of workflowData.pollResponses) {
+				if (pollResponse.closeFunction) {
+					try {
+						await pollResponse.closeFunction();
+					} catch (error:any) {
+						Logger.error(
+							// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/restrict-template-expressions
+							`There was a problem deactivating polling trigger of workflow "${id}": "${error.message}"`,
+							{
+								workflowId: id,
+							},
+						);
+					}
+				}
+			}
+		}
+
+		delete this.workflowData[id];
+	}
+}

package/src/lib/BinaryDataManager/FileSystem.ts

@@ -0,0 +1,214 @@
+import { promises as fs } from 'fs';
+import * as path from 'path';
+import { v4 as uuid } from 'uuid';
+
+import { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
+
+const PREFIX_METAFILE = 'binarymeta';
+const PREFIX_PERSISTED_METAFILE = 'persistedmeta';
+
+export class BinaryDataFileSystem implements IBinaryDataManager {
+	private storagePath: string;
+
+	private binaryDataTTL: number;
+
+	private persistedBinaryDataTTL: number;
+
+	constructor(config: IBinaryDataConfig) {
+		this.storagePath = config.localStoragePath;
+		this.binaryDataTTL = config.binaryDataTTL;
+		this.persistedBinaryDataTTL = config.persistedBinaryDataTTL;
+	}
+
+	async init(startPurger = false): Promise<void> {
+		if (startPurger) {
+			setInterval(async () => {
+				await this.deleteMarkedFiles();
+			}, this.binaryDataTTL * 30000);
+
+			setInterval(async () => {
+				await this.deleteMarkedPersistedFiles();
+			}, this.persistedBinaryDataTTL * 30000);
+		}
+
+		return fs
+			.readdir(this.storagePath)
+			.catch(async () => fs.mkdir(this.storagePath, { recursive: true }))
+			.then(async () => fs.readdir(this.getBinaryDataMetaPath()))
+			.catch(async () => fs.mkdir(this.getBinaryDataMetaPath(), { recursive: true }))
+			.then(async () => fs.readdir(this.getBinaryDataPersistMetaPath()))
+			.catch(async () => fs.mkdir(this.getBinaryDataPersistMetaPath(), { recursive: true }))
+			.then(async () => this.deleteMarkedFiles())
+			.then(async () => this.deleteMarkedPersistedFiles())
+			.then(() => {});
+	}
+
+	async storeBinaryData(binaryBuffer: Buffer, executionId: string): Promise<string> {
+		const binaryDataId = this.generateFileName(executionId);
+		return this.addBinaryIdToPersistMeta(executionId, binaryDataId).then(async () =>
+			this.saveToLocalStorage(binaryBuffer, binaryDataId).then(() => binaryDataId),
+		);
+	}
+
+	async retrieveBinaryDataByIdentifier(identifier: string): Promise<Buffer> {
+		return this.retrieveFromLocalStorage(identifier);
+	}
+
+	async markDataForDeletionByExecutionId(executionId: string): Promise<void> {
+		const tt = new Date(new Date().getTime() + this.binaryDataTTL * 60000);
+		return fs.writeFile(
+			path.join(this.getBinaryDataMetaPath(), `${PREFIX_METAFILE}_${executionId}_${tt.valueOf()}`),
+			'',
+		);
+	}
+
+	async deleteMarkedFiles(): Promise<void> {
+		return this.deleteMarkedFilesByMeta(this.getBinaryDataMetaPath(), PREFIX_METAFILE);
+	}
+
+	async deleteMarkedPersistedFiles(): Promise<void> {
+		return this.deleteMarkedFilesByMeta(
+			this.getBinaryDataPersistMetaPath(),
+			PREFIX_PERSISTED_METAFILE,
+		);
+	}
+
+	private async addBinaryIdToPersistMeta(executionId: string, identifier: string): Promise<void> {
+		const currentTime = new Date().getTime();
+		const timeAtNextHour = currentTime + 3600000 - (currentTime % 3600000);
+		const timeoutTime = timeAtNextHour + this.persistedBinaryDataTTL * 60000;
+
+		const filePath = path.join(
+			this.getBinaryDataPersistMetaPath(),
+			`${PREFIX_PERSISTED_METAFILE}_${executionId}_${timeoutTime}`,
+		);
+
+		return fs
+			.readFile(filePath)
+			.catch(async () => fs.writeFile(filePath, identifier))
+			.then(() => {});
+	}
+
+	private async deleteMarkedFilesByMeta(metaPath: string, filePrefix: string): Promise<void> {
+		const currentTimeValue = new Date().valueOf();
+		const metaFileNames = await fs.readdir(metaPath);
+
+		const execsAdded: { [key: string]: number } = {};
+
+		const proms = metaFileNames.reduce(
+			(prev, curr) => {
+				const [prefix, executionId, ts] = curr.split('_');
+
+				if (prefix !== filePrefix) {
+					return prev;
+				}
+
+				const execTimestamp = parseInt(ts, 10);
+
+				if (execTimestamp < currentTimeValue) {
+					if (execsAdded[executionId]) {
+						// do not delete data, only meta file
+						prev.push(this.deleteMetaFileByPath(path.join(metaPath, curr)));
+						return prev;
+					}
+
+					execsAdded[executionId] = 1;
+					prev.push(
+						this.deleteBinaryDataByExecutionId(executionId).then(async () =>
+							this.deleteMetaFileByPath(path.join(metaPath, curr)),
+						),
+					);
+				}
+
+				return prev;
+			},
+			[Promise.resolve()],
+		);
+
+		return Promise.all(proms).then(() => {});
+	}
+
+	async duplicateBinaryDataByIdentifier(binaryDataId: string, prefix: string): Promise<string> {
+		const newBinaryDataId = this.generateFileName(prefix);
+
+		return fs
+			.copyFile(
+				path.join(this.storagePath, binaryDataId),
+				path.join(this.storagePath, newBinaryDataId),
+			)
+			.then(() => newBinaryDataId);
+	}
+
+	async deleteBinaryDataByExecutionId(executionId: string): Promise<void> {
+		const regex = new RegExp(`${executionId}_*`);
+		const filenames = await fs.readdir(path.join(this.storagePath));
+
+		const proms = filenames.reduce(
+			(allProms, filename) => {
+				if (regex.test(filename)) {
+					allProms.push(fs.rm(path.join(this.storagePath, filename)));
+				}
+
+				return allProms;
+			},
+			[Promise.resolve()],
+		);
+
+		return Promise.all(proms).then(async () => Promise.resolve());
+	}
+
+	async deleteBinaryDataByIdentifier(identifier: string): Promise<void> {
+		return this.deleteFromLocalStorage(identifier);
+	}
+
+	async persistBinaryDataForExecutionId(executionId: string): Promise<void> {
+		return fs.readdir(this.getBinaryDataPersistMetaPath()).then(async (metafiles) => {
+			const proms = metafiles.reduce(
+				(prev, curr) => {
+					if (curr.startsWith(`${PREFIX_PERSISTED_METAFILE}_${executionId}_`)) {
+						prev.push(fs.rm(path.join(this.getBinaryDataPersistMetaPath(), curr)));
+						return prev;
+					}
+
+					return prev;
+				},
+				[Promise.resolve()],
+			);
+
+			return Promise.all(proms).then(() => {});
+		});
+	}
+
+	private generateFileName(prefix: string): string {
+		return `${prefix}_${uuid()}`;
+	}
+
+	private getBinaryDataMetaPath() {
+		return path.join(this.storagePath, 'meta');
+	}
+
+	private getBinaryDataPersistMetaPath() {
+		return path.join(this.storagePath, 'persistMeta');
+	}
+
+	private async deleteMetaFileByPath(metafilePath: string): Promise<void> {
+		return fs.rm(metafilePath);
+	}
+
+	private async deleteFromLocalStorage(identifier: string) {
+		return fs.rm(path.join(this.storagePath, identifier));
+	}
+
+	private async saveToLocalStorage(data: Buffer, identifier: string) {
+		await fs.writeFile(path.join(this.storagePath, identifier), data);
+	}
+
+	private async retrieveFromLocalStorage(identifier: string): Promise<Buffer> {
+		const filePath = path.join(this.storagePath, identifier);
+		try {
+			return await fs.readFile(filePath);
+		} catch (e) {
+			throw new Error(`Error finding file: ${filePath}`);
+		}
+	}
+}

package/src/lib/BinaryDataManager/index.ts

@@ -0,0 +1,187 @@
+import { IBinaryData, INodeExecutionData } from '@fsai-flow/workflow';
+import { BINARY_ENCODING } from '../Constants';
+import { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
+import { BinaryDataFileSystem } from './FileSystem';
+
+export class BinaryDataManager {
+	private static instance: BinaryDataManager;
+
+	private managers: {
+		[key: string]: IBinaryDataManager;
+	};
+
+	private binaryDataMode: string;
+
+	private availableModes: string[];
+
+	constructor(config: IBinaryDataConfig) {
+		this.binaryDataMode = config.mode;
+		this.availableModes = config.availableModes.split(',');
+		this.managers = {};
+	}
+
+	static async init(config: IBinaryDataConfig, mainManager = false): Promise<void> {
+		if (BinaryDataManager.instance) {
+			throw new Error('Binary Data Manager already initialized');
+		}
+
+		BinaryDataManager.instance = new BinaryDataManager(config);
+
+		if (BinaryDataManager.instance.availableModes.includes('filesystem')) {
+			BinaryDataManager.instance.managers['filesystem'] = new BinaryDataFileSystem(config);
+			await BinaryDataManager.instance.managers['filesystem'].init(mainManager);
+		}
+
+		return undefined;
+	}
+
+	static getInstance(): BinaryDataManager {
+		if (!BinaryDataManager.instance) {
+			throw new Error('Binary Data Manager not initialized');
+		}
+
+		return BinaryDataManager.instance;
+	}
+
+	async storeBinaryData(
+		binaryData: IBinaryData,
+		binaryBuffer: Buffer,
+		executionId: string,
+	): Promise<IBinaryData> {
+		const retBinaryData = binaryData;
+
+		if (this.managers[this.binaryDataMode]) {
+			return this.managers[this.binaryDataMode]
+				.storeBinaryData(binaryBuffer, executionId)
+				.then((filename) => {
+					retBinaryData.id = this.generateBinaryId(filename);
+					return retBinaryData;
+				});
+		}
+
+		retBinaryData.data = binaryBuffer.toString(BINARY_ENCODING);
+		return binaryData;
+	}
+
+	async retrieveBinaryData(binaryData: IBinaryData): Promise<Buffer> {
+		if (binaryData.id) {
+			return this.retrieveBinaryDataByIdentifier(binaryData.id);
+		}
+
+		return Buffer.from(binaryData.data, BINARY_ENCODING);
+	}
+
+	async retrieveBinaryDataByIdentifier(identifier: string): Promise<Buffer> {
+		const { mode, id } = this.splitBinaryModeFileId(identifier);
+		if (this.managers[mode]) {
+			return this.managers[mode].retrieveBinaryDataByIdentifier(id);
+		}
+
+		throw new Error('Storage mode used to store binary data not available');
+	}
+
+	async markDataForDeletionByExecutionId(executionId: string): Promise<void> {
+		if (this.managers[this.binaryDataMode]) {
+			return this.managers[this.binaryDataMode].markDataForDeletionByExecutionId(executionId);
+		}
+
+		return Promise.resolve();
+	}
+
+	async persistBinaryDataForExecutionId(executionId: string): Promise<void> {
+		if (this.managers[this.binaryDataMode]) {
+			return this.managers[this.binaryDataMode].persistBinaryDataForExecutionId(executionId);
+		}
+
+		return Promise.resolve();
+	}
+
+	async deleteBinaryDataByExecutionId(executionId: string): Promise<void> {
+		if (this.managers[this.binaryDataMode]) {
+			return this.managers[this.binaryDataMode].deleteBinaryDataByExecutionId(executionId);
+		}
+
+		return Promise.resolve();
+	}
+
+	async duplicateBinaryData(
+		inputData: Array<INodeExecutionData[] | null> | unknown,
+		executionId: string,
+	): Promise<INodeExecutionData[][]> {
+		if (inputData && this.managers[this.binaryDataMode]) {
+			const returnInputData = (inputData as INodeExecutionData[][]).map(
+				async (executionDataArray) => {
+					if (executionDataArray) {
+						return Promise.all(
+							executionDataArray.map((executionData) => {
+								if (executionData.binary) {
+									return this.duplicateBinaryDataInExecData(executionData, executionId);
+								}
+
+								return executionData;
+							}),
+						);
+					}
+
+					return executionDataArray;
+				},
+			);
+
+			return Promise.all(returnInputData);
+		}
+
+		return Promise.resolve(inputData as INodeExecutionData[][]);
+	}
+
+	private generateBinaryId(filename: string) {
+		return `${this.binaryDataMode}:${filename}`;
+	}
+
+	private splitBinaryModeFileId(fileId: string): { mode: string; id: string } {
+		const [mode, id] = fileId.split(':');
+		return { mode, id };
+	}
+
+	private async duplicateBinaryDataInExecData(
+		executionData: INodeExecutionData,
+		executionId: string,
+	): Promise<INodeExecutionData> {
+		const binaryManager = this.managers[this.binaryDataMode];
+
+		if (executionData.binary) {
+			const binaryDataKeys = Object.keys(executionData.binary);
+			const bdPromises = binaryDataKeys.map(async (key: string) => {
+				if (!executionData.binary) {
+					return { key, newId: undefined };
+				}
+
+				const binaryDataId = executionData.binary[key].id;
+				if (!binaryDataId) {
+					return { key, newId: undefined };
+				}
+
+				return binaryManager
+					?.duplicateBinaryDataByIdentifier(
+						this.splitBinaryModeFileId(binaryDataId).id,
+						executionId,
+					)
+					.then((filename) => ({
+						newId: this.generateBinaryId(filename),
+						key,
+					}));
+			});
+
+			return Promise.all(bdPromises).then((b) => {
+				return b.reduce((acc, curr) => {
+					if (acc.binary && curr) {
+						acc.binary[curr.key].id = curr.newId;
+					}
+
+					return acc;
+				}, executionData);
+			});
+		}
+
+		return executionData;
+	}
+}