@fsai-flow/core 0.0.4 → 0.1.0

package/dist/lib/ActiveWorkflows.js

@@ -0,0 +1,465 @@
+"use strict";
+/* eslint-disable no-continue */
+/* eslint-disable no-await-in-loop */
+/* eslint-disable no-restricted-syntax */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ActiveWorkflows = void 0;
+const workflow_1 = require("@fsai-flow/workflow");
+const cron_1 = require("cron");
+const crypto_1 = require("../utils/crypto");
+const RedisLeaderElectionManager_1 = require("./RedisLeaderElectionManager");
+class ActiveWorkflows {
+    constructor(redisConfig) {
+        this.workflowData = {};
+        this.instanceLeaderElection = null;
+        this.pollingWorkflows = new Map();
+        this.isLeader = false;
+        this.isInitializingLeaderElection = false;
+        this.leaderElectionInitialized = false;
+        this.leaderElectionEnabled = false;
+        this.redisConfig = redisConfig;
+        this.leaderElectionEnabled = redisConfig !== undefined;
+        if (!this.leaderElectionEnabled) {
+            workflow_1.LoggerProxy.info("📴 Redis leader election disabled - workflows will run on all instances");
+            this.isLeader = true; // In non-queue mode, every instance is a "leader"
+            this.leaderElectionInitialized = true;
+        }
+    }
+    /**
+     * Returns if the workflow is active
+     *
+     * @param {string} id The id of the workflow to check
+     * @returns {boolean}
+     * @memberof ActiveWorkflows
+     */
+    isActive(id) {
+        return Object.hasOwn(this.workflowData, id);
+    }
+    /**
+     * Returns the ids of the currently active workflows
+     *
+     * @returns {string[]}
+     * @memberof ActiveWorkflows
+     */
+    allActiveWorkflows() {
+        return Object.keys(this.workflowData);
+    }
+    /**
+     * Returns the Workflow data for the workflow with
+     * the given id if it is currently active
+     *
+     * @param {string} id
+     * @returns {(WorkflowData | undefined)}
+     * @memberof ActiveWorkflows
+     */
+    get(id) {
+        return this.workflowData[id];
+    }
+    /**
+     * Makes a workflow active
+     *
+     * @param {string} id The id of the workflow to activate
+     * @param {Workflow} workflow The workflow to activate
+     * @param {IWorkflowExecuteAdditionalData} additionalData The additional data which is needed to run workflows
+     * @returns {Promise<void>}
+     * @memberof ActiveWorkflows
+     */
+    async add(id, workflow, additionalData, mode, activation, getTriggerFunctions, getPollFunctions) {
+        this.workflowData[id] = {};
+        const triggerNodes = workflow.getTriggerNodes();
+        let triggerResponse;
+        this.workflowData[id].triggerResponses = [];
+        for (const triggerNode of triggerNodes) {
+            triggerResponse = await workflow.runTrigger(triggerNode, getTriggerFunctions, additionalData, mode, activation);
+            if (triggerResponse !== undefined) {
+                // If a response was given save it
+                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                this.workflowData[id].triggerResponses?.push(triggerResponse);
+            }
+        }
+        const pollNodes = workflow.getPollNodes();
+        if (pollNodes.length) {
+            this.workflowData[id].pollResponses = [];
+            for (const pollNode of pollNodes) {
+                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                this.workflowData[id].pollResponses?.push(await this.activatePolling(pollNode, workflow, additionalData, getPollFunctions, mode, activation));
+            }
+        }
+    }
+    /**
+     * Initialize global leader election (called once per instance)
+     */
+    async initializeLeaderElection() {
+        // Skip leader election if disabled
+        if (!this.leaderElectionEnabled) {
+            workflow_1.LoggerProxy.debug("📴 Leader election disabled - skipping initialization");
+            return;
+        }
+        // Prevent race condition - only one initialization at a time
+        if (this.leaderElectionInitialized || this.isInitializingLeaderElection) {
+            workflow_1.LoggerProxy.debug("⏳ Leader election already initialized or in progress, waiting...");
+            // Wait for initialization to complete
+            while (this.isInitializingLeaderElection) {
+                await new Promise((resolve) => setTimeout(resolve, 100));
+            }
+            return;
+        }
+        this.isInitializingLeaderElection = true;
+        try {
+            const nodeId = process.env["POD_NAME"] ||
+                process.env["HOSTNAME"] ||
+                `flowx-instance-${Math.random().toString(36).substr(2, 9)}`;
+            const lockName = "global-polling-leadership";
+            workflow_1.LoggerProxy.info(`🌟 Initializing global polling leadership for instance ${nodeId}`);
+            this.instanceLeaderElection = new RedisLeaderElectionManager_1.RedisLeaderElectionManager(lockName, this.redisConfig, // We know it's not null because leaderElectionEnabled is true
+            {
+                onStartedLeading: () => {
+                    workflow_1.LoggerProxy.info(`🏆 Instance ${nodeId} became GLOBAL POLLING LEADER - starting all registered workflows (current count: ${this.pollingWorkflows.size})`);
+                    this.isLeader = true;
+                    // Set initialized flag before starting workflows to prevent timing issue
+                    this.leaderElectionInitialized = true;
+                    this.startAllPollingWorkflows();
+                },
+                onStoppedLeading: () => {
+                    workflow_1.LoggerProxy.info(`📉 Instance ${nodeId} lost GLOBAL POLLING LEADERSHIP - stopping all workflows`);
+                    this.isLeader = false;
+                    this.stopAllPollingWorkflows();
+                },
+                onNewLeader: (_identity) => {
+                    //Logger.info(`👑 New global polling leader: ${identity}`);
+                },
+            });
+            await this.instanceLeaderElection.start();
+            // Only set this if we didn't become leader during start() - prevent double-setting
+            if (!this.leaderElectionInitialized) {
+                this.leaderElectionInitialized = true;
+            }
+            workflow_1.LoggerProxy.info(`✅ Global leader election initialized for ${nodeId}`);
+        }
+        finally {
+            this.isInitializingLeaderElection = false;
+        }
+    }
+    /**
+     * Start all registered polling workflows (called when becoming leader)
+     */
+    startAllPollingWorkflows() {
+        workflow_1.LoggerProxy.debug(`🔍 startAllPollingWorkflows called - isLeader: ${this.isLeader}, initialized: ${this.leaderElectionInitialized}, workflows: ${this.pollingWorkflows.size}`);
+        // Safety check: Only start if we're truly the leader and initialization is complete
+        if (!this.isLeader || !this.leaderElectionInitialized) {
+            workflow_1.LoggerProxy.warn(`⚠️  Cannot start workflows - not leader (isLeader: ${this.isLeader}, initialized: ${this.leaderElectionInitialized})`);
+            return;
+        }
+        workflow_1.LoggerProxy.info(`🚀 Starting all ${this.pollingWorkflows.size} registered polling workflows`);
+        workflow_1.LoggerProxy.debug(`📋 Registered workflow keys: ${Array.from(this.pollingWorkflows.keys()).join(", ")}`);
+        let startedCount = 0;
+        for (const [workflowKey, pollingWorkflow] of this.pollingWorkflows) {
+            try {
+                // Skip if already running (double-check safety)
+                if (pollingWorkflow.activeCronJobs.length > 0) {
+                    workflow_1.LoggerProxy.warn(`⚠️  Workflow ${workflowKey} already has active cron jobs - skipping`);
+                    continue;
+                }
+                workflow_1.LoggerProxy.info(`⏰ Starting cron jobs for workflow "${pollingWorkflow.workflow.name}", node "${pollingWorkflow.node.name}" (${pollingWorkflow.cronTimes.length} schedules)`);
+                // Create and start cron jobs for this workflow
+                pollingWorkflow.activeCronJobs = pollingWorkflow.cronTimes.map((cronTime) => new cron_1.CronJob(cronTime, pollingWorkflow.executeTrigger, undefined, true, pollingWorkflow.timezone));
+                startedCount++;
+                workflow_1.LoggerProxy.info(`✅ Started ${pollingWorkflow.activeCronJobs.length} cron jobs for workflow "${pollingWorkflow.workflow.name}"`);
+            }
+            catch (error) {
+                workflow_1.LoggerProxy.error(`❌ Failed to start polling for ${workflowKey}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        workflow_1.LoggerProxy.info(`🎯 Successfully started ${startedCount} of ${this.pollingWorkflows.size} polling workflows`);
+    }
+    /**
+     * Stop all polling workflows (called when losing leadership)
+     */
+    stopAllPollingWorkflows() {
+        workflow_1.LoggerProxy.info(`🛑 Stopping all ${this.pollingWorkflows.size} polling workflows`);
+        for (const [workflowKey, pollingWorkflow] of this.pollingWorkflows) {
+            try {
+                // Skip if already stopped (safety check)
+                if (pollingWorkflow.activeCronJobs.length === 0) {
+                    workflow_1.LoggerProxy.debug(`⏭️  Workflow ${workflowKey} already stopped - skipping`);
+                    continue;
+                }
+                workflow_1.LoggerProxy.info(`⏹️  Stopping cron jobs for workflow "${pollingWorkflow.workflow.name}", node "${pollingWorkflow.node.name}"`);
+                // Stop all cron jobs for this workflow
+                for (const job of pollingWorkflow.activeCronJobs) {
+                    job.stop();
+                }
+                pollingWorkflow.activeCronJobs = [];
+                workflow_1.LoggerProxy.debug(`✅ Stopped polling for ${workflowKey}`);
+            }
+            catch (error) {
+                workflow_1.LoggerProxy.error(`❌ Failed to stop polling for ${workflowKey}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+    }
+    /**
+     * Activates polling for the given node with global Redis leader election
+     *
+     * @param {INode} node
+     * @param {Workflow} workflow
+     * @param {IWorkflowExecuteAdditionalData} additionalData
+     * @param {IGetExecutePollFunctions} getPollFunctions
+     * @returns {Promise<IPollResponse>}
+     * @memberof ActiveWorkflows
+     */
+    async activatePolling(node, workflow, additionalData, getPollFunctions, mode, activation) {
+        const pollFunctions = getPollFunctions(workflow, node, additionalData, mode, activation);
+        // Get polling configuration
+        const pollTimes = pollFunctions.getNodeParameter("pollTimes");
+        // Build cron times
+        const cronTimes = this.buildCronTimes(pollTimes);
+        const timezone = pollFunctions.getTimezone();
+        // Validate cron times
+        this.validateCronTimes(cronTimes);
+        // Create unique workflow key
+        const workflowKey = `${workflow.id}-${node.name}`;
+        // Check if this workflow is already registered (prevents duplicate registrations)
+        if (this.pollingWorkflows.has(workflowKey)) {
+            workflow_1.LoggerProxy.warn(`⚠️  Workflow ${workflowKey} is already registered - returning existing closeFunction`);
+            const _existingWorkflow = this.pollingWorkflows.get(workflowKey);
+            return {
+                closeFunction: async () => {
+                    workflow_1.LoggerProxy.info(`🗑️  Removing duplicate registration for: "${workflow.name}" (node: ${node.name})`);
+                    // Just log - don't actually remove since this is a duplicate
+                },
+            };
+        }
+        // Create the polling execution function
+        const executeTrigger = async () => {
+            try {
+                workflow_1.LoggerProxy.info(`Polling trigger initiated for workflow "${workflow.name}"`, {
+                    workflowName: workflow.name,
+                    workflowId: workflow.id,
+                });
+                const pollResponse = await workflow.runPoll(node, pollFunctions);
+                if (pollResponse !== null) {
+                    // eslint-disable-next-line no-underscore-dangle
+                    pollFunctions.__emit(pollResponse);
+                }
+            }
+            catch (error) {
+                const err = error;
+                workflow_1.LoggerProxy.error(`Polling trigger failed for workflow "${workflow.name}"`, {
+                    workflowName: workflow.name,
+                    workflowId: workflow.id,
+                    nodeName: node.name,
+                    error: err.message,
+                    statusCode: err.statusCode,
+                    httpCode: err.httpCode,
+                    description: err.description,
+                    cause: typeof err.cause === "object" &&
+                        err.cause instanceof Error
+                        ? err.cause.message
+                        : err.cause,
+                });
+            }
+        };
+        // Register this polling workflow
+        const pollingWorkflow = {
+            workflow,
+            node,
+            cronTimes,
+            timezone,
+            executeTrigger,
+            activeCronJobs: [],
+        };
+        this.pollingWorkflows.set(workflowKey, pollingWorkflow);
+        workflow_1.LoggerProxy.info(`📋 Registered polling workflow: "${workflow.name}" (node: ${node.name}) [Key: ${workflowKey}] - Total workflows: ${this.pollingWorkflows.size}`);
+        // Initialize global leader election if enabled
+        await this.initializeLeaderElection();
+        // Start workflow if we're the leader (or if leader election is disabled)
+        if (this.isLeader &&
+            this.leaderElectionInitialized &&
+            pollingWorkflow.activeCronJobs.length === 0) {
+            const reason = this.leaderElectionEnabled
+                ? "Leader established"
+                : "Leader election disabled";
+            workflow_1.LoggerProxy.info(`🔥 Post-registration start: "${workflow.name}" [Key: ${workflowKey}] - ${reason}, starting immediately`);
+            try {
+                pollingWorkflow.activeCronJobs = pollingWorkflow.cronTimes.map((cronTime) => new cron_1.CronJob(cronTime, pollingWorkflow.executeTrigger, undefined, true, pollingWorkflow.timezone));
+                workflow_1.LoggerProxy.info(`✅ Post-registration: Started ${pollingWorkflow.activeCronJobs.length} cron jobs for "${workflow.name}"`);
+            }
+            catch (error) {
+                workflow_1.LoggerProxy.error(`❌ Failed to start newly registered workflow ${workflowKey}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        else {
+            const leaderElectionStatus = this.leaderElectionEnabled
+                ? "enabled"
+                : "disabled";
+            workflow_1.LoggerProxy.debug(`⏳ Post-registration wait: "${workflow.name}" [Key: ${workflowKey}] (isLeader: ${this.isLeader}, initialized: ${this.leaderElectionInitialized}, activeCronJobs: ${pollingWorkflow.activeCronJobs.length}, leaderElection: ${leaderElectionStatus})`);
+        }
+        // Return cleanup function
+        return {
+            closeFunction: async () => {
+                workflow_1.LoggerProxy.info(`🗑️  Removing polling workflow: "${workflow.name}" (node: ${node.name})`);
+                // Stop cron jobs for this workflow
+                const workflow_to_remove = this.pollingWorkflows.get(workflowKey);
+                if (workflow_to_remove) {
+                    for (const job of workflow_to_remove.activeCronJobs) {
+                        job.stop();
+                    }
+                }
+                // Remove from registered workflows
+                this.pollingWorkflows.delete(workflowKey);
+                workflow_1.LoggerProxy.debug(`✅ Removed polling workflow ${workflowKey}`);
+            },
+        };
+    }
+    /**
+     * Extract cron time building logic (same as original)
+     */
+    buildCronTimes(pollTimes) {
+        const parameterOrder = [
+            "second",
+            "minute",
+            "hour",
+            "dayOfMonth",
+            "month",
+            "weekday",
+        ];
+        const cronTimes = [];
+        if (pollTimes.item !== undefined) {
+            for (const item of pollTimes.item) {
+                let cronTime;
+                if (item.mode === "custom") {
+                    cronTimes.push(item["cronExpression"].trim());
+                    continue;
+                }
+                if (item.mode === "everyMinute") {
+                    cronTimes.push(`${(0, crypto_1.secureRandomNumber)(1, 60).toString()} * * * * *`);
+                    continue;
+                }
+                if (item.mode === "everyX") {
+                    if (item["unit"] === "minutes") {
+                        cronTimes.push(`${(0, crypto_1.secureRandomNumber)(1, 60).toString()} */${item["value"]} * * * *`);
+                    }
+                    else if (item["unit"] === "hours") {
+                        cronTimes.push(`${(0, crypto_1.secureRandomNumber)(1, 60).toString()} 0 */${item["value"]} * * *`);
+                    }
+                    else if (item["unit"] === "seconds") {
+                        cronTimes.push(`*/${item["value"]} * * * * *`);
+                    }
+                    continue;
+                }
+                cronTime = [];
+                for (const parameterName of parameterOrder) {
+                    if (item[parameterName] !== undefined) {
+                        cronTime.push(item[parameterName]);
+                    }
+                    else if (parameterName === "second") {
+                        cronTime.push((0, crypto_1.secureRandomNumber)(1, 60).toString());
+                    }
+                    else {
+                        cronTime.push("*");
+                    }
+                }
+                cronTimes.push(cronTime.join(" "));
+            }
+        }
+        return cronTimes;
+    }
+    /**
+     * Validate cron times (same as original)
+     */
+    validateCronTimes(cronTimes) {
+        for (const cronTime of cronTimes) {
+            const cronTimeParts = cronTime.split(" ");
+            if (cronTimeParts.length === 6 && cronTimeParts[0].includes("/")) {
+                const secondsInterval = Number.parseInt(cronTimeParts[0].replace("*/", ""), 10);
+                if (secondsInterval < 1) {
+                    throw new Error("The polling interval is too short. It has to be at least 1 second!");
+                }
+            }
+            else if (cronTimeParts.length > 0 &&
+                cronTimeParts[0].includes("*") &&
+                !cronTimeParts[0].includes("/")) {
+                throw new Error("The polling interval is too short. It has to be at least a minute!");
+            }
+        }
+    }
+    /**
+     * Makes a workflow inactive
+     *
+     * @param {string} id The id of the workflow to deactivate
+     * @returns {Promise<void>}
+     * @memberof ActiveWorkflows
+     */
+    async remove(id) {
+        if (!this.isActive(id)) {
+            // Workflow is currently not registered
+            throw new Error(`The workflow with the id "${id}" is currently not active and can so not be removed`);
+        }
+        const workflowData = this.workflowData[id];
+        if (workflowData.triggerResponses) {
+            for (const triggerResponse of workflowData.triggerResponses) {
+                if (triggerResponse.closeFunction) {
+                    try {
+                        await triggerResponse.closeFunction();
+                    }
+                    catch (error) {
+                        workflow_1.LoggerProxy.error(`There was a problem deactivating trigger of workflow "${id}": "${error instanceof Error ? error.message : String(error)}"`, {
+                            workflowId: id,
+                        });
+                    }
+                }
+            }
+        }
+        if (workflowData.pollResponses) {
+            for (const pollResponse of workflowData.pollResponses) {
+                if (pollResponse.closeFunction) {
+                    try {
+                        await pollResponse.closeFunction();
+                    }
+                    catch (error) {
+                        workflow_1.LoggerProxy.error(`There was a problem deactivating polling trigger of workflow "${id}": "${error instanceof Error ? error.message : String(error)}"`, {
+                            workflowId: id,
+                        });
+                    }
+                }
+            }
+        }
+        // Remove polling workflows from the global map (closeFunction will handle stopping cron jobs)
+        const pollingWorkflowsToRemove = [];
+        for (const [workflowKey, pollingWorkflow] of this.pollingWorkflows) {
+            if (pollingWorkflow.workflow.id === id) {
+                pollingWorkflowsToRemove.push(workflowKey);
+            }
+        }
+        for (const workflowKey of pollingWorkflowsToRemove) {
+            const pollingWorkflow = this.pollingWorkflows.get(workflowKey);
+            if (pollingWorkflow) {
+                workflow_1.LoggerProxy.info(`🗑️  Removing polling workflow: "${pollingWorkflow.workflow.name}" (node: ${pollingWorkflow.node.name})`);
+                // Stop cron jobs for this workflow
+                for (const job of pollingWorkflow.activeCronJobs) {
+                    job.stop();
+                }
+                // Remove from registered workflows
+                this.pollingWorkflows.delete(workflowKey);
+                workflow_1.LoggerProxy.debug(`✅ Removed polling workflow ${workflowKey}`);
+            }
+        }
+        // If no more polling workflows are registered and leader election is enabled, stop it
+        if (this.pollingWorkflows.size === 0 &&
+            this.instanceLeaderElection &&
+            this.leaderElectionEnabled) {
+            workflow_1.LoggerProxy.info("📴 No more polling workflows - stopping global leader election");
+            try {
+                await this.instanceLeaderElection.stop();
+                this.instanceLeaderElection = null;
+                this.isLeader = false;
+                this.leaderElectionInitialized = false;
+            }
+            catch (error) {
+                workflow_1.LoggerProxy.error(`There was a problem stopping global leader election: "${error instanceof Error ? error.message : String(error)}"`);
+            }
+        }
+        delete this.workflowData[id];
+    }
+}
+exports.ActiveWorkflows = ActiveWorkflows;

package/dist/{src/lib → lib}/BinaryDataManager/FileSystem.d.ts

@@ -1,4 +1,4 @@
-import { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces';
+import type { IBinaryDataConfig, IBinaryDataManager } from "../Interfaces";
 export declare class BinaryDataFileSystem implements IBinaryDataManager {
     private storagePath;
     private binaryDataTTL;

package/dist/lib/BinaryDataManager/FileSystem.js

@@ -0,0 +1,180 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BinaryDataFileSystem = void 0;
+const node_fs_1 = require("node:fs");
+const path = __importStar(require("node:path"));
+const uuid_1 = require("uuid");
+const PREFIX_METAFILE = "binarymeta";
+const PREFIX_PERSISTED_METAFILE = "persistedmeta";
+class BinaryDataFileSystem {
+    constructor(config) {
+        this.storagePath = config.localStoragePath;
+        this.binaryDataTTL = config.binaryDataTTL;
+        this.persistedBinaryDataTTL = config.persistedBinaryDataTTL;
+    }
+    async init(startPurger = false) {
+        if (startPurger) {
+            setInterval(async () => {
+                await this.deleteMarkedFiles();
+            }, this.binaryDataTTL * 30000);
+            setInterval(async () => {
+                await this.deleteMarkedPersistedFiles();
+            }, this.persistedBinaryDataTTL * 30000);
+        }
+        return node_fs_1.promises
+            .readdir(this.storagePath)
+            .catch(async () => node_fs_1.promises.mkdir(this.storagePath, { recursive: true }))
+            .then(async () => node_fs_1.promises.readdir(this.getBinaryDataMetaPath()))
+            .catch(async () => node_fs_1.promises.mkdir(this.getBinaryDataMetaPath(), { recursive: true }))
+            .then(async () => node_fs_1.promises.readdir(this.getBinaryDataPersistMetaPath()))
+            .catch(async () => node_fs_1.promises.mkdir(this.getBinaryDataPersistMetaPath(), { recursive: true }))
+            .then(async () => this.deleteMarkedFiles())
+            .then(async () => this.deleteMarkedPersistedFiles())
+            .then(() => { });
+    }
+    async storeBinaryData(binaryBuffer, executionId) {
+        const binaryDataId = this.generateFileName(executionId);
+        return this.addBinaryIdToPersistMeta(executionId, binaryDataId).then(async () => this.saveToLocalStorage(binaryBuffer, binaryDataId).then(() => binaryDataId));
+    }
+    async retrieveBinaryDataByIdentifier(identifier) {
+        return this.retrieveFromLocalStorage(identifier);
+    }
+    async markDataForDeletionByExecutionId(executionId) {
+        const tt = new Date(Date.now() + this.binaryDataTTL * 60000);
+        return node_fs_1.promises.writeFile(path.join(this.getBinaryDataMetaPath(), `${PREFIX_METAFILE}_${executionId}_${tt.valueOf()}`), "");
+    }
+    async deleteMarkedFiles() {
+        return this.deleteMarkedFilesByMeta(this.getBinaryDataMetaPath(), PREFIX_METAFILE);
+    }
+    async deleteMarkedPersistedFiles() {
+        return this.deleteMarkedFilesByMeta(this.getBinaryDataPersistMetaPath(), PREFIX_PERSISTED_METAFILE);
+    }
+    async addBinaryIdToPersistMeta(executionId, identifier) {
+        const currentTime = Date.now();
+        const timeAtNextHour = currentTime + 3600000 - (currentTime % 3600000);
+        const timeoutTime = timeAtNextHour + this.persistedBinaryDataTTL * 60000;
+        const filePath = path.join(this.getBinaryDataPersistMetaPath(), `${PREFIX_PERSISTED_METAFILE}_${executionId}_${timeoutTime}`);
+        return node_fs_1.promises
+            .readFile(filePath)
+            .catch(async () => node_fs_1.promises.writeFile(filePath, identifier))
+            .then(() => { });
+    }
+    async deleteMarkedFilesByMeta(metaPath, filePrefix) {
+        const currentTimeValue = Date.now();
+        const metaFileNames = await node_fs_1.promises.readdir(metaPath);
+        const execsAdded = {};
+        const proms = metaFileNames.reduce((prev, curr) => {
+            const [prefix, executionId, ts] = curr.split("_");
+            if (prefix !== filePrefix) {
+                return prev;
+            }
+            const execTimestamp = Number.parseInt(ts, 10);
+            if (execTimestamp < currentTimeValue) {
+                if (execsAdded[executionId]) {
+                    // do not delete data, only meta file
+                    prev.push(this.deleteMetaFileByPath(path.join(metaPath, curr)));
+                    return prev;
+                }
+                execsAdded[executionId] = 1;
+                prev.push(this.deleteBinaryDataByExecutionId(executionId).then(async () => this.deleteMetaFileByPath(path.join(metaPath, curr))));
+            }
+            return prev;
+        }, [Promise.resolve()]);
+        return Promise.all(proms).then(() => { });
+    }
+    async duplicateBinaryDataByIdentifier(binaryDataId, prefix) {
+        const newBinaryDataId = this.generateFileName(prefix);
+        return node_fs_1.promises
+            .copyFile(path.join(this.storagePath, binaryDataId), path.join(this.storagePath, newBinaryDataId))
+            .then(() => newBinaryDataId);
+    }
+    async deleteBinaryDataByExecutionId(executionId) {
+        const regex = new RegExp(`${executionId}_*`);
+        const filenames = await node_fs_1.promises.readdir(path.join(this.storagePath));
+        const proms = filenames.reduce((allProms, filename) => {
+            if (regex.test(filename)) {
+                allProms.push(node_fs_1.promises.rm(path.join(this.storagePath, filename)));
+            }
+            return allProms;
+        }, [Promise.resolve()]);
+        return Promise.all(proms).then(async () => Promise.resolve());
+    }
+    async deleteBinaryDataByIdentifier(identifier) {
+        return this.deleteFromLocalStorage(identifier);
+    }
+    async persistBinaryDataForExecutionId(executionId) {
+        return node_fs_1.promises
+            .readdir(this.getBinaryDataPersistMetaPath())
+            .then(async (metafiles) => {
+            const proms = metafiles.reduce((prev, curr) => {
+                if (curr.startsWith(`${PREFIX_PERSISTED_METAFILE}_${executionId}_`)) {
+                    prev.push(node_fs_1.promises.rm(path.join(this.getBinaryDataPersistMetaPath(), curr)));
+                    return prev;
+                }
+                return prev;
+            }, [Promise.resolve()]);
+            return Promise.all(proms).then(() => { });
+        });
+    }
+    generateFileName(prefix) {
+        return `${prefix}_${(0, uuid_1.v4)()}`;
+    }
+    getBinaryDataMetaPath() {
+        return path.join(this.storagePath, "meta");
+    }
+    getBinaryDataPersistMetaPath() {
+        return path.join(this.storagePath, "persistMeta");
+    }
+    async deleteMetaFileByPath(metafilePath) {
+        return node_fs_1.promises.rm(metafilePath);
+    }
+    async deleteFromLocalStorage(identifier) {
+        return node_fs_1.promises.rm(path.join(this.storagePath, identifier));
+    }
+    async saveToLocalStorage(data, identifier) {
+        await node_fs_1.promises.writeFile(path.join(this.storagePath, identifier), data);
+    }
+    async retrieveFromLocalStorage(identifier) {
+        const filePath = path.join(this.storagePath, identifier);
+        try {
+            return await node_fs_1.promises.readFile(filePath);
+        }
+        catch (_e) {
+            throw new Error(`Error finding file: ${filePath}`);
+        }
+    }
+}
+exports.BinaryDataFileSystem = BinaryDataFileSystem;

package/dist/{src/lib → lib}/BinaryDataManager/index.d.ts

@@ -1,5 +1,5 @@
-import { IBinaryData, INodeExecutionData } from '@fsai-flow/workflow';
-import { IBinaryDataConfig } from '../Interfaces';
+import type { IBinaryData, INodeExecutionData } from "@fsai-flow/workflow";
+import type { IBinaryDataConfig } from "../Interfaces";
 export declare class BinaryDataManager {
     private static instance;
     private managers;