codehooks-js 1.3.5 → 1.3.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codehooks-js",
-  "version": "1.3.5",
+  "version": "1.3.7",
   "type": "module",
   "description": "Codehooks.io official library - provides express.JS like syntax",
   "main": "index.js",

package/types/index.d.ts

@@ -67,6 +67,7 @@ declare class Codehooks {
         configure(config: {
             collectionName: string;
             queuePrefix: string;
+            timeout: number;
         }): void;
         getDefinition(stepsName: string, stepName: string): Function;
         validateStepDefinition(step: Function): void;
@@ -80,6 +81,9 @@ declare class Codehooks {
         continue(stepsName: string, instanceId: string): Promise<{
             qId: string;
         }>;
+        continueAllTimedOut(): Promise<Array<{
+            qId: string;
+        }>>;
         getStepsStatus(id: string): Promise<any>;
         getInstances(filter: any): Promise<any[]>;
         cancelSteps(id: string): Promise<any>;

package/types/workflow/engine.d.mts

@@ -11,6 +11,8 @@ declare class StepsEngine extends EventEmitter<[never]> {
     static instance: any;
     static collectionName: string;
     static queuePrefix: string;
+    static timeout: number;
+    static maxStepCount: number;
     /**
      * Set the collection name for storing steps data
      * @param {string} name - Collection name
@@ -23,6 +25,18 @@ declare class StepsEngine extends EventEmitter<[never]> {
      * @throws {Error} If prefix is not a non-empty string
      */
     static setQueuePrefix(prefix: string): void;
+    /**
+     * Set the timeout for steps jobs
+     * @param {number} timeout - Timeout in milliseconds
+     * @throws {Error} If timeout is not a positive number
+     */
+    static setTimeout(timeout: number): void;
+    /**
+     * Set the maximum step count for a steps workflow
+     * @param {number} maxStepCount - Maximum step count
+     * @throws {Error} If maxStepCount is not a positive number
+     */
+    static setMaxStepCount(maxStepCount: number): void;
     /**
      * Get the singleton instance of StepsEngine
      * @returns {StepsEngine} The singleton instance
@@ -35,10 +49,12 @@ declare class StepsEngine extends EventEmitter<[never]> {
      * @param {Object} config - Configuration object
      * @param {string} config.collectionName - Collection name
      * @param {string} config.queuePrefix - Queue prefix
+     * @param {number} config.timeout - Timeout in milliseconds
      */
     configure(config: {
         collectionName: string;
         queuePrefix: string;
+        timeout: number;
     }): void;
     /**
      * Get the step definition for a specific step
@@ -111,6 +127,13 @@ declare class StepsEngine extends EventEmitter<[never]> {
     continue(stepsName: string, instanceId: string): Promise<{
         qId: string;
     }>;
+    /**
+     * Continue all timed out steps instances
+     * @returns {Promise<Array<{qId: string}>>} Array of results containing queue IDs for continued workflows
+     */
+    continueAllTimedOut(): Promise<Array<{
+        qId: string;
+    }>>;
     /**
      * Get the status of a steps instance
      * @param {string} id - ID of the steps instance

package/types/workflow/index.d.mts

@@ -5,6 +5,7 @@ export namespace StepsConfig {
 export const configure: (config: {
     collectionName: string;
     queuePrefix: string;
+    timeout: number;
 }) => void;
 export { Steps };
 export default StepsEngine;

package/workflow/engine.mjs

@@ -22,6 +22,8 @@ class StepsEngine extends EventEmitter {
     // Configuration
     static collectionName = 'workflowdata';
     static queuePrefix = 'workflowqueue';
+    static timeout = 30000;
+    static maxStepCount = 3;
 
     /**
      * Set the collection name for storing steps data
@@ -47,11 +49,36 @@ class StepsEngine extends EventEmitter {
         StepsEngine.queuePrefix = prefix.trim();
     }
 
+    /**
+     * Set the timeout for steps jobs
+     * @param {number} timeout - Timeout in milliseconds
+     * @throws {Error} If timeout is not a positive number
+     */
+    static setTimeout(timeout) {
+        if (typeof timeout !== 'number' || timeout <= 0) {
+            throw new Error('Timeout must be a positive number');
+        }
+        StepsEngine.timeout = timeout;
+    }
+
+    /**
+     * Set the maximum step count for a steps workflow
+     * @param {number} maxStepCount - Maximum step count
+     * @throws {Error} If maxStepCount is not a positive number
+     */
+    static setMaxStepCount(maxStepCount) {
+        if (typeof maxStepCount !== 'number' || maxStepCount <= 0) {    
+            throw new Error('Maximum step count must be a positive number');
+        }
+        StepsEngine.maxStepCount = maxStepCount;
+    }
+
     /**
      * Configure the steps engine
      * @param {Object} config - Configuration object
      * @param {string} config.collectionName - Collection name
      * @param {string} config.queuePrefix - Queue prefix
+     * @param {number} config.timeout - Timeout in milliseconds
      */
     configure(config) {
         if (config.collectionName) {
@@ -60,6 +87,12 @@ class StepsEngine extends EventEmitter {
         if (config.queuePrefix) {
             StepsEngine.setQueuePrefix(config.queuePrefix);
         }
+        if (config.timeout) {
+            StepsEngine.setTimeout(config.timeout);
+        }
+        if (config.maxStepCount) {
+            StepsEngine.setMaxStepCount(config.maxStepCount);
+        }
     }
 
     /**
@@ -118,39 +151,37 @@ class StepsEngine extends EventEmitter {
      * @throws {Error} If step execution fails
      */
     async handleNextStep(stepsName, nextStep, newState, instanceId, options) {
-        try {
-            // Handle array of next steps
-            if (Array.isArray(nextStep)) {
-                const results = [];
-                for (const step of nextStep) {
-                    const qid = await this.handleNextStep(stepsName, step, newState, instanceId, options);
-                    results.push(qid);
-                }
-                return results;
-            }
+        
+        // open the connection to the database
+        const connection = await Datastore.open();            
 
-            // Handle single next step
-            StepsEngine.getInstance().emit('stepStarted', { workflowName: stepsName, step: nextStep, state: newState, instanceId });
-            const connection = await Datastore.open();
-            // remove the _id from the newState
-            delete newState._id;
-            // Update the existing steps state in the database
-            newState = await connection.updateOne(StepsEngine.collectionName,
-                { _id: instanceId },
-                { $set: { ...newState, nextStep: nextStep, updatedAt: new Date().toISOString() } });
-            
-            StepsEngine.getInstance().emit('stateUpdated', { workflowName: stepsName, state: newState, instanceId });
-            
-            
-            
-            
-            try {
-                // Get the next step function
-                const func = StepsEngine.getInstance().getDefinition(stepsName, nextStep);
+        // Handle single next step
+        StepsEngine.getInstance().emit('stepStarted', { workflowName: stepsName, step: nextStep, state: newState, instanceId });
+        
+        // remove the _id from the newState
+        delete newState._id;
+        // increment the step count
+        newState.stepCount[nextStep] = (newState.stepCount[nextStep] || 0) + 1;
+        // Update the existing steps state in the database
+        newState = await connection.updateOne(StepsEngine.collectionName,
+            { _id: instanceId },
+            { $set: { ...newState, nextStep: nextStep, updatedAt: new Date().toISOString(), stepCount: newState.stepCount } });
+        
+        StepsEngine.getInstance().emit('stateUpdated', { workflowName: stepsName, state: newState, instanceId });    
+        
+        try {
+            // Get the next step function
+            const func = StepsEngine.getInstance().getDefinition(stepsName, nextStep);
 
-                // Wrap the callback in a Promise to ensure proper async handling
-                await new Promise((resolve, reject) => {
-                    func({...newState, instanceId: newState._id}, async (nextStep, userState, options) => {
+            // Wrap the callback in a Promise to ensure proper async handling
+            await new Promise(async (resolve, reject) => {
+                // check if the step count is greater than the max step count
+                if (newState.stepCount[nextStep] > StepsEngine.maxStepCount) {
+                    reject(new Error(`Step ${nextStep} has been executed ${newState.stepCount[nextStep]} times, which is greater than the maximum step count of ${StepsEngine.maxStepCount}`));
+                    return;
+                }
+                try {
+                    await func({...newState, instanceId: newState._id}, async (nextStep, userState, options) => {
                         try {
                             // Protect system-level properties
                             const protectedState = {
@@ -159,7 +190,10 @@ class StepsEngine extends EventEmitter {
                                 createdAt: newState.createdAt,
                                 updatedAt: newState.updatedAt,
                                 instanceId: newState.instanceId,
-                                workflowName: newState.workflowName
+                                workflowName: newState.workflowName,
+                                stepCount: newState.stepCount,
+                                parallelSteps: newState.parallelSteps,
+                                previousStep: newState.nextStep
                             };
                             
                             // Merge states with userState taking precedence, but protecting system fields
@@ -168,6 +202,27 @@ class StepsEngine extends EventEmitter {
                                 ...protectedState
                             };
 
+                            // update the parallel steps metadata
+                            if (mergedState.parallelSteps && mergedState.parallelSteps[mergedState.nextStep]) {
+                                // get a fresh copy of the parallel steps
+                                const fresh = await connection.findOne(StepsEngine.collectionName, { _id: instanceId });                                
+                                fresh.parallelSteps[mergedState.nextStep].done = true;
+                                fresh.parallelSteps[mergedState.nextStep].nextStep = nextStep;                           
+                                delete fresh._id;
+                                await connection.updateOne(StepsEngine.collectionName,
+                                    { _id: instanceId },
+                                    { $set: { ...fresh, parallelSteps: fresh.parallelSteps } }); 
+                                // Check if all parallel steps are done
+                                const allStepsDone = Object.values(fresh.parallelSteps).every(step => step.done);
+                                if (!allStepsDone) {
+                                    console.debug('Waiting for other parallel steps to complete');
+                                    resolve();
+                                    return;
+                                } else {
+                                    console.debug('All parallel steps are done');
+                                }
+                            }
+
                             // If there is no next step, the workflow is completed
                             if (nextStep === null) {
                                 delete mergedState._id;
@@ -182,16 +237,38 @@ class StepsEngine extends EventEmitter {
                                 return;
                             }
 
-                            // Enqueue the next step
-                            console.debug('enqueuing step', nextStep, instanceId);
-                            await connection.enqueue(`${StepsEngine.queuePrefix}_${stepsName}_${nextStep}`, {
+                            // Enqueue the next step, single or parallel                            
+                            if (Array.isArray(nextStep)) {
+                                const now = new Date().toISOString();
+                                const metadata = nextStep.reduce((acc, step) => {
+                                    acc[step] = { done: false, startTime: now };
+                                    return acc;
+                                }, {});
+                                const metadataDoc = await connection.updateOne(StepsEngine.collectionName,
+                                    { _id: instanceId },
+                                    { $set: { parallelSteps: metadata } });
+                                //console.log('metadataDoc', metadataDoc);
+                                // enqueue all steps in parallel
+                                for (const step of nextStep) {
+                                    console.debug('enqueue step', step, instanceId);
+                                    await connection.enqueue(`${StepsEngine.queuePrefix}_${stepsName}_${step}`, {
+                                        stepsName: stepsName,
+                                        goto: step,
+                                        state: mergedState,
+                                        options: options,
+                                        instanceId: instanceId
+                                    });
+                                }
+                            } else {
+                                console.debug('enqueue step', nextStep, instanceId);
+                                await connection.enqueue(`${StepsEngine.queuePrefix}_${stepsName}_${nextStep}`, {
                                 stepsName: stepsName,
                                 goto: nextStep,
                                 state: mergedState,
                                 options: options,
                                 instanceId: instanceId
                             });
-                            
+                            }                           
                             StepsEngine.getInstance().emit('stepEnqueued', { workflowName: stepsName, step: nextStep, state: newState, instanceId });
                             resolve();
                         } catch (error) {
@@ -199,20 +276,15 @@ class StepsEngine extends EventEmitter {
                             reject(error);
                         }
                     });
-                });
-            } catch (error) {
-                console.error('Error in step function: '+nextStep, error.message);
-                StepsEngine.getInstance().emit('error', { workflowName: stepsName, step: nextStep, state: newState, instanceId, error: error.message });
-                const connection = await Datastore.open();
-                await connection.updateOne(StepsEngine.collectionName,
-                    { _id: instanceId },
-                    { $set: { lastError: error.message, updatedAt: new Date().toISOString() } });
-                throw error;
-            }            
-        }
-        catch (error) {            
+                } catch (error) {
+                    console.error('Error executing step function:', error);
+                    reject(error);
+                }
+            });
+        } catch (error) {
+            console.error('error in handleNextStep outer', error.message);
             throw error;
-        }
+        }            
     }
 
     /**
@@ -233,13 +305,18 @@ class StepsEngine extends EventEmitter {
                 if (stepName !== undefined) {
                     //console.log('registering queue for step', `${StepsEngine.queuePrefix}_${name}_${stepName}`);
                     app.worker(`${StepsEngine.queuePrefix}_${name}_${stepName}`, async function(req, res) {                        
-                        try {
-                            
+                        try {                            
                             const { stepsName, goto, state, instanceId, options } = req.body.payload;
-                            console.debug('worker job', stepName, instanceId);
+                            console.debug('dequeue step', stepName, instanceId);
                             const qid = await StepsEngine.getInstance().handleNextStep(stepsName, goto, state, instanceId, options);
                         } catch (error) {
-                            console.error('Error in step function: ' + stepName, error.message);
+                            const { stepsName, goto, state, instanceId, options } = req.body.payload;
+                            const connection = await Datastore.open();
+                            await connection.updateOne(StepsEngine.collectionName,
+                                { _id: instanceId },
+                                { $set: { lastError: error, updatedAt: new Date().toISOString() } });
+                            console.error('Error in function: ' + stepName, error);
+                            StepsEngine.getInstance().emit('error', error);
                         } finally {
                             res.end();
                         }
@@ -277,7 +354,7 @@ class StepsEngine extends EventEmitter {
                 const connection = await Datastore.open();
                 // Create a new steps state in the database
                 const newState = await connection.insertOne(StepsEngine.collectionName,
-                    { ...initialState, nextStep: firstStepName, createdAt: new Date().toISOString(), workflowName: name });
+                    { ...initialState, nextStep: firstStepName, createdAt: new Date().toISOString(), workflowName: name, stepCount: {} });
                 const { _id: ID } = await connection.enqueue(`${StepsEngine.queuePrefix}_${name}_${firstStepName}`, {
                     stepsName: name,
                     goto: firstStepName,
@@ -339,7 +416,11 @@ class StepsEngine extends EventEmitter {
         if (!state) {
             throw new Error(`No steps found with instanceId: ${instanceId}`);
         }
-        
+        // update all step counts to 0  
+        for (const step in state.stepCount) {
+            state.stepCount[step] = 0;
+        }
+        await connection.updateOne(StepsEngine.collectionName, { _id: instanceId }, { $set: { stepCount: state.stepCount } });
         console.debug('continue state', state);
         StepsEngine.getInstance().emit('workflowContinued', { stepsName, step: state.nextStep, instanceId });                    
 
@@ -352,10 +433,33 @@ class StepsEngine extends EventEmitter {
                 instanceId: instanceId
             });
 
-            resolve({ qId: ID });
+            resolve({ instanceId });
         });
     }
 
+    /**
+     * Continue all timed out steps instances
+     * @returns {Promise<Array<{qId: string}>>} Array of results containing queue IDs for continued workflows
+     */
+    async continueAllTimedOut() {
+        const db = await Datastore.open();
+        const timedOutWorkflows = await db.collection(StepsEngine.collectionName).find({nextStep: {$ne: null}}).toArray();
+        const now = new Date();
+        const results = [];
+        for (const workflow of timedOutWorkflows) {
+            const createdAt = new Date(workflow.createdAt);
+            const diffMillis = now.getTime() - createdAt.getTime();
+            if (diffMillis > StepsEngine.timeout) {
+                const diffMinutes = diffMillis / (1000 * 60);
+                console.log('Timed out:', workflow._id, workflow.nextStep, `(${diffMinutes.toFixed(1)} minutes old)`);
+                const result = await this.continue(workflow.workflowName, workflow._id);    
+                console.log('Continued:', result._id);
+                results.push(result);
+            }
+        }
+        return results;
+    }
+
     /**
      * Get the status of a steps instance
      * @param {string} id - ID of the steps instance