multi-tasks 2.0.9 → 2.1.1

package/README.md

@@ -14,8 +14,7 @@ How to use:
 let multiTasks = require('multi-tasks').multiTasks;
 
 
-//Step1, if you have tasks that need to be executed simultaneously, 
-//       provide them as an array, multi-tasks will automatically split them and execute.
+//Step1, create your tasks that need to be executed simultaneously as an array.
 let alltasks = [];
 for(let i=0;i<50;i++){
     alltasks.push({
@@ -26,9 +25,12 @@ for(let i=0;i<50;i++){
 
 //Step2, provide a function to process a certain sub-task and return the result data
 let processTask = (task, helper)=>{
-    let {taskCount} = task;
+    let {taskCount} = task;//get your task data
+
+    //Run your processing logic here...
 
-    return 'a result which is not a promise'; 
+    //then return the result as a plain javascript object or an async promise
+    return 'result data'; 
 };
 
 //Step3, run!
@@ -109,6 +111,8 @@ let processTask = (task, helper)=>{
 
 Changelog:
 
+- 2.1.1 Fixbug: recreate a new one when a worker collapsed unexpectedly.
+- 2.1.0 Add a new output directory "log" where you can view the process time of each subtask
 - 2.0.9 Update readme examples
 - 2.0.8 Update readme examples
 - 2.0.7 Update changelog

package/examples/example0/run-readme.js

@@ -13,7 +13,7 @@ for(let i=0;i<50;i++){
 let processTask = (task, helper)=>{
     let {taskCount} = task;
 
-    return 'a result which is not a promise'; 
+    return 'result data'; 
 };
 
 //Step2.1, or return a promise
@@ -76,10 +76,10 @@ let processTask = (task, helper)=>{
 
 //Step3, run!
 multiTasks.start({
-    tasks: alltasks, 
+    initialTasks: alltasks, 
     processTask, 
     taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
-    taskId: 'my-task',
+    taskId: 'my-readme-task',
     numberOfWorkers: 3, //Assign how many workers are working in parallel
     //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
     onFinish: (report)=>{

package/examples/example0/run.js

@@ -25,6 +25,12 @@ let processTask = (task, helper)=>{
     if(taskCount===5) aaa =  bbb;
 
     if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is also OK
+    
+    if(taskCount===7) {return new Promise((resolve, reject)=>{//async exception
+        setTimeout(()=>{
+            a=beforeAll;
+        }, 10 * Math.random())
+    });}
 
     //by default, you should return a promise, all returned data can be found in the results/succ folder
     return new Promise((resolve, reject)=>{
@@ -44,7 +50,7 @@ multiTasks({
     initialTasks: alltasks, 
     processTask, 
     taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
-    taskId: 'my-task',
+    taskId: 'my-task' +  Math.random(),
     numberOfWorkers: 3, //Assign how many workers are working in parallel
     //autoCloseAfterCompletion: true,
     onFinish: (report)=>{

package/examples/example0/runm.mjs

@@ -0,0 +1,53 @@
+import {multiTasks} from '../../index.js';
+
+//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
+let alltasks = [];
+for(let i=0;i<50;i++){
+    let task_props = {
+        index: i,
+        name: `task-${i}`,
+        description: `This prop is for a subtask`
+    };
+    alltasks.push(task_props);
+}
+
+//Step2, Provide a process function to handle a certain sub-task and return the result data
+let processTask = (task, helper)=>{
+    //console.log('this is the subtask data you created above:', task);
+    let {taskCount} = task;
+
+    console.log(taskCount)
+
+
+    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
+    if(taskCount===3) throw 'exception';
+    if(taskCount===4) return Promise.reject({err:'a test error'});
+    if(taskCount===5) aaa =  bbb;
+
+    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is also OK
+
+    //by default, you should return a promise, all returned data can be found in the results/succ folder
+    return new Promise((resolve, reject)=>{
+        setTimeout(()=>{
+            if(taskCount % 2 === 0)helper.createNewTasks({//
+                msg:'a new task'
+            });
+            taskCount === 2 ? reject('a rejected error') : resolve({
+                message: 'this is a result from a demo, random data=' + Math.random()
+            });
+        }, 10 * Math.random())
+    })
+}
+
+//Step3, run!
+multiTasks({
+    initialTasks: alltasks, 
+    processTask, 
+    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
+    taskId: 'my-task',
+    numberOfWorkers: 3, //Assign how many workers are working in parallel
+    //autoCloseAfterCompletion: true,
+    onFinish: (report)=>{
+        console.log('finish callback', report);
+    }
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "2.0.9",
+  "version": "2.1.1",
   "description": "",
   "main": "index.js",
   "directories": {

package/test/test_resume/step1_start_and_crash.js

@@ -35,7 +35,7 @@ let processTask = (task, helper)=>{
             taskCount === 2 ? reject('a rejected error') : resolve({
                 message: 'this is a result from a demo, random data=' + Math.random()
             });
-        }, 10 * Math.random())
+        }, 500 * Math.random())
     })
 }
 

package/test/test_resume/test.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+rm -rf ./tmp
 node step1_start_and_crash
 sleep 3
 node step2_resume

package/workers/Multithread.js

@@ -61,58 +61,68 @@ const start = (config)=>{
         const AsyncQueue = require(`../utils/asyncQueue`);
         const queue = new AsyncQueue();  
 
+        let initWorker = (worker)=>{
+            WorkerMgr.addWorker(worker);
+            worker.on('message', function(message) {
+                if(message === MSG_REQUEST_NEW_TASK){
+                    queue.enqueue(async ()=>{
+                        return TaskMgr.updateNewTasks();
+                    });//use queue to make sure no I/O conflict
+                    queue.enqueue(async () => {
+                        return TaskMgr.popNewTask().then((subTask)=>{
+                            if(!subTask){
+                                //no tasks left
+                                console.log('[Master]: no tasks left!');
+                                //console.log('workers', cluster.workers);
+                                if(config.autoCloseAfterCompletion){
+                                    killWorker(worker);
+                                }else{//waiting for new tasks
+                                    worker.send(MSG_NO_TASK_FOUND,()=>{}, {
+                                        keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
+                                    });
+                                }
+                            }else{
+                                let terminateData = {taskCount,subTask,config, id: worker.id};
+                                if(USER_CONFIG.shouldTerminate && USER_CONFIG.shouldTerminate(terminateData)===true) {
+                                    killAllWorkers();
+                                    return process.exit(0);
+                                }
+                                //send new tasks to the worker
+                                worker.send({
+                                    taskCount,
+                                    startTimestamp: new Date()*1,
+                                    config,
+                                    subTask
+                                },()=>{}, {
+                                    keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
+                                });
+                                taskCount++;
+                            }
+                        });
+                    });
+                }else 
+                if(message.MSG_SET_GLOBAL_PARAMS){
+                    console.log('xxxxxxxxxxxxxxxxxxxxxxxxxx', message)
+                }
+            });
+        };
+
         let initPromise = config.__resume ? asMaster.resume(config) : asMaster.init(config);
         initPromise.then((updated_config)=>{
             console.log(`  new tasks=${TaskMgr.getTasks('new').length}`);
             config = updated_config;
             for (let i = 0; i < numOfWorkers; i++) {
                 let worker = cluster.fork();
-                WorkerMgr.addWorker(worker);
-                worker.on('message', function(message) {
-                    if(message === MSG_REQUEST_NEW_TASK){
-                        queue.enqueue(async ()=>{
-                            return TaskMgr.updateNewTasks();
-                        });//use queue to make sure no I/O conflict
-                        queue.enqueue(async () => {
-                            return TaskMgr.popNewTask().then((subTask)=>{
-                                if(!subTask){
-                                    //no tasks left
-                                    console.log('[Master]: no tasks left!');
-                                    //console.log('workers', cluster.workers);
-                                    if(config.autoCloseAfterCompletion){
-                                        killWorker(worker);
-                                    }else{//waiting for new tasks
-                                        worker.send(MSG_NO_TASK_FOUND,()=>{}, {
-                                            keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
-                                        });
-                                    }
-                                }else{
-                                    let terminateData = {taskCount,subTask,config, id: worker.id};
-                                    if(USER_CONFIG.shouldTerminate && USER_CONFIG.shouldTerminate(terminateData)===true) {
-                                        killAllWorkers();
-                                        return process.exit(0);
-                                    }
-                                    //send new tasks to the worker
-                                    worker.send({
-                                        taskCount,
-                                        startTimestamp: new Date()*1,
-                                        config,
-                                        subTask
-                                    },()=>{}, {
-                                        keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
-                                    });
-                                    taskCount++;
-                                }
-                            });
-                        });
-                    }else 
-                    if(message.MSG_SET_GLOBAL_PARAMS){
-                        console.log('xxxxxxxxxxxxxxxxxxxxxxxxxx', message)
-                    }
-                });
+                initWorker(worker);
             }
         });
-        cluster.on('exit', (worker, code, signal) => { 
+        cluster.on('exit', (worker, code, signal) => {             
+            if (code !== 0 && !worker.exitedAfterDisconnect) {
+                console.log('Worker died unexpectedly starting a new one...');
+                let worker = cluster.fork();
+                initWorker(worker);
+                return;
+            }
             let allkilled = true; 
             for (const worker of Object.values(cluster.workers)) {
                 allkilled = allkilled && worker.killed;
@@ -156,9 +166,12 @@ function subWorker(){
                 let sendMsg = (msg)=>{
                     worker.send(msg);
                 };
-                runTask(config, subTask, sendMsg).then((report)=>{
-                    let endTimestamp = new Date()*1;
-                    let cost = startTimestamp - endTimestamp;
+                let subTaskLog = {
+                    taskCount,
+                    startTimestamp
+                };
+                runTask(config, subTask, subTaskLog, sendMsg).then((tasklog)=>{
+                    let {cost, startTimestamp, endTimestamp} = tasklog;
     
                     console.log(`[Task Finished] "${config.masterTaskId}"."${subTask.subid}".`, getDateTimeTxt(), `cost=${cost}ms`)
                 }).then(()=>{
@@ -173,7 +186,7 @@ function subWorker(){
     worker.send(MSG_REQUEST_NEW_TASK);//request new task
 
 };
-function runTask(config, subTask, sendMsg){
+function runTask(config, subTask, subTaskLog, sendMsg){
     return new Promise((resolve)=>{ 
         let resultPromise = Promise.resolve({});
         let resultData;
@@ -185,6 +198,7 @@ function runTask(config, subTask, sendMsg){
                 params
             });
         };
+        TaskMgr.saveTaskLog(subTask.subid, subTaskLog);
         try{
             resultPromise = USER_CONFIG.processTask(subTask, helper);//rewrite this method in subclass
         }catch(e){
@@ -198,11 +212,24 @@ function runTask(config, subTask, sendMsg){
             if(typeof ex !== 'string') ex = ex.toString();
             errors.push({ex});
         }).finally(()=>{
+            let {startTimestamp} = subTaskLog;
+            let endTimestamp = new Date()*1;
+            let cost = endTimestamp - startTimestamp;
+            subTaskLog = {
+                ...subTaskLog,
+                endTimestamp,
+                cost
+            };
             let hasError = errors.length > 0;
-            if(hasError)console.log('hasError=====', hasError, errors)
+            if(hasError)console.log('hasError=====', hasError, errors);            
+            TaskMgr.saveTaskLog(subTask.subid, subTaskLog);
             TaskMgr.saveTaskResult(subTask.subid, hasError, hasError ? errors : resultData);
             TaskMgr.mvTask(subTask.subid, 'running',(hasError ? 'finished_with_errors':'finished'), (newPath)=>{
-                resolve();
+                resolve({
+                    startTimestamp,
+                    endTimestamp,
+                    cost
+                });
             });   
         });
     });

package/workers/TaskMgr.js

@@ -43,19 +43,21 @@ const TaskMgr = {
         };
         //let taskfolder = pathutil.parse(__filename).dir;
         //let taskFolder = pathutil.resolve(taskfolder, `./tasks/${masterTaskId}`);
-        progressRootFolder = pathutil.resolve(taskFolder, './progress');
-        resultsRootFolder = pathutil.resolve(taskFolder, './results');
+        const progressRootFolder = pathutil.resolve(taskFolder, './progress');
+        const resultsRootFolder = pathutil.resolve(taskFolder, './results');
+        const logsRootFolder = pathutil.resolve(taskFolder, './logs');
 
-        taskFolderNew = pathutil.resolve(progressRootFolder, './new');
-        taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
-        taskFolderFinished = pathutil.resolve(progressRootFolder, './finished');
-        taskFolderFailed= pathutil.resolve(progressRootFolder, './finished_with_errors');
+        const taskFolderNew = pathutil.resolve(progressRootFolder, './new');
+        const taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
+        const taskFolderFinished = pathutil.resolve(progressRootFolder, './finished');
+        const taskFolderFailed= pathutil.resolve(progressRootFolder, './finished_with_errors');
 
-        taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
-        taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
+        const taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
+        const taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
         //let mainRptFolder = pathutil.resolve(taskfolder, './report');
     
         //makeDir.sync(mainRptFolder);
+        makeDir.sync(logsRootFolder);
         makeDir.sync(taskFolderNew);
         makeDir.sync(taskFolderRunning);
         makeDir.sync(taskFolderFinished);
@@ -68,6 +70,7 @@ const TaskMgr = {
             taskFolder,
             progressRootFolder,
             resultsRootFolder,
+            logsRootFolder,
 
             taskFolderNew,
             taskFolderRunning,
@@ -208,6 +211,19 @@ const TaskMgr = {
             fcontent = JSON.stringify(data);
         };
         fs.writeFileSync(fpath, fcontent);
+    },
+    saveTaskLog:(subid, log)=>{
+        let {logsRootFolder} = main_config;
+        let folder = logsRootFolder;
+        let fpath = pathutil.resolve(folder, `${subid}.json`);
+        let fcontent;
+        log.subid = subid;
+        if(typeof log === 'undefined') {
+            fcontent = '';
+        }else{
+            fcontent = JSON.stringify(log);
+        };
+        fs.writeFileSync(fpath, fcontent);
     }
 };
 module.exports = TaskMgr;