multi-tasks 1.1.4 → 1.2.1

package/README.md

@@ -9,6 +9,25 @@ npm install multi-tasks
 
 How to use:
 
+```javascript
+//see examples/example0
+let multiTasks = require('multi-tasks').multiTasks;
+multiTasks({
+    tasks: [...], //You need to provide an array containing all the subtasks you want to execute, multi-tasks will automatically split them and execute them in a multi-threaded manner
+    processTask:(task)=>{//You need to provide a processTask function to process a certain sub-task and return the result data
+        //process your task
+        let data = {...};
+        return Promise.resolve(data);
+    }, 
+    multi_task_parent_folder: `C:/tmp-demo`, //a directory to store progress and results files
+    quotaOfEachTask:2, //The number of tasks assigned to each worker
+    numberOfWorks: 6
+});
+
+```
+
+Or, to provide the Producer & Consumer
+
 ```javascript
 let master = require('multi-tasks').master;
 let RunnerProducer;//See examples, please provide a Producer class to specify all tasks you have, multi-tasks will automatically split them and execute them in a multi-threaded manner
@@ -24,12 +43,14 @@ master.start(RunnerProducer, RunnerConsumer, {
 Have a try:
 
 ```shell
-node example0/run
-node example1/run
+node examples/example0/run
+node examples/example1/run
 ```
 
 Changelog:
 
+- 1.2.1 Handle exceptions and errors in subtasks
+- 1.2.0 Simplified usage by providing the function way and support return Promise
 - 1.1.4 Remove make-dir
 - 1.1.3 Simplified usage, see example0
 - 1.1.2 
@@ -37,4 +58,8 @@ Changelog:
 - 1.1.0 Simplified the usage of a customized Consumer, see example0
 - 1.0.8 Fix examples
 - 1.0.7 Remove moment
-- 1.0.6 Performance optimization
+- 1.0.6 Performance optimization
+
+Github:
+
+[https://github.com/zhanglei923/multi-tasks](https://github.com/zhanglei923/multi-tasks)

package/examples/example0/run.js

@@ -0,0 +1,29 @@
+let multiTasks = require('../../index').multiTasks;
+
+let alltasks = []
+for(let i=0;i<12;i++){
+    let task_props = {
+        index: i,
+        name: `task-${i}`,
+        description: `This prop is for a subtask`
+    };
+    alltasks.push(task_props);
+}
+
+let processTask = (actionData)=>{
+    console.log('this is the actionData of one task', actionData);
+    let {index} = actionData;
+    if(index===3) throw 'exception';
+    if(index===4) return Promise.reject({err:'a test error'});
+    return Promise.resolve({
+        message: 'this is a result from a demo, random data=' + Math.random()
+    });
+}
+
+multiTasks({
+    tasks: alltasks, //You need to provide the data of all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
+    processTask, //You need to provide a processTask function to process a certain sub-task and return the result data
+    multi_task_parent_folder: `C:/tmp-demo`, //a directory to store progress and results files
+    quotaOfEachTask:2, //The number of tasks assigned to each worker
+    numberOfWorks: 6
+});

package/examples/example1/RunnerConsumer.js

@@ -0,0 +1,26 @@
+let SuperConsumer = require('../../index').SuperConsumer;
+
+class TestConsumer extends SuperConsumer {
+  
+  processTask(actionData){
+        console.log(`processing data:`, actionData);
+        //you can also return a data instead of a promise object
+    let {index} = actionData;
+    if(index===3) throw 'exception';
+    if(index===4) return Promise.reject({err:'a test error'});
+        return new Promise((resolve) => {
+          setTimeout(() => {
+              let result = {
+                task:{            
+                  ...actionData,
+                },
+                result: {
+                  message: 'this is a result from a demo, random data=' + Math.random()
+                }
+              };
+              resolve(result);              
+          }, 100);
+      });
+    }
+  }
+module.exports = TestConsumer;

package/{example1 → examples/example1}/RunnerProducer.js

@@ -1,4 +1,4 @@
-let SuperProducer = require('../index').SuperProducer;
+let SuperProducer = require('../../index').SuperProducer;
 class TestProducer extends SuperProducer {
   
     // constructor
@@ -13,7 +13,7 @@ class TestProducer extends SuperProducer {
         /**
          * Generate all your tasks in this method, return as an array;
          * These data will be allocated to multiple threads (workers) for running; 
-         * You can obtain each of the data and process it in the 'process' method of your consumer.
+         * You can obtain each of the data and process it in the 'processTask' method of your consumer.
          * 
          * **/
 

package/{example0 → examples/example1}/run.js

@@ -1,6 +1,6 @@
 let RunnerProducer = require('./RunnerProducer');//You need to provide a Producer to return all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
 let RunnerConsumer = require('./RunnerConsumer');//You need to provide a Consumer to process a certain sub-task and return the processed data
-let master = require('../index').master;
+let master = require('../../index').master;
 
 master.start(RunnerProducer, RunnerConsumer, {
     multi_task_parent_folder: `C:/tmp-demo`,

package/{example1 → examples/example2}/RunnerConsumer.js

@@ -1,4 +1,4 @@
-let SuperConsumer = require('../index').SuperConsumer;
+let SuperConsumer = require('../../index').SuperConsumer;
 
 
 let _do = (mainTaskInfo, actions, callback)=>{

package/{example0 → examples/example2}/RunnerProducer.js

@@ -1,4 +1,4 @@
-let SuperProducer = require('../index').SuperProducer;
+let SuperProducer = require('../../index').SuperProducer;
 class TestProducer extends SuperProducer {
   
     // constructor
@@ -13,7 +13,7 @@ class TestProducer extends SuperProducer {
         /**
          * Generate all your tasks in this method, return as an array;
          * These data will be allocated to multiple threads (workers) for running; 
-         * You can obtain each of the data and process it in the 'process' method of your consumer.
+         * You can obtain each of the data and process it in the 'processTask' method of your consumer.
          * 
          * **/
 

package/{example1 → examples/example2}/run.js

@@ -6,7 +6,7 @@ let multi_task_parent_folder = `C:/tmp-demo`;
 
 let producer = new RunnerProducer(`workers_of_${taskId}`, multi_task_parent_folder)
 let consumer = new RunnerConsumer();
-let master = require('../index').master;
+let master = require('../../index').master;
 
 console.log(producer.getTaskName())
 //return;

package/index.js

@@ -1,8 +1,37 @@
 const SuperProducer = require('./workers/SuperProducer');
 const SuperConsumer = require('./workers/SuperConsumer');
-const master = require('./workers/master')
+const master = require('./workers/master');
+const multiTasks = (config)=>{
+    if(!config.tasks){
+        throw 'Please provide tasks data';
+    };
+    if(!config.processTask){
+        throw 'Please provide processTask function';
+    };
+    class TestProducer extends SuperProducer {      
+        // constructor
+        setTaskTrunkInfo(trunkInfo){
+            this.trunkInfo = trunkInfo;
+        }
+        getAllActions() {
+            return Promise.resolve(config.tasks);//return all your tasks as an array
+        }
+    }
+    class TestConsumer extends SuperConsumer {  
+        processTask(taskProp){
+            console.log(`processing data:`, taskProp);
+            let result = config.processTask(taskProp);
+            return Promise.resolve(result);
+        }
+    }
+
+    master.start(TestProducer, TestConsumer, {
+        ...config
+    });
+};
 module.exports = {
     SuperProducer,
     SuperConsumer,
-    master
+    master,
+    multiTasks
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "1.1.4",
+  "version": "1.2.1",
   "description": "",
   "main": "index.js",
   "directories": {

package/workers/SuperConsumer.js

@@ -12,32 +12,48 @@ class SuperConsumer {
     setTaskTrunkInfo(trunkInfo){
         this.trunkInfo = trunkInfo;
     }
-    _run (mainTaskInfo, subTaskInfo, actions, results, callback){
+    _run (mainTaskInfo, subTaskInfo, actions, results, errors, callback){
         if(actions.length === 0) return callback({
                                             finishedAt: new Date()*1,
                                             results,
+                                            errors,
                                             subTaskInfo
                                         });
-        let taskInfo = {
-            mainid: this.mainid,
-            subid: this.subid
-        };
-        let actionData = actions.shift();
+        let taskProp = actions.shift();
     
-        if(this.process){
-            let result = this.process(taskInfo, actionData);//rewrite this method in subclass
-            if(result){
-                results.push(result);
+        if(this.processTask){
+            let result;
+            let hasException = false;
+            try{
+                result = this.processTask(taskProp);//rewrite this method in subclass
+            }catch(e){
+                hasException = true;
+                result = {taskProp, exception: 'Exception:' + e.toString()}
             }
-        }
-    
-        this._run(mainTaskInfo, subTaskInfo, actions, results, callback);
+            if(typeof result === 'undefined') result = {};
+            let ispromise = result instanceof Promise;
+            if(!ispromise){result = Promise.resolve(result);}
+            result.then((taskResult)=>{
+                if(!hasException){
+                    results.push({taskProp, taskResult});
+                }else{
+                    errors.push(taskResult);
+                }
+            }).catch((e)=>{
+                errors.push({taskProp, exception: 'Exception:' + e.toString()});
+            }).finally(()=>{
+                this._run(mainTaskInfo, subTaskInfo, actions, results, errors, callback);
+            });            
+        }else{
+            this._run(mainTaskInfo, subTaskInfo, actions, results, errors, callback);
+        }    
     };
     run (mainTaskInfo, subTaskInfo, taskInfo, callback){
         let results = [];
+        let errors = [];
         let actions = subTaskInfo.actions;
         mainTaskInfo.defination = this.trunkInfo.defination;
-        this._run(mainTaskInfo, subTaskInfo, actions, results, (result)=>{
+        this._run(mainTaskInfo, subTaskInfo, actions, results, errors, (result)=>{
             callback(result)
         })
 
@@ -55,13 +71,18 @@ class SuperConsumer {
             let subid = subTaskInfo.subid;
             let taskfolderNew = mainTaskInfo.taskFolderNew;
             let taskFolderResult = mainTaskInfo.taskFolderResult;
+            let taskFolderResultFailed = mainTaskInfo.taskFolderResultFailed;            
             this.mvTask(mainid, subid,'new','running', (taskFile)=>{//mark as running
                 let taskContent = fs.readFileSync(taskFile, 'utf8');
                 let taskInfo = JSON.parse(taskContent);
                 this.run(mainTaskInfo, subTaskInfo, taskInfo, (taskReport)=>{//invoke
-                    let resultfile = pathutil.resolve(taskFolderResult, `${subid}.json`)
+                    let {results, errors} = taskReport;
+                    let hasError = false;
+                    if(errors &&  errors.length > 0) hasError = true;
+                    let folder = !hasError ? taskFolderResult : taskFolderResultFailed;
+                    let resultfile = pathutil.resolve(folder, `${subid}.json`)
                     fs.writeFileSync(resultfile, JSON.stringify(taskReport));
-                    this.mvTask(mainid, subid,'running','finished', ()=>{//mark as finished
+                    this.mvTask(mainid, subid,'running',(hasError ? 'finished_with_errors':'finished'), ()=>{//mark as finished
                         resolve({
                             mainid, 
                             subid,
@@ -73,7 +94,7 @@ class SuperConsumer {
         });
     }
     mvTask (mainid, subid, from, to, callback) {
-        let __thisdir = this.mainTaskInfo.taskFolder;
+        let __thisdir = this.mainTaskInfo.progressRootFolder;
         let oldPath = pathutil.resolve(__thisdir, `./${from}/${subid}.task.json`);
         let newPath = pathutil.resolve(__thisdir, `./${to}/${subid}.task.json`);
         fs.rename(oldPath, newPath, (err) => {

package/workers/SuperProducer.js

@@ -39,17 +39,25 @@ class SuperProducer {
         let taskFolder = this.taskfolder;
         //let taskfolder = pathutil.parse(__filename).dir;
         //let taskFolder = pathutil.resolve(taskfolder, `./tasks/${masterTaskId}`);
-        let taskFolderNew = pathutil.resolve(taskFolder, './new');
-        let taskFolderRunning = pathutil.resolve(taskFolder, './running');
-        let taskFolderFinished = pathutil.resolve(taskFolder, './finished');
-        let taskFolderResult = pathutil.resolve(taskFolder, './result');
+        let progressRootFolder = pathutil.resolve(taskFolder, './progress');
+        let resultsRootFolder = pathutil.resolve(taskFolder, './results');
+
+        let taskFolderNew = pathutil.resolve(progressRootFolder, './new');
+        let taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
+        let taskFolderFinished = pathutil.resolve(progressRootFolder, './finished');
+        let taskFolderFailed= pathutil.resolve(progressRootFolder, './finished_with_errors');
+
+        let taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
+        let taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
         //let mainRptFolder = pathutil.resolve(taskfolder, './report');
     
         //makeDir.sync(mainRptFolder);
         makeDir.sync(taskFolderNew);
         makeDir.sync(taskFolderRunning);
         makeDir.sync(taskFolderFinished);
+        makeDir.sync(taskFolderFailed);
         makeDir.sync(taskFolderResult);
+        makeDir.sync(taskFolderResultFailed);
 
         this.getAllActions().then((allactions)=>{
 
@@ -72,11 +80,15 @@ class SuperProducer {
                 masterTaskId,
                 allSubTasksId,
                 //mainRptFolder,
+                progressRootFolder,
+                resultsRootFolder,
+
                 taskFolder,
                 taskFolderNew,
                 taskFolderRunning,
                 taskFolderFinished,
                 taskFolderResult,
+                taskFolderResultFailed,
             });
         });
     }

package/example0/RunnerConsumer.js

@@ -1,18 +0,0 @@
-let SuperConsumer = require('../index').SuperConsumer;
-
-class TestConsumer extends SuperConsumer {
-  
-    process(taskInfo, actionData){
-        console.log(`processing data:`, taskInfo, actionData);
-        return {
-          task:{            
-            ...taskInfo, 
-            ...actionData,
-          },
-          result: {
-            message: 'this is a result from a demo, random data=' + Math.random()
-          }
-        }
-    }
-  }
-module.exports = TestConsumer;