multi-tasks 1.2.0 → 1.2.2

package/README.md

@@ -12,14 +12,38 @@ How to use:
 ```javascript
 //see examples/example0
 let multiTasks = require('multi-tasks').multiTasks;
+//You need to provide the data of all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
+let alltasks = [];
+for(let i=0;i<12;i++){
+    let task_props = {
+        index: i,
+        name: `task-${i}`,
+        description: `This prop is for a subtask`
+    };
+    alltasks.push(task_props);
+}
+
+//You need to provide a process function to handle a certain sub-task and return the result data
+let processTask = (task)=>{
+    console.log('this is the subtask data you created above:', task);
+    let {index} = task;
+
+    //any exceptions will be captured
+    if(index===3) throw 'exception';
+    if(index===4) return Promise.reject({err:'a test error'});
+
+    //the returned data will be saved
+    return Promise.resolve({
+        message: 'this is a result from a demo, random data=' + Math.random()
+    });
+}
+
 multiTasks({
-    tasks: [...], //You need to provide an array containing all the subtasks you want to execute, multi-tasks will automatically split them and execute them in a multi-threaded manner
-    taskResolver:(task)=>{//You need to provide a taskResolver function to process a certain sub-task and return the result data
-        return Promise.resolve(data);
-    }, 
-    multi_task_parent_folder: `C:/tmp-demo`, //a directory to store progress and results files
-    quotaOfEachTask:2, //The number of tasks assigned to each worker
-    numberOfWorks: 6
+    tasks: alltasks, 
+    processTask, 
+    multi_task_parent_folder: `C:/tmp-demo`, //a directory to store progress and results files, you can check the progress here
+    quotaOfEachTask:1, //The number of subtasks assigned to each worker at one time, the default is 1
+    numberOfWorks: 6 //Assign how many workers are working in parallel
 });
 
 ```
@@ -47,6 +71,8 @@ node examples/example1/run
 
 Changelog:
 
+- 1.2.2 Update README and examples
+- 1.2.1 Handle exceptions and errors in subtasks
 - 1.2.0 Simplified usage by providing the function way and support return Promise
 - 1.1.4 Remove make-dir
 - 1.1.3 Simplified usage, see example0
@@ -56,3 +82,7 @@ Changelog:
 - 1.0.8 Fix examples
 - 1.0.7 Remove moment
 - 1.0.6 Performance optimization
+
+Github:
+
+[https://github.com/zhanglei923/multi-tasks](https://github.com/zhanglei923/multi-tasks)

package/examples/example0/run.js

@@ -1,7 +1,8 @@
 let multiTasks = require('../../index').multiTasks;
 
-let alltasks = []
-for(let i=0;i<200;i++){
+//You need to provide the data of all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
+let alltasks = [];
+for(let i=0;i<12;i++){
     let task_props = {
         index: i,
         name: `task-${i}`,
@@ -10,17 +11,25 @@ for(let i=0;i<200;i++){
     alltasks.push(task_props);
 }
 
-let taskResolver = (actionData)=>{
-    console.log('this is the actionData of one task', actionData);
+//You need to provide a process function to handle a certain sub-task and return the result data
+let processTask = (task)=>{
+    console.log('this is the subtask data you created above:', task);
+    let {index} = task;
+
+    //any exceptions will be captured
+    if(index===3) throw 'exception';
+    if(index===4) return Promise.reject({err:'a test error'});
+
+    //the returned data will be saved
     return Promise.resolve({
         message: 'this is a result from a demo, random data=' + Math.random()
     });
 }
 
 multiTasks({
-    tasks: alltasks, //You need to provide the data of all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
-    taskResolver, //You need to provide a taskResolver function to process a certain sub-task and return the result data
-    multi_task_parent_folder: `C:/tmp-demo`, //a directory to store progress and results files
-    quotaOfEachTask:2, //The number of tasks assigned to each worker
-    numberOfWorks: 6
+    tasks: alltasks, 
+    processTask, 
+    multi_task_parent_folder: `C:/tmp-demo`, //a directory to store progress and results files, you can check the progress here
+    quotaOfEachTask:1, //The number of subtasks assigned to each worker at one time, the default is 1
+    numberOfWorks: 6 //Assign how many workers are working in parallel
 });

package/examples/example1/RunnerConsumer.js

@@ -2,9 +2,12 @@ let SuperConsumer = require('../../index').SuperConsumer;
 
 class TestConsumer extends SuperConsumer {
   
-    process(actionData){
+  processTask(actionData){
         console.log(`processing data:`, actionData);
         //you can also return a data instead of a promise object
+    let {index} = actionData;
+    if(index===3) throw 'exception';
+    if(index===4) return Promise.reject({err:'a test error'});
         return new Promise((resolve) => {
           setTimeout(() => {
               let result = {

package/examples/example1/RunnerProducer.js

@@ -13,7 +13,7 @@ class TestProducer extends SuperProducer {
         /**
          * Generate all your tasks in this method, return as an array;
          * These data will be allocated to multiple threads (workers) for running; 
-         * You can obtain each of the data and process it in the 'process' method of your consumer.
+         * You can obtain each of the data and process it in the 'processTask' method of your consumer.
          * 
          * **/
 

package/examples/example2/RunnerProducer.js

@@ -13,7 +13,7 @@ class TestProducer extends SuperProducer {
         /**
          * Generate all your tasks in this method, return as an array;
          * These data will be allocated to multiple threads (workers) for running; 
-         * You can obtain each of the data and process it in the 'process' method of your consumer.
+         * You can obtain each of the data and process it in the 'processTask' method of your consumer.
          * 
          * **/
 

package/index.js

@@ -5,8 +5,8 @@ const multiTasks = (config)=>{
     if(!config.tasks){
         throw 'Please provide tasks data';
     };
-    if(!config.taskResolver){
-        throw 'Please provide taskResolver function';
+    if(!config.processTask){
+        throw 'Please provide processTask function';
     };
     class TestProducer extends SuperProducer {      
         // constructor
@@ -18,9 +18,9 @@ const multiTasks = (config)=>{
         }
     }
     class TestConsumer extends SuperConsumer {  
-        process(taskInfo, actionData){
-            console.log(`processing data:`, taskInfo, actionData);
-            let result = config.taskResolver(actionData);
+        processTask(taskProp){
+            console.log(`processing data:`, taskProp);
+            let result = config.processTask(taskProp);
             return Promise.resolve(result);
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "1.2.0",
+  "version": "1.2.2",
   "description": "",
   "main": "index.js",
   "directories": {

package/workers/SuperConsumer.js

@@ -12,38 +12,48 @@ class SuperConsumer {
     setTaskTrunkInfo(trunkInfo){
         this.trunkInfo = trunkInfo;
     }
-    _run (mainTaskInfo, subTaskInfo, actions, results, callback){
+    _run (mainTaskInfo, subTaskInfo, actions, results, errors, callback){
         if(actions.length === 0) return callback({
                                             finishedAt: new Date()*1,
                                             results,
+                                            errors,
                                             subTaskInfo
                                         });
-        let taskInfo = {
-            mainid: this.mainid,
-            subid: this.subid
-        };
-        let actionData = actions.shift();
+        let taskProp = actions.shift();
     
-        if(this.process){
-            let result = this.process(actionData);//rewrite this method in subclass
+        if(this.processTask){
+            let result;
+            let hasException = false;
+            try{
+                result = this.processTask(taskProp);//rewrite this method in subclass
+            }catch(e){
+                hasException = true;
+                result = {taskProp, exception: 'Exception:' + e.toString()}
+            }
             if(typeof result === 'undefined') result = {};
             let ispromise = result instanceof Promise;
-            if(! ispromise){
-                result = Promise.resolve(result);
-            }
-            result.then((data)=>{
-                results.push(data);
-                this._run(mainTaskInfo, subTaskInfo, actions, results, callback);
+            if(!ispromise){result = Promise.resolve(result);}
+            result.then((taskResult)=>{
+                if(!hasException){
+                    results.push({taskProp, taskResult});
+                }else{
+                    errors.push(taskResult);
+                }
+            }).catch((e)=>{
+                errors.push({taskProp, exception: 'Exception:' + e.toString()});
+            }).finally(()=>{
+                this._run(mainTaskInfo, subTaskInfo, actions, results, errors, callback);
             });            
         }else{
-            this._run(mainTaskInfo, subTaskInfo, actions, results, callback);
+            this._run(mainTaskInfo, subTaskInfo, actions, results, errors, callback);
         }    
     };
     run (mainTaskInfo, subTaskInfo, taskInfo, callback){
         let results = [];
+        let errors = [];
         let actions = subTaskInfo.actions;
         mainTaskInfo.defination = this.trunkInfo.defination;
-        this._run(mainTaskInfo, subTaskInfo, actions, results, (result)=>{
+        this._run(mainTaskInfo, subTaskInfo, actions, results, errors, (result)=>{
             callback(result)
         })
 
@@ -61,13 +71,18 @@ class SuperConsumer {
             let subid = subTaskInfo.subid;
             let taskfolderNew = mainTaskInfo.taskFolderNew;
             let taskFolderResult = mainTaskInfo.taskFolderResult;
+            let taskFolderResultFailed = mainTaskInfo.taskFolderResultFailed;            
             this.mvTask(mainid, subid,'new','running', (taskFile)=>{//mark as running
                 let taskContent = fs.readFileSync(taskFile, 'utf8');
                 let taskInfo = JSON.parse(taskContent);
                 this.run(mainTaskInfo, subTaskInfo, taskInfo, (taskReport)=>{//invoke
-                    let resultfile = pathutil.resolve(taskFolderResult, `${subid}.json`)
+                    let {results, errors} = taskReport;
+                    let hasError = false;
+                    if(errors &&  errors.length > 0) hasError = true;
+                    let folder = !hasError ? taskFolderResult : taskFolderResultFailed;
+                    let resultfile = pathutil.resolve(folder, `${subid}.json`)
                     fs.writeFileSync(resultfile, JSON.stringify(taskReport));
-                    this.mvTask(mainid, subid,'running','finished', ()=>{//mark as finished
+                    this.mvTask(mainid, subid,'running',(hasError ? 'finished_with_errors':'finished'), ()=>{//mark as finished
                         resolve({
                             mainid, 
                             subid,
@@ -79,7 +94,7 @@ class SuperConsumer {
         });
     }
     mvTask (mainid, subid, from, to, callback) {
-        let __thisdir = this.mainTaskInfo.taskFolder;
+        let __thisdir = this.mainTaskInfo.progressRootFolder;
         let oldPath = pathutil.resolve(__thisdir, `./${from}/${subid}.task.json`);
         let newPath = pathutil.resolve(__thisdir, `./${to}/${subid}.task.json`);
         fs.rename(oldPath, newPath, (err) => {

package/workers/SuperProducer.js

@@ -39,17 +39,25 @@ class SuperProducer {
         let taskFolder = this.taskfolder;
         //let taskfolder = pathutil.parse(__filename).dir;
         //let taskFolder = pathutil.resolve(taskfolder, `./tasks/${masterTaskId}`);
-        let taskFolderNew = pathutil.resolve(taskFolder, './new');
-        let taskFolderRunning = pathutil.resolve(taskFolder, './running');
-        let taskFolderFinished = pathutil.resolve(taskFolder, './finished');
-        let taskFolderResult = pathutil.resolve(taskFolder, './result');
+        let progressRootFolder = pathutil.resolve(taskFolder, './progress');
+        let resultsRootFolder = pathutil.resolve(taskFolder, './results');
+
+        let taskFolderNew = pathutil.resolve(progressRootFolder, './new');
+        let taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
+        let taskFolderFinished = pathutil.resolve(progressRootFolder, './finished');
+        let taskFolderFailed= pathutil.resolve(progressRootFolder, './finished_with_errors');
+
+        let taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
+        let taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
         //let mainRptFolder = pathutil.resolve(taskfolder, './report');
     
         //makeDir.sync(mainRptFolder);
         makeDir.sync(taskFolderNew);
         makeDir.sync(taskFolderRunning);
         makeDir.sync(taskFolderFinished);
+        makeDir.sync(taskFolderFailed);
         makeDir.sync(taskFolderResult);
+        makeDir.sync(taskFolderResultFailed);
 
         this.getAllActions().then((allactions)=>{
 
@@ -72,11 +80,15 @@ class SuperProducer {
                 masterTaskId,
                 allSubTasksId,
                 //mainRptFolder,
+                progressRootFolder,
+                resultsRootFolder,
+
                 taskFolder,
                 taskFolderNew,
                 taskFolderRunning,
                 taskFolderFinished,
                 taskFolderResult,
+                taskFolderResultFailed,
             });
         });
     }