multi-tasks 2.0.4 → 2.0.6

package/README.md

@@ -14,7 +14,8 @@ How to use:
 let multiTasks = require('multi-tasks').multiTasks;
 
 
-//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
+//Step1, if you have tasks that need to be executed simultaneously, 
+//       provide them as an array, multi-tasks will automatically split them and execute.
 let alltasks = [];
 for(let i=0;i<50;i++){
     alltasks.push({
@@ -23,14 +24,44 @@ for(let i=0;i<50;i++){
     });
 };
 
-//Step2, Provide a function to process a certain sub-task and return the result data
+//Step2, provide a function to process a certain sub-task and return the result data
 let processTask = (task, helper)=>{
     let {taskCount} = task;
 
     return 'a result which is not a promise'; 
 };
 
-//or return a promise
+//Step3, run!
+multiTasks({
+    initialTasks: alltasks, 
+    processTask, 
+    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
+    taskId: 'my-task',
+    numberOfWorkers: 3, //how many workers are working in parallel
+    //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
+    shouldTerminate:(info)=>{
+        //return true if you need to terminate the whole process
+    },
+    onFinish: (report)=>{
+        console.log('finish callback', report);
+    }
+});
+
+```
+
+Have a try:
+
+```shell
+node examples/example0/run
+node examples/example1/run
+```
+
+More about the task processing function
+
+```javascript
+//Example1, return a promise for async processes,
+//          you can return a promise or a non-promise result, 
+//          all result data can be found in the results/succ folder
 let processTask = (task, helper)=>{
     let {taskCount} = task;
 
@@ -41,13 +72,13 @@ let processTask = (task, helper)=>{
     })
 };
 
-//or, dynamically create a new task if find something new while processing
+//Example2, dynamically create a new task while processing
 let processTask = (task, helper)=>{
     let {taskCount} = task;
 
     
     if(taskCount % 2 === 0){
-        helper.createNewTasks({//You can dynamically create a new task if find something new while processing
+        helper.createNewTasks({//dynamically create a new task if find something new while processing
             msg:'a new task'
         });
         return;
@@ -60,23 +91,27 @@ let processTask = (task, helper)=>{
     })
 };
 
-//or, this example shows how to generate/handle exceptions
+//Example3, generate/throw exceptions in a task method
 let processTask = (task, helper)=>{
     let {taskCount} = task;
 
     //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
     if(taskCount===3) throw 'exception';
-    if(taskCount===4) return Promise.reject({err:'a test error'});
-    if(taskCount===5) aaa =  bbb;
+    if(taskCount===4) return Promise.reject({err:'a test error'});//use Promise.reject method
+    if(taskCount===5) aaa =  bbb;//this undefined exception will be captured by multi-tasks
+
+    return {data:'succ'};
+}
+
 
-    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is OK
+//Example4, dynamically create a new task while processing
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
 
-    //by default, you should return a promise, all returned data can be found in the results/succ folder
-    //but to return a non-promise result is also OK, see above
     return new Promise((resolve, reject)=>{
         setTimeout(()=>{
             if(taskCount % 2 === 0){
-                helper.createNewTasks({//You can dynamically create a new task if find something new while processing
+                helper.createNewTasks({//create a new task if needed
                     msg:'a new task'
                 });
                 resolve()
@@ -88,34 +123,11 @@ let processTask = (task, helper)=>{
         }, 10)
     })
 }
-
-//Step3, run!
-multiTasks({
-    initialTasks: alltasks, 
-    processTask, 
-    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
-    taskId: 'my-task',
-    numberOfWorkers: 3, //Assign how many workers are working in parallel
-    //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
-    shouldTerminate:(info)=>{
-        //return true if you need to terminate the whole process
-    },
-    onFinish: (report)=>{
-        console.log('finish callback', report);
-    }
-});
-
-```
-
-Have a try:
-
-```shell
-node examples/example0/run
-node examples/example1/run
 ```
 
 Changelog:
 
+- 2.0.5 Update readme examples
 - 2.0.4 Support resume from a failed task
 - 2.0.3 Fix: mkdir bug on windows
 - 2.0.2 new feature: support shouldTerminate

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "2.0.4",
+  "version": "2.0.6",
   "description": "",
   "main": "index.js",
   "directories": {

package/examples/example1/RunnerConsumer.js

@@ -1,26 +0,0 @@
-let SuperConsumer = require('../../index').SuperConsumer;
-
-class TestConsumer extends SuperConsumer {
-  
-  processTask(actionData){
-        console.log(`processing data:`, actionData);
-        //you can also return a data instead of a promise object
-    let {index} = actionData;
-    if(index===3) throw 'exception';
-    if(index===4) return Promise.reject({err:'a test error'});
-        return new Promise((resolve) => {
-          setTimeout(() => {
-              let result = {
-                task:{            
-                  ...actionData,
-                },
-                result: {
-                  message: 'this is a result from a demo, random data=' + Math.random()
-                }
-              };
-              resolve(result);              
-          }, 100);
-      });
-    }
-  }
-module.exports = TestConsumer;

package/examples/example1/RunnerProducer.js

@@ -1,35 +0,0 @@
-let SuperProducer = require('../../index').SuperProducer;
-class TestProducer extends SuperProducer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    getAllActions() {
-        /**
-         * Generate all your tasks in this method, return as an array;
-         * These data will be allocated to multiple threads (workers) for running; 
-         * You can obtain each of the data and process it in the 'processTask' method of your consumer.
-         * 
-         * **/
-
-        return new Promise((resolve, reject)=>{
-            let trunkInfo = this.trunkInfo;
-            let alltasks = []
-            for(let i=0;i<200;i++){
-                let task_props = {
-                    index: i,
-                    name: `task-${i}`,
-                    description: `This prop is for a subtask`
-                };
-                alltasks.push(task_props);
-            }
-            resolve(alltasks);//return all your tasks as an array
-        })
-    }
-  }
-module.exports = TestProducer;

package/examples/example1/run.js

@@ -1,9 +0,0 @@
-let RunnerProducer = require('./RunnerProducer');//You need to provide a Producer to return all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
-let RunnerConsumer = require('./RunnerConsumer');//You need to provide a Consumer to process a certain sub-task and return the processed data
-let master = require('../../index').master;
-
-master.start(RunnerProducer, RunnerConsumer, {
-    multi_task_parent_folder: `C:/tmp-demo`,
-    quotaOfEachTask:2, //The number of tasks assigned to each worker
-    numberOfWorkers: 6
-});

package/examples/example2/RunnerConsumer.js

@@ -1,40 +0,0 @@
-let SuperConsumer = require('../../index').SuperConsumer;
-
-
-let _do = (mainTaskInfo, actions, callback)=>{
-    if(actions.length === 0) return callback({
-        data:'subtask finished, random data='+Math.random()
-    });
-    let actionData = actions.shift();
-
-    //start to write your process code here
-
-    console.log('actionData', actionData)
-
-    //end of your process code.
-
-    //trigger next task
-    _do(mainTaskInfo, actions, callback);
-};
-
-
-class TestConsumer extends SuperConsumer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
-        let actions = subTaskInfo.actions;
-        mainTaskInfo.defination = this.trunkInfo.defination;
-        _do(mainTaskInfo, actions, (result)=>{
-            callback(result)
-        })
-
-    }
-  }
-module.exports = TestConsumer;

package/examples/example2/RunnerProducer.js

@@ -1,35 +0,0 @@
-let SuperProducer = require('../../index').SuperProducer;
-class TestProducer extends SuperProducer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    getAllActions() {
-        /**
-         * Generate all your tasks in this method, return as an array;
-         * These data will be allocated to multiple threads (workers) for running; 
-         * You can obtain each of the data and process it in the 'processTask' method of your consumer.
-         * 
-         * **/
-
-        return new Promise((resolve, reject)=>{
-            let trunkInfo = this.trunkInfo;
-            let alltasks = []
-            for(let i=0;i<200;i++){
-                let task_props = {
-                    index: i,
-                    name: `task-${i}`,
-                    description: `This prop is for a subtask`
-                };
-                alltasks.push(task_props);
-            }
-            resolve(alltasks);//return all your tasks as an array
-        })
-    }
-  }
-module.exports = TestProducer;

package/examples/example2/run.js

@@ -1,23 +0,0 @@
-let RunnerProducer = require('./RunnerProducer');//switch to require from node_modules in your project
-let RunnerConsumer = require('./RunnerConsumer');//switch to require from node_modules in your project
-
-let taskId = Math.random()+'';
-let multi_task_parent_folder = `C:/tmp-demo`;
-
-let producer = new RunnerProducer(`workers_of_${taskId}`, multi_task_parent_folder)
-let consumer = new RunnerConsumer();
-let master = require('../../index').master;
-
-console.log(producer.getTaskName())
-//return;
-
-let taskConfig = {
-    multi_task_parent_folder
-}
-//console.log(resp)
-producer.setTaskTrunkInfo(taskConfig);
-consumer.setTaskTrunkInfo(taskConfig)
-master.run(producer, consumer, {
-    quotaOfEachTask:2,
-    numberOfWorkers: 6
-});