multi-tasks 2.0.2 → 2.0.3

package/examples/example0/run.js

@@ -1,53 +1,53 @@
-let multiTasks = require('../../index').multiTasks;
-
-//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
-let alltasks = [];
-for(let i=0;i<50;i++){
-    let task_props = {
-        index: i,
-        name: `task-${i}`,
-        description: `This prop is for a subtask`
-    };
-    alltasks.push(task_props);
-}
-
-//Step2, Provide a process function to handle a certain sub-task and return the result data
-let processTask = (task, helper)=>{
-    //console.log('this is the subtask data you created above:', task);
-    let {taskCount} = task;
-
-    console.log(taskCount)
-
-
-    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
-    if(taskCount===3) throw 'exception';
-    if(taskCount===4) return Promise.reject({err:'a test error'});
-    if(taskCount===5) aaa =  bbb;
-
-    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is also OK
-
-    //by default, you should return a promise, all returned data can be found in the results/succ folder
-    return new Promise((resolve, reject)=>{
-        setTimeout(()=>{
-            if(taskCount % 2 === 0)helper.createNewTasks({//
-                msg:'a new task'
-            });
-            taskCount === 2 ? reject('a rejected error') : resolve({
-                message: 'this is a result from a demo, random data=' + Math.random()
-            });
-        }, 10 * Math.random())
-    })
-}
-
-//Step3, run!
-multiTasks.start({
-    tasks: alltasks, 
-    processTask, 
-    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
-    taskId: 'my-task',
-    numberOfWorkers: 3, //Assign how many workers are working in parallel
-    //autoCloseAfterCompletion: true,
-    onFinish: (report)=>{
-        console.log('finish callback', report);
-    }
+let multiTasks = require('../../index').multiTasks;
+
+//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
+let alltasks = [];
+for(let i=0;i<50;i++){
+    let task_props = {
+        index: i,
+        name: `task-${i}`,
+        description: `This prop is for a subtask`
+    };
+    alltasks.push(task_props);
+}
+
+//Step2, Provide a process function to handle a certain sub-task and return the result data
+let processTask = (task, helper)=>{
+    //console.log('this is the subtask data you created above:', task);
+    let {taskCount} = task;
+
+    console.log(taskCount)
+
+
+    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
+    if(taskCount===3) throw 'exception';
+    if(taskCount===4) return Promise.reject({err:'a test error'});
+    if(taskCount===5) aaa =  bbb;
+
+    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is also OK
+
+    //by default, you should return a promise, all returned data can be found in the results/succ folder
+    return new Promise((resolve, reject)=>{
+        setTimeout(()=>{
+            if(taskCount % 2 === 0)helper.createNewTasks({//
+                msg:'a new task'
+            });
+            taskCount === 2 ? reject('a rejected error') : resolve({
+                message: 'this is a result from a demo, random data=' + Math.random()
+            });
+        }, 10 * Math.random())
+    })
+}
+
+//Step3, run!
+multiTasks.start({
+    tasks: alltasks, 
+    processTask, 
+    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
+    taskId: 'my-task',
+    numberOfWorkers: 3, //Assign how many workers are working in parallel
+    //autoCloseAfterCompletion: true,
+    onFinish: (report)=>{
+        console.log('finish callback', report);
+    }
 });

package/examples/example1/RunnerConsumer.js

@@ -1,26 +1,26 @@
-let SuperConsumer = require('../../index').SuperConsumer;
-
-class TestConsumer extends SuperConsumer {
-  
-  processTask(actionData){
-        console.log(`processing data:`, actionData);
-        //you can also return a data instead of a promise object
-    let {index} = actionData;
-    if(index===3) throw 'exception';
-    if(index===4) return Promise.reject({err:'a test error'});
-        return new Promise((resolve) => {
-          setTimeout(() => {
-              let result = {
-                task:{            
-                  ...actionData,
-                },
-                result: {
-                  message: 'this is a result from a demo, random data=' + Math.random()
-                }
-              };
-              resolve(result);              
-          }, 100);
-      });
-    }
-  }
+let SuperConsumer = require('../../index').SuperConsumer;
+
+class TestConsumer extends SuperConsumer {
+  
+  processTask(actionData){
+        console.log(`processing data:`, actionData);
+        //you can also return a data instead of a promise object
+    let {index} = actionData;
+    if(index===3) throw 'exception';
+    if(index===4) return Promise.reject({err:'a test error'});
+        return new Promise((resolve) => {
+          setTimeout(() => {
+              let result = {
+                task:{            
+                  ...actionData,
+                },
+                result: {
+                  message: 'this is a result from a demo, random data=' + Math.random()
+                }
+              };
+              resolve(result);              
+          }, 100);
+      });
+    }
+  }
 module.exports = TestConsumer;

package/examples/example1/RunnerProducer.js

@@ -1,35 +1,35 @@
-let SuperProducer = require('../../index').SuperProducer;
-class TestProducer extends SuperProducer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    getAllActions() {
-        /**
-         * Generate all your tasks in this method, return as an array;
-         * These data will be allocated to multiple threads (workers) for running; 
-         * You can obtain each of the data and process it in the 'processTask' method of your consumer.
-         * 
-         * **/
-
-        return new Promise((resolve, reject)=>{
-            let trunkInfo = this.trunkInfo;
-            let alltasks = []
-            for(let i=0;i<200;i++){
-                let task_props = {
-                    index: i,
-                    name: `task-${i}`,
-                    description: `This prop is for a subtask`
-                };
-                alltasks.push(task_props);
-            }
-            resolve(alltasks);//return all your tasks as an array
-        })
-    }
-  }
+let SuperProducer = require('../../index').SuperProducer;
+class TestProducer extends SuperProducer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    getAllActions() {
+        /**
+         * Generate all your tasks in this method, return as an array;
+         * These data will be allocated to multiple threads (workers) for running; 
+         * You can obtain each of the data and process it in the 'processTask' method of your consumer.
+         * 
+         * **/
+
+        return new Promise((resolve, reject)=>{
+            let trunkInfo = this.trunkInfo;
+            let alltasks = []
+            for(let i=0;i<200;i++){
+                let task_props = {
+                    index: i,
+                    name: `task-${i}`,
+                    description: `This prop is for a subtask`
+                };
+                alltasks.push(task_props);
+            }
+            resolve(alltasks);//return all your tasks as an array
+        })
+    }
+  }
 module.exports = TestProducer;

package/examples/example1/run.js

@@ -1,9 +1,9 @@
-let RunnerProducer = require('./RunnerProducer');//You need to provide a Producer to return all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
-let RunnerConsumer = require('./RunnerConsumer');//You need to provide a Consumer to process a certain sub-task and return the processed data
-let master = require('../../index').master;
-
-master.start(RunnerProducer, RunnerConsumer, {
-    multi_task_parent_folder: `C:/tmp-demo`,
-    quotaOfEachTask:2, //The number of tasks assigned to each worker
-    numberOfWorkers: 6
+let RunnerProducer = require('./RunnerProducer');//You need to provide a Producer to return all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
+let RunnerConsumer = require('./RunnerConsumer');//You need to provide a Consumer to process a certain sub-task and return the processed data
+let master = require('../../index').master;
+
+master.start(RunnerProducer, RunnerConsumer, {
+    multi_task_parent_folder: `C:/tmp-demo`,
+    quotaOfEachTask:2, //The number of tasks assigned to each worker
+    numberOfWorkers: 6
 });

package/examples/example2/RunnerConsumer.js

@@ -1,40 +1,40 @@
-let SuperConsumer = require('../../index').SuperConsumer;
-
-
-let _do = (mainTaskInfo, actions, callback)=>{
-    if(actions.length === 0) return callback({
-        data:'subtask finished, random data='+Math.random()
-    });
-    let actionData = actions.shift();
-
-    //start to write your process code here
-
-    console.log('actionData', actionData)
-
-    //end of your process code.
-
-    //trigger next task
-    _do(mainTaskInfo, actions, callback);
-};
-
-
-class TestConsumer extends SuperConsumer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
-        let actions = subTaskInfo.actions;
-        mainTaskInfo.defination = this.trunkInfo.defination;
-        _do(mainTaskInfo, actions, (result)=>{
-            callback(result)
-        })
-
-    }
-  }
+let SuperConsumer = require('../../index').SuperConsumer;
+
+
+let _do = (mainTaskInfo, actions, callback)=>{
+    if(actions.length === 0) return callback({
+        data:'subtask finished, random data='+Math.random()
+    });
+    let actionData = actions.shift();
+
+    //start to write your process code here
+
+    console.log('actionData', actionData)
+
+    //end of your process code.
+
+    //trigger next task
+    _do(mainTaskInfo, actions, callback);
+};
+
+
+class TestConsumer extends SuperConsumer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
+        let actions = subTaskInfo.actions;
+        mainTaskInfo.defination = this.trunkInfo.defination;
+        _do(mainTaskInfo, actions, (result)=>{
+            callback(result)
+        })
+
+    }
+  }
 module.exports = TestConsumer;

package/examples/example2/RunnerProducer.js

@@ -1,35 +1,35 @@
-let SuperProducer = require('../../index').SuperProducer;
-class TestProducer extends SuperProducer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    getAllActions() {
-        /**
-         * Generate all your tasks in this method, return as an array;
-         * These data will be allocated to multiple threads (workers) for running; 
-         * You can obtain each of the data and process it in the 'processTask' method of your consumer.
-         * 
-         * **/
-
-        return new Promise((resolve, reject)=>{
-            let trunkInfo = this.trunkInfo;
-            let alltasks = []
-            for(let i=0;i<200;i++){
-                let task_props = {
-                    index: i,
-                    name: `task-${i}`,
-                    description: `This prop is for a subtask`
-                };
-                alltasks.push(task_props);
-            }
-            resolve(alltasks);//return all your tasks as an array
-        })
-    }
-  }
+let SuperProducer = require('../../index').SuperProducer;
+class TestProducer extends SuperProducer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    getAllActions() {
+        /**
+         * Generate all your tasks in this method, return as an array;
+         * These data will be allocated to multiple threads (workers) for running; 
+         * You can obtain each of the data and process it in the 'processTask' method of your consumer.
+         * 
+         * **/
+
+        return new Promise((resolve, reject)=>{
+            let trunkInfo = this.trunkInfo;
+            let alltasks = []
+            for(let i=0;i<200;i++){
+                let task_props = {
+                    index: i,
+                    name: `task-${i}`,
+                    description: `This prop is for a subtask`
+                };
+                alltasks.push(task_props);
+            }
+            resolve(alltasks);//return all your tasks as an array
+        })
+    }
+  }
 module.exports = TestProducer;

package/examples/example2/run.js

@@ -1,23 +1,23 @@
-let RunnerProducer = require('./RunnerProducer');//switch to require from node_modules in your project
-let RunnerConsumer = require('./RunnerConsumer');//switch to require from node_modules in your project
-
-let taskId = Math.random()+'';
-let multi_task_parent_folder = `C:/tmp-demo`;
-
-let producer = new RunnerProducer(`workers_of_${taskId}`, multi_task_parent_folder)
-let consumer = new RunnerConsumer();
-let master = require('../../index').master;
-
-console.log(producer.getTaskName())
-//return;
-
-let taskConfig = {
-    multi_task_parent_folder
-}
-//console.log(resp)
-producer.setTaskTrunkInfo(taskConfig);
-consumer.setTaskTrunkInfo(taskConfig)
-master.run(producer, consumer, {
-    quotaOfEachTask:2,
-    numberOfWorkers: 6
+let RunnerProducer = require('./RunnerProducer');//switch to require from node_modules in your project
+let RunnerConsumer = require('./RunnerConsumer');//switch to require from node_modules in your project
+
+let taskId = Math.random()+'';
+let multi_task_parent_folder = `C:/tmp-demo`;
+
+let producer = new RunnerProducer(`workers_of_${taskId}`, multi_task_parent_folder)
+let consumer = new RunnerConsumer();
+let master = require('../../index').master;
+
+console.log(producer.getTaskName())
+//return;
+
+let taskConfig = {
+    multi_task_parent_folder
+}
+//console.log(resp)
+producer.setTaskTrunkInfo(taskConfig);
+consumer.setTaskTrunkInfo(taskConfig)
+master.run(producer, consumer, {
+    quotaOfEachTask:2,
+    numberOfWorkers: 6
 });

package/index.js

@@ -1,4 +1,4 @@
-const multiTasks = require('./workers');
-module.exports = {
-    multiTasks
+const multiTasks = require('./workers');
+module.exports = {
+    multiTasks
 }

package/monitors/cli.js

@@ -1,28 +1,28 @@
-const cliProgress = require('cli-progress');
-
-// create new container
-const multibar = new cliProgress.MultiBar({
-    clearOnComplete: false,
-    hideCursor: true,
-    format: ' {bar} | {filename} | {value}/{total}',
-}, cliProgress.Presets.shades_grey);
-
-// add bars
-const b1 = multibar.create(200, 0);
-const b2 = multibar.create(1000, 0);
-
-// control bars
-b1.increment();
-b2.update(20, {filename: "test1.txt"});
-b1.update(20, {filename: "helloworld.txt"});
-
-b2.setTotal(2000);
-let v = 200;
-setInterval(()=>{
-    v=v+20
-    b2.update(v, {filename: "2222.txt"});
-
-    // stop all bars
-    
-    //multibar.stop();
-}, 300)
+const cliProgress = require('cli-progress');
+
+// create new container
+const multibar = new cliProgress.MultiBar({
+    clearOnComplete: false,
+    hideCursor: true,
+    format: ' {bar} | {filename} | {value}/{total}',
+}, cliProgress.Presets.shades_grey);
+
+// add bars
+const b1 = multibar.create(200, 0);
+const b2 = multibar.create(1000, 0);
+
+// control bars
+b1.increment();
+b2.update(20, {filename: "test1.txt"});
+b1.update(20, {filename: "helloworld.txt"});
+
+b2.setTotal(2000);
+let v = 200;
+setInterval(()=>{
+    v=v+20
+    b2.update(v, {filename: "2222.txt"});
+
+    // stop all bars
+    
+    //multibar.stop();
+}, 300)

package/package.json

@@ -1,23 +1,23 @@
-{
-  "name": "multi-tasks",
-  "version": "2.0.2",
-  "description": "",
-  "main": "index.js",
-  "directories": {
-    "example": "example"
-  },
-  "scripts": {
-    "test": "jest"
-  },
-  "keywords": [
-    "multitask",
-    "worker",
-    "multithread"
-  ],
-  "author": "zhanglei923@gmail.com",
-  "license": "MIT",
-  "devDependencies": {
-    "cli-progress": "^3.12.0",
-    "jest": "^29.7.0"
-  }
-}
+{
+  "name": "multi-tasks",
+  "version": "2.0.3",
+  "description": "",
+  "main": "index.js",
+  "directories": {
+    "example": "example"
+  },
+  "scripts": {
+    "test": "jest"
+  },
+  "keywords": [
+    "multitask",
+    "worker",
+    "multithread"
+  ],
+  "author": "zhanglei923@gmail.com",
+  "license": "MIT",
+  "devDependencies": {
+    "cli-progress": "^3.12.0",
+    "jest": "^29.7.0"
+  }
+}

package/test/splitter.test.js

@@ -1,17 +1,17 @@
-let splitter = require('../workers/taskSplitter');
-
-let createArray = (len)=>{  
-  let alltasks = []
-  for(let i=0;i<len;i++){
-      alltasks.push(i)
-  }
-  return alltasks;
-}
-
-test('test taskSplitter', async () => {
-    
-  expect(splitter.split(createArray(3), 5).length).toBe(1);
-  expect(splitter.split(createArray(16), 5).length).toBe(4);
-  expect(splitter.split(createArray(101), 1).length).toBe(101);
-  expect(splitter.split(createArray(101), 33).length).toBe(4);
+let splitter = require('../workers/taskSplitter');
+
+let createArray = (len)=>{  
+  let alltasks = []
+  for(let i=0;i<len;i++){
+      alltasks.push(i)
+  }
+  return alltasks;
+}
+
+test('test taskSplitter', async () => {
+    
+  expect(splitter.split(createArray(3), 5).length).toBe(1);
+  expect(splitter.split(createArray(16), 5).length).toBe(4);
+  expect(splitter.split(createArray(101), 1).length).toBe(101);
+  expect(splitter.split(createArray(101), 33).length).toBe(4);
 });

package/test/test_resume/step1_start_and_crash.js

@@ -0,0 +1,56 @@
+let multiTasks = require('../../index').multiTasks;
+
+//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
+let alltasks = [];
+for(let i=0;i<50;i++){
+    let task_props = {
+        index: i,
+        name: `task-${i}`,
+        description: `This prop is for a subtask`
+    };
+    alltasks.push(task_props);
+}
+
+//Step2, Provide a process function to handle a certain sub-task and return the result data
+let processTask = (task, helper)=>{
+    //console.log('this is the subtask data you created above:', task);
+    let {taskCount} = task;
+
+    console.log(taskCount)
+
+
+    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
+    if(taskCount===3) throw 'exception';
+    if(taskCount===4) return Promise.reject({err:'a test error'});
+    if(taskCount===5) aaa =  bbb;
+
+    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is also OK
+
+    //by default, you should return a promise, all returned data can be found in the results/succ folder
+    return new Promise((resolve, reject)=>{
+        setTimeout(()=>{
+            if(taskCount % 2 === 0)helper.createNewTasks({//
+                msg:'a new task'
+            });
+            taskCount === 2 ? reject('a rejected error') : resolve({
+                message: 'this is a result from a demo, random data=' + Math.random()
+            });
+        }, 10 * Math.random())
+    })
+}
+
+//Step3, run!
+multiTasks.start({
+    tasks: alltasks, 
+    processTask, 
+    taskRootFolder: `./tmp`, //a directory to store progress and results files, you can check the progress here
+    taskId: 'test_resume',
+    numberOfWorkers: 3, //Assign how many workers are working in parallel
+    //autoCloseAfterCompletion: true,
+    shouldTerminate:(info)=>{
+        if(info.taskCount > 30) return true;
+    },
+    onFinish: (report)=>{
+        console.log('finish callback', report);
+    }
+});