multi-tasks 1.2.7 → 2.0.0

package/LICENSE

@@ -1,21 +1,21 @@
-MIT License
-
-Copyright (c) 2024 Lei Zhang
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+MIT License
+
+Copyright (c) 2024 Lei Zhang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,97 +1,137 @@
-# multi-tasks
-multi-tasks
-
-Install:
-
-```javascript
-npm install multi-tasks
-```
-
-How to use:
-
-```javascript
-//see examples/example0
-let multiTasks = require('multi-tasks').multiTasks;
-//You need to provide the data of all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
-let alltasks = [];
-for(let i=0;i<100;i++){
-    let task_props = {
-        index: i,
-        name: `task-${i}`,
-        description: `This prop is for a subtask`
-    };
-    alltasks.push(task_props);
-}
-
-//You need to provide a process function to handle a certain sub-task and return the result data
-let processTask = (task)=>{
-    console.log('this is the subtask data you created above:', task);
-    let {index} = task;
-
-    //any exceptions will be captured
-    if(index===3) throw 'exception';
-    if(index===4) return Promise.reject({err:'a test error'});
-
-    //the returned data will be saved
-    return Promise.resolve({
-        message: 'this is a result from a demo, random data=' + Math.random()
-    });
-}
-
-multiTasks({
-    tasks: alltasks, 
-    processTask, 
-    multi_task_parent_folder: `C:/tmp-demo`, //a directory to store progress and results files, you can check the progress here
-    quotaOfEachTask:1, //The number of subtasks assigned to each worker at one time, the default is 1
-    numberOfWorkers: 6, //Assign how many workers are working in parallel, default is the number of cpu cores minus one
-    onFinish: (report)=>{
-        console.log(report);//the report can be viewed here
-    }
-});
-
-```
-
-Or the older way, to provide the Producer & Consumer
-
-```javascript
-let master = require('multi-tasks').master;
-let RunnerProducer;//See examples, please provide a Producer class to specify all tasks you have, multi-tasks will automatically split them and execute them in a multi-threaded manner
-let RunnerConsumer;//See examples, please provide a Consumer class to process a certain sub-task and return the processed data
-
-master.start(RunnerProducer, RunnerConsumer, {
-    multi_task_parent_folder,
-    quotaOfEachTask,
-    numberOfWorkers,
-    onFinish
-});
-```
-
-Have a try:
-
-```shell
-node examples/example0/run
-node examples/example1/run
-```
-
-Changelog:
-
-- 1.2.7 Small updates
-- 1.2.6 Support onFinish event
-- 1.2.5 Rename numberOfWorks to numberOfWorkers, the old one are still supported ;-)
-- 1.2.4 Fix: opt.numberOfWorkers not work
-- 1.2.3 Update README
-- 1.2.2 Update README and examples
-- 1.2.1 Handle exceptions and errors in subtasks
-- 1.2.0 Simplified usage by providing the function way and support return Promise
-- 1.1.4 Remove make-dir
-- 1.1.3 Simplified usage, see example0
-- 1.1.2 
-- 1.1.1 Rename files, updated changelog
-- 1.1.0 Simplified the usage of a customized Consumer, see example0
-- 1.0.8 Fix examples
-- 1.0.7 Remove moment
-- 1.0.6 Performance optimization
-
-Github:
-
-[https://github.com/zhanglei923/multi-tasks](https://github.com/zhanglei923/multi-tasks)
+# multi-tasks
+multi-tasks
+
+Install:
+
+```javascript
+npm install multi-tasks
+```
+
+How to use:
+
+```javascript
+//see examples/example0
+let multiTasks = require('multi-tasks').multiTasks;
+
+
+//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
+let alltasks = [];
+for(let i=0;i<50;i++){
+    alltasks.push({
+        name: `task-${i}`,
+        data: `This prop is for a subtask`
+    });
+};
+
+//Step2, Provide a function to process a certain sub-task and return the result data
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    return 'a result which is not a promise'; 
+};
+
+//Step2.1, or return a promise
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    return new Promise((resolve, reject)=>{
+        resolve({
+            data:`task${taskCount} complete`
+        })
+    })
+};
+
+//Step2.2, dynamically create a new task if find something new while processing
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    
+    if(taskCount % 2 === 0){
+        helper.createNewTasks({//You can dynamically create a new task if find something new while processing
+            msg:'a new task'
+        });
+        return;
+    }
+
+    return new Promise((resolve, reject)=>{
+        resolve({
+            data:`task${taskCount} complete`
+        })
+    })
+};
+
+//Step2.3, this example shows how to handle exceptions/errors
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
+    if(taskCount===3) throw 'exception';
+    if(taskCount===4) return Promise.reject({err:'a test error'});
+    if(taskCount===5) aaa =  bbb;
+
+    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is OK
+
+    //by default, you should return a promise, all returned data can be found in the results/succ folder
+    //but to return a non-promise result is also OK, see above
+    return new Promise((resolve, reject)=>{
+        setTimeout(()=>{
+            if(taskCount % 2 === 0){
+                helper.createNewTasks({//You can dynamically create a new task if find something new while processing
+                    msg:'a new task'
+                });
+                resolve()
+            }else{
+                resolve({
+                    data:`task${taskCount} complete`
+                })
+            }
+        }, 10)
+    })
+}
+
+//Step3, run!
+multiTasks({
+    tasks: alltasks, 
+    processTask, 
+    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
+    taskId: 'my-task',
+    numberOfWorkers: 3, //Assign how many workers are working in parallel
+    //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
+    onFinish: (report)=>{
+        console.log('finish callback', report);
+    }
+});
+
+```
+
+Have a try:
+
+```shell
+node examples/example0/run
+node examples/example1/run
+```
+
+Changelog:
+
+- 2.0.0 Rewritten with a new architecture to support dynamic tasks.
+- 1.2.8 Fix: create task folder failed on MacOS
+- 1.2.7 Small updates
+- 1.2.6 Support onFinish event
+- 1.2.5 Rename numberOfWorks to numberOfWorkers, the old one are still supported ;-)
+- 1.2.4 Fix: opt.numberOfWorkers not work
+- 1.2.3 Update README
+- 1.2.2 Update README and examples
+- 1.2.1 Handle exceptions and errors in subtasks
+- 1.2.0 Simplified usage by providing the function way and support return Promise
+- 1.1.4 Remove make-dir
+- 1.1.3 Simplified usage, see example0
+- 1.1.2 
+- 1.1.1 Rename files, updated changelog
+- 1.1.0 Simplified the usage of a customized Consumer, see example0
+- 1.0.8 Fix examples
+- 1.0.7 Remove moment
+- 1.0.6 Performance optimization
+
+Github:
+
+[https://github.com/zhanglei923/multi-tasks](https://github.com/zhanglei923/multi-tasks)

package/cmd.js

@@ -0,0 +1,42 @@
+import chalk from 'chalk';
+import readline from 'readline';
+
+
+// 初始化数值  
+let values = [0, 0, 0];  
+  
+// 更新并显示数值的函数  
+function updateAndDisplayValues() {  
+  // 增加一些随机值  
+  values = values.map(value => value + Math.floor(Math.random() * 10));  
+  
+  // 清除当前行并将光标移回行首  
+  process.stdout.clearLine();  
+  process.stdout.cursorTo(0);  
+  
+  // 显示更新后的数值，不换行  
+  process.stdout.write(chalk.green(`Value 1: ${values[0]}\t`));  
+  process.stdout.write(chalk.yellow(`Value 2: ${values[1]}\t`));  
+  process.stdout.write(chalk.blue(`Value 3: ${values[2]}\r`)); // \r 将光标移回行首，准备下一次输出  
+}  
+  
+// 创建readline接口实例以监听键盘输入  
+const rl = readline.createInterface({  
+  input: process.stdin,  
+  output: process.stdout  
+});  
+  
+// 每秒更新和显示数值  
+setInterval(updateAndDisplayValues, 1000);  
+  
+// 监听'q'键退出  
+rl.on('line', (input) => {  
+  if (input.trim().toLowerCase() === 'q') {  
+    clearInterval(intervalId); // 清除定时器  
+    rl.close(); // 关闭readline接口  
+    process.exit(0); // 退出程序  
+  }  
+});  
+  
+// 存储定时器的ID，以便稍后清除它  
+let intervalId = setInterval(updateAndDisplayValues, 1000);

package/examples/example0/run-readme.js

@@ -0,0 +1,88 @@
+let multiTasks = require('../../index').multiTasks;
+
+//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
+let alltasks = [];
+for(let i=0;i<50;i++){
+    alltasks.push({
+        name: `task-${i}`,
+        data: `This prop is for a subtask`
+    });
+};
+
+//Step2, Provide a function to process a certain sub-task and return the result data
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    return 'a result which is not a promise'; 
+};
+
+//Step2.1, or return a promise
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    return new Promise((resolve, reject)=>{
+        resolve({
+            data:`task${taskCount} complete`
+        })
+    })
+};
+
+//Step2.2, dynamically create a new task if find something new while processing
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    
+    if(taskCount % 2 === 0){
+        helper.createNewTasks({//You can dynamically create a new task if find something new while processing
+            msg:'a new task'
+        });
+        return;
+    }
+
+    return new Promise((resolve, reject)=>{
+        resolve({
+            data:`task${taskCount} complete`
+        })
+    })
+};
+//Step2.3, this example shows how to handle exceptions/errors
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
+    if(taskCount===3) throw 'exception';
+    if(taskCount===4) return Promise.reject({err:'a test error'});
+    if(taskCount===5) aaa =  bbb;
+
+    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is OK
+
+    //by default, you should return a promise, all returned data can be found in the results/succ folder
+    //but to return a non-promise result is also OK, see above
+    return new Promise((resolve, reject)=>{
+        setTimeout(()=>{
+            if(taskCount % 2 === 0){
+                helper.createNewTasks({//You can dynamically create a new task if find something new while processing
+                    msg:'a new task'
+                });
+                resolve()
+            }else{
+                resolve({
+                    data:`task${taskCount} complete`
+                })
+            }
+        }, 10)
+    })
+}
+
+//Step3, run!
+multiTasks({
+    tasks: alltasks, 
+    processTask, 
+    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
+    taskId: 'my-task',
+    numberOfWorkers: 3, //Assign how many workers are working in parallel
+    //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
+    onFinish: (report)=>{
+        console.log('finish callback', report);
+    }
+});

package/examples/example0/run.js

@@ -1,38 +1,53 @@
-let multiTasks = require('../../index').multiTasks;
-
-//You need to provide the data of all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
-let alltasks = [];
-for(let i=0;i<100;i++){
-    let task_props = {
-        index: i,
-        name: `task-${i}`,
-        description: `This prop is for a subtask`
-    };
-    alltasks.push(task_props);
-}
-
-//You need to provide a process function to handle a certain sub-task and return the result data
-let processTask = (task)=>{
-    console.log('this is the subtask data you created above:', task);
-    let {index} = task;
-
-    //any exceptions will be captured
-    if(index===3) throw 'exception';
-    if(index===4) return Promise.reject({err:'a test error'});
-
-    //the returned data will be saved
-    return Promise.resolve({
-        message: 'this is a result from a demo, random data=' + Math.random()
-    });
-}
-
-multiTasks({
-    tasks: alltasks, 
-    processTask, 
-    multi_task_parent_folder: `C:/tmp-demo`, //a directory to store progress and results files, you can check the progress here
-    quotaOfEachTask:1, //The number of subtasks assigned to each worker at one time, the default is 1
-    numberOfWorkers: 6, //Assign how many workers are working in parallel
-    onFinish: (report)=>{
-        console.log(report);
-    }
+let multiTasks = require('../../index').multiTasks;
+
+//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
+let alltasks = [];
+for(let i=0;i<50;i++){
+    let task_props = {
+        index: i,
+        name: `task-${i}`,
+        description: `This prop is for a subtask`
+    };
+    alltasks.push(task_props);
+}
+
+//Step2, Provide a process function to handle a certain sub-task and return the result data
+let processTask = (task, helper)=>{
+    //console.log('this is the subtask data you created above:', task);
+    let {taskCount} = task;
+
+    console.log(taskCount)
+
+
+    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
+    if(taskCount===3) throw 'exception';
+    if(taskCount===4) return Promise.reject({err:'a test error'});
+    if(taskCount===5) aaa =  bbb;
+
+    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is also OK
+
+    //by default, you should return a promise, all returned data can be found in the results/succ folder
+    return new Promise((resolve, reject)=>{
+        setTimeout(()=>{
+            if(taskCount % 2 === 0)helper.createNewTasks({//
+                msg:'a new task'
+            });
+            taskCount === 2 ? reject('a rejected error') : resolve({
+                message: 'this is a result from a demo, random data=' + Math.random()
+            });
+        }, 10 * Math.random())
+    })
+}
+
+//Step3, run!
+multiTasks({
+    tasks: alltasks, 
+    processTask, 
+    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
+    taskId: 'my-task',
+    numberOfWorkers: 3, //Assign how many workers are working in parallel
+    //autoCloseAfterCompletion: true,
+    onFinish: (report)=>{
+        console.log('finish callback', report);
+    }
 });

package/examples/example1/RunnerConsumer.js

@@ -1,26 +1,26 @@
-let SuperConsumer = require('../../index').SuperConsumer;
-
-class TestConsumer extends SuperConsumer {
-  
-  processTask(actionData){
-        console.log(`processing data:`, actionData);
-        //you can also return a data instead of a promise object
-    let {index} = actionData;
-    if(index===3) throw 'exception';
-    if(index===4) return Promise.reject({err:'a test error'});
-        return new Promise((resolve) => {
-          setTimeout(() => {
-              let result = {
-                task:{            
-                  ...actionData,
-                },
-                result: {
-                  message: 'this is a result from a demo, random data=' + Math.random()
-                }
-              };
-              resolve(result);              
-          }, 100);
-      });
-    }
-  }
+let SuperConsumer = require('../../index').SuperConsumer;
+
+class TestConsumer extends SuperConsumer {
+  
+  processTask(actionData){
+        console.log(`processing data:`, actionData);
+        //you can also return a data instead of a promise object
+    let {index} = actionData;
+    if(index===3) throw 'exception';
+    if(index===4) return Promise.reject({err:'a test error'});
+        return new Promise((resolve) => {
+          setTimeout(() => {
+              let result = {
+                task:{            
+                  ...actionData,
+                },
+                result: {
+                  message: 'this is a result from a demo, random data=' + Math.random()
+                }
+              };
+              resolve(result);              
+          }, 100);
+      });
+    }
+  }
 module.exports = TestConsumer;

package/examples/example1/RunnerProducer.js

@@ -1,35 +1,35 @@
-let SuperProducer = require('../../index').SuperProducer;
-class TestProducer extends SuperProducer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    getAllActions() {
-        /**
-         * Generate all your tasks in this method, return as an array;
-         * These data will be allocated to multiple threads (workers) for running; 
-         * You can obtain each of the data and process it in the 'processTask' method of your consumer.
-         * 
-         * **/
-
-        return new Promise((resolve, reject)=>{
-            let trunkInfo = this.trunkInfo;
-            let alltasks = []
-            for(let i=0;i<200;i++){
-                let task_props = {
-                    index: i,
-                    name: `task-${i}`,
-                    description: `This prop is for a subtask`
-                };
-                alltasks.push(task_props);
-            }
-            resolve(alltasks);//return all your tasks as an array
-        })
-    }
-  }
+let SuperProducer = require('../../index').SuperProducer;
+class TestProducer extends SuperProducer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    getAllActions() {
+        /**
+         * Generate all your tasks in this method, return as an array;
+         * These data will be allocated to multiple threads (workers) for running; 
+         * You can obtain each of the data and process it in the 'processTask' method of your consumer.
+         * 
+         * **/
+
+        return new Promise((resolve, reject)=>{
+            let trunkInfo = this.trunkInfo;
+            let alltasks = []
+            for(let i=0;i<200;i++){
+                let task_props = {
+                    index: i,
+                    name: `task-${i}`,
+                    description: `This prop is for a subtask`
+                };
+                alltasks.push(task_props);
+            }
+            resolve(alltasks);//return all your tasks as an array
+        })
+    }
+  }
 module.exports = TestProducer;

package/examples/example1/run.js

@@ -1,9 +1,9 @@
-let RunnerProducer = require('./RunnerProducer');//You need to provide a Producer to return all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
-let RunnerConsumer = require('./RunnerConsumer');//You need to provide a Consumer to process a certain sub-task and return the processed data
-let master = require('../../index').master;
-
-master.start(RunnerProducer, RunnerConsumer, {
-    multi_task_parent_folder: `C:/tmp-demo`,
-    quotaOfEachTask:2, //The number of tasks assigned to each worker
-    numberOfWorkers: 6
+let RunnerProducer = require('./RunnerProducer');//You need to provide a Producer to return all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
+let RunnerConsumer = require('./RunnerConsumer');//You need to provide a Consumer to process a certain sub-task and return the processed data
+let master = require('../../index').master;
+
+master.start(RunnerProducer, RunnerConsumer, {
+    multi_task_parent_folder: `C:/tmp-demo`,
+    quotaOfEachTask:2, //The number of tasks assigned to each worker
+    numberOfWorkers: 6
 });