multi-tasks 1.2.8 → 2.0.0

package/README.md

@@ -12,60 +12,98 @@ How to use:
 ```javascript
 //see examples/example0
 let multiTasks = require('multi-tasks').multiTasks;
-//You need to provide the data of all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
-let alltasks = [];
-for(let i=0;i<100;i++){
-    let task_props = {
-        index: i,
-        name: `task-${i}`,
-        description: `This prop is for a subtask`
-    };
-    alltasks.push(task_props);
-}
-
-//You need to provide a process function to handle a certain sub-task and return the result data
-let processTask = (task)=>{
-    console.log('this is the subtask data you created above:', task);
-    let {index} = task;
 
-    //any exceptions will be captured
-    if(index===3) throw 'exception';
-    if(index===4) return Promise.reject({err:'a test error'});
 
-    //the returned data will be saved
-    return Promise.resolve({
-        message: 'this is a result from a demo, random data=' + Math.random()
+//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
+let alltasks = [];
+for(let i=0;i<50;i++){
+    alltasks.push({
+        name: `task-${i}`,
+        data: `This prop is for a subtask`
     });
+};
+
+//Step2, Provide a function to process a certain sub-task and return the result data
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    return 'a result which is not a promise'; 
+};
+
+//Step2.1, or return a promise
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    return new Promise((resolve, reject)=>{
+        resolve({
+            data:`task${taskCount} complete`
+        })
+    })
+};
+
+//Step2.2, dynamically create a new task if find something new while processing
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    
+    if(taskCount % 2 === 0){
+        helper.createNewTasks({//You can dynamically create a new task if find something new while processing
+            msg:'a new task'
+        });
+        return;
+    }
+
+    return new Promise((resolve, reject)=>{
+        resolve({
+            data:`task${taskCount} complete`
+        })
+    })
+};
+
+//Step2.3, this example shows how to handle exceptions/errors
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
+    if(taskCount===3) throw 'exception';
+    if(taskCount===4) return Promise.reject({err:'a test error'});
+    if(taskCount===5) aaa =  bbb;
+
+    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is OK
+
+    //by default, you should return a promise, all returned data can be found in the results/succ folder
+    //but to return a non-promise result is also OK, see above
+    return new Promise((resolve, reject)=>{
+        setTimeout(()=>{
+            if(taskCount % 2 === 0){
+                helper.createNewTasks({//You can dynamically create a new task if find something new while processing
+                    msg:'a new task'
+                });
+                resolve()
+            }else{
+                resolve({
+                    data:`task${taskCount} complete`
+                })
+            }
+        }, 10)
+    })
 }
 
+//Step3, run!
 multiTasks({
     tasks: alltasks, 
     processTask, 
-    multi_task_parent_folder: `C:/tmp-demo`, //a directory to store progress and results files, you can check the progress here
-    quotaOfEachTask:1, //The number of subtasks assigned to each worker at one time, the default is 1
-    numberOfWorkers: 6, //Assign how many workers are working in parallel, default is the number of cpu cores minus one
+    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
+    taskId: 'my-task',
+    numberOfWorkers: 3, //Assign how many workers are working in parallel
+    //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
     onFinish: (report)=>{
-        console.log(report);//the report can be viewed here
+        console.log('finish callback', report);
     }
 });
 
 ```
 
-Or the older way, to provide the Producer & Consumer
-
-```javascript
-let master = require('multi-tasks').master;
-let RunnerProducer;//See examples, please provide a Producer class to specify all tasks you have, multi-tasks will automatically split them and execute them in a multi-threaded manner
-let RunnerConsumer;//See examples, please provide a Consumer class to process a certain sub-task and return the processed data
-
-master.start(RunnerProducer, RunnerConsumer, {
-    multi_task_parent_folder,
-    quotaOfEachTask,
-    numberOfWorkers,
-    onFinish
-});
-```
-
 Have a try:
 
 ```shell
@@ -75,6 +113,7 @@ node examples/example1/run
 
 Changelog:
 
+- 2.0.0 Rewritten with a new architecture to support dynamic tasks.
 - 1.2.8 Fix: create task folder failed on MacOS
 - 1.2.7 Small updates
 - 1.2.6 Support onFinish event

package/cmd.js

@@ -0,0 +1,42 @@
+import chalk from 'chalk';
+import readline from 'readline';
+
+
+// 初始化数值  
+let values = [0, 0, 0];  
+  
+// 更新并显示数值的函数  
+function updateAndDisplayValues() {  
+  // 增加一些随机值  
+  values = values.map(value => value + Math.floor(Math.random() * 10));  
+  
+  // 清除当前行并将光标移回行首  
+  process.stdout.clearLine();  
+  process.stdout.cursorTo(0);  
+  
+  // 显示更新后的数值，不换行  
+  process.stdout.write(chalk.green(`Value 1: ${values[0]}\t`));  
+  process.stdout.write(chalk.yellow(`Value 2: ${values[1]}\t`));  
+  process.stdout.write(chalk.blue(`Value 3: ${values[2]}\r`)); // \r 将光标移回行首，准备下一次输出  
+}  
+  
+// 创建readline接口实例以监听键盘输入  
+const rl = readline.createInterface({  
+  input: process.stdin,  
+  output: process.stdout  
+});  
+  
+// 每秒更新和显示数值  
+setInterval(updateAndDisplayValues, 1000);  
+  
+// 监听'q'键退出  
+rl.on('line', (input) => {  
+  if (input.trim().toLowerCase() === 'q') {  
+    clearInterval(intervalId); // 清除定时器  
+    rl.close(); // 关闭readline接口  
+    process.exit(0); // 退出程序  
+  }  
+});  
+  
+// 存储定时器的ID，以便稍后清除它  
+let intervalId = setInterval(updateAndDisplayValues, 1000);

package/examples/example0/run-readme.js

@@ -0,0 +1,88 @@
+let multiTasks = require('../../index').multiTasks;
+
+//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
+let alltasks = [];
+for(let i=0;i<50;i++){
+    alltasks.push({
+        name: `task-${i}`,
+        data: `This prop is for a subtask`
+    });
+};
+
+//Step2, Provide a function to process a certain sub-task and return the result data
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    return 'a result which is not a promise'; 
+};
+
+//Step2.1, or return a promise
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    return new Promise((resolve, reject)=>{
+        resolve({
+            data:`task${taskCount} complete`
+        })
+    })
+};
+
+//Step2.2, dynamically create a new task if find something new while processing
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    
+    if(taskCount % 2 === 0){
+        helper.createNewTasks({//You can dynamically create a new task if find something new while processing
+            msg:'a new task'
+        });
+        return;
+    }
+
+    return new Promise((resolve, reject)=>{
+        resolve({
+            data:`task${taskCount} complete`
+        })
+    })
+};
+//Step2.3, this example shows how to handle exceptions/errors
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
+    if(taskCount===3) throw 'exception';
+    if(taskCount===4) return Promise.reject({err:'a test error'});
+    if(taskCount===5) aaa =  bbb;
+
+    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is OK
+
+    //by default, you should return a promise, all returned data can be found in the results/succ folder
+    //but to return a non-promise result is also OK, see above
+    return new Promise((resolve, reject)=>{
+        setTimeout(()=>{
+            if(taskCount % 2 === 0){
+                helper.createNewTasks({//You can dynamically create a new task if find something new while processing
+                    msg:'a new task'
+                });
+                resolve()
+            }else{
+                resolve({
+                    data:`task${taskCount} complete`
+                })
+            }
+        }, 10)
+    })
+}
+
+//Step3, run!
+multiTasks({
+    tasks: alltasks, 
+    processTask, 
+    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
+    taskId: 'my-task',
+    numberOfWorkers: 3, //Assign how many workers are working in parallel
+    //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
+    onFinish: (report)=>{
+        console.log('finish callback', report);
+    }
+});

package/examples/example0/run.js

@@ -1,8 +1,8 @@
 let multiTasks = require('../../index').multiTasks;
 
-//You need to provide the data of all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
+//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
 let alltasks = [];
-for(let i=0;i<100;i++){
+for(let i=0;i<50;i++){
     let task_props = {
         index: i,
         name: `task-${i}`,
@@ -11,28 +11,43 @@ for(let i=0;i<100;i++){
     alltasks.push(task_props);
 }
 
-//You need to provide a process function to handle a certain sub-task and return the result data
-let processTask = (task)=>{
-    console.log('this is the subtask data you created above:', task);
-    let {index} = task;
+//Step2, Provide a process function to handle a certain sub-task and return the result data
+let processTask = (task, helper)=>{
+    //console.log('this is the subtask data you created above:', task);
+    let {taskCount} = task;
 
-    //any exceptions will be captured
-    if(index===3) throw 'exception';
-    if(index===4) return Promise.reject({err:'a test error'});
+    console.log(taskCount)
 
-    //the returned data will be saved
-    return Promise.resolve({
-        message: 'this is a result from a demo, random data=' + Math.random()
-    });
+
+    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
+    if(taskCount===3) throw 'exception';
+    if(taskCount===4) return Promise.reject({err:'a test error'});
+    if(taskCount===5) aaa =  bbb;
+
+    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is also OK
+
+    //by default, you should return a promise, all returned data can be found in the results/succ folder
+    return new Promise((resolve, reject)=>{
+        setTimeout(()=>{
+            if(taskCount % 2 === 0)helper.createNewTasks({//
+                msg:'a new task'
+            });
+            taskCount === 2 ? reject('a rejected error') : resolve({
+                message: 'this is a result from a demo, random data=' + Math.random()
+            });
+        }, 10 * Math.random())
+    })
 }
 
+//Step3, run!
 multiTasks({
     tasks: alltasks, 
     processTask, 
-    multi_task_parent_folder: `C:/tmp-demo`, //a directory to store progress and results files, you can check the progress here
-    quotaOfEachTask:1, //The number of subtasks assigned to each worker at one time, the default is 1
-    numberOfWorkers: 6, //Assign how many workers are working in parallel
+    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
+    taskId: 'my-task',
+    numberOfWorkers: 3, //Assign how many workers are working in parallel
+    //autoCloseAfterCompletion: true,
     onFinish: (report)=>{
-        console.log(report);
+        console.log('finish callback', report);
     }
 });

package/index.js

@@ -1,37 +1,4 @@
-const SuperProducer = require('./workers/SuperProducer');
-const SuperConsumer = require('./workers/SuperConsumer');
-const master = require('./workers/master');
-const multiTasks = (config)=>{
-    if(!config.tasks){
-        throw 'Please provide tasks data';
-    };
-    if(!config.processTask){
-        throw 'Please provide processTask function';
-    };
-    class TestProducer extends SuperProducer {      
-        // constructor
-        setTaskTrunkInfo(trunkInfo){
-            this.trunkInfo = trunkInfo;
-        }
-        getAllActions() {
-            return Promise.resolve(config.tasks);//return all your tasks as an array
-        }
-    }
-    class TestConsumer extends SuperConsumer {  
-        processTask(taskProp){
-            console.log(`processing data:`, taskProp);
-            let result = config.processTask(taskProp);
-            return Promise.resolve(result);
-        }
-    }
-
-    master.start(TestProducer, TestConsumer, {
-        ...config
-    });
-};
+const multiTasks = require('./workers');
 module.exports = {
-    SuperProducer,
-    SuperConsumer,
-    master,
     multiTasks
 }

package/monitors/cli.js

@@ -0,0 +1,28 @@
+const cliProgress = require('cli-progress');
+
+// create new container
+const multibar = new cliProgress.MultiBar({
+    clearOnComplete: false,
+    hideCursor: true,
+    format: ' {bar} | {filename} | {value}/{total}',
+}, cliProgress.Presets.shades_grey);
+
+// add bars
+const b1 = multibar.create(200, 0);
+const b2 = multibar.create(1000, 0);
+
+// control bars
+b1.increment();
+b2.update(20, {filename: "test1.txt"});
+b1.update(20, {filename: "helloworld.txt"});
+
+b2.setTotal(2000);
+let v = 200;
+setInterval(()=>{
+    v=v+20
+    b2.update(v, {filename: "2222.txt"});
+
+    // stop all bars
+    
+    //multibar.stop();
+}, 300)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "1.2.8",
+  "version": "2.0.0",
   "description": "",
   "main": "index.js",
   "directories": {
@@ -17,6 +17,7 @@
   "author": "zhanglei923@gmail.com",
   "license": "MIT",
   "devDependencies": {
+    "cli-progress": "^3.12.0",
     "jest": "^29.7.0"
   }
 }

package/todo.txt

@@ -0,0 +1,4 @@
+Features:
+  - Continue execution from the point of interruption
+  - Re-run
+  - Show status in command line

package/utils/asyncQueue.js

@@ -0,0 +1,32 @@
+class AsyncQueue {  
+    constructor() {  
+      this.queue = [];  
+      this.isProcessing = false;  
+    }  
+    enqueue(task) {      
+      this.queue.push(task);  
+      
+      if (!this.isProcessing) {  
+        this.processNext();  
+      } 
+    }  
+    async processNext() {  
+      this.isProcessing = true;  
+      if (this.queue.length === 0) {  
+        this.isProcessing = false;  
+        return;  
+      }  
+      const task = this.queue.shift();  
+      let result = task();
+      if(!result instanceof Promise) result = Promise.resolve(result);
+    
+      try {  
+        await result; 
+      } catch (error) {  
+        console.error('Error processing task:', error);  
+      }  
+      this.processNext(); // 处理完一个任务后，继续处理下一个任务  
+    }  
+  }  
+
+module.exports = AsyncQueue;

package/utils/backward-v1.js

@@ -0,0 +1,11 @@
+function v1tov2(config) {      
+    if(config.multi_task_parent_folder) config.taskRootFolder = config.multi_task_parent_folder;//backward compatible
+    delete config.multi_task_parent_folder;
+    
+    if(config.numberOfWorks && !config.numberOfWorkers) config.numberOfWorkers = config.numberOfWorks; //Backward compatible with the wrong variable name "numberOfWorks"
+
+
+    return config;
+}  
+
+module.exports = {v1tov2};

package/utils/makedir.js

@@ -25,5 +25,31 @@ function sync(directoryPath) {
         }
     }
 }
+function getDirectFiles(dir, callback) {  
+    let childrenfiles = [];
+    if(!fs.existsSync(dir)){
+        console.log('FATAL: directory is missing:', dir);
+        return process.exit(1);        
+    }
+    fs.readdir(dir, (err, files) => {  
+        if (err) {  
+            console.error("Could not list the directory.", err);  
+            return callback([]);
+        }
+        files.forEach(file => {  
+            const filePath = path.join(dir, file);  
+            //console.log(filePath)
+            try{
+                let stats = fs.statSync(filePath);
+                if (stats.isFile()) {  
+                    childrenfiles.push(file); 
+                }  
+            }catch(e){
+                //
+            }
+        });
+        callback(childrenfiles);
+    });  
+}  
 
-module.exports = {sync};
+module.exports = {sync, getDirectFiles};

package/utils/randoms.js

@@ -0,0 +1,15 @@
+
+const fmtDigit = (n)=>{
+    return n > 9 ? "" + n: "0" + n;
+}
+const getDateTimeTxt = ()=>{
+    let d=new Date();
+    let YYYY=d.getFullYear()
+    let MM=fmtDigit(d.getMonth()+1);
+    let DD=fmtDigit(d.getDate());
+    let HH=fmtDigit(d.getHours());
+    let mm=fmtDigit(d.getMinutes());
+    let ss=fmtDigit(d.getSeconds());
+    return `${YYYY}-${MM}-${DD}_${HH}-${mm}-${ss}`;
+};
+module.exports = {getDateTimeTxt};

package/utils/test/test_asyncqueue.js

@@ -0,0 +1,116 @@
+const AsyncQueue = require(`../asyncQueue`);
+const queue = new AsyncQueue();  
+
+let count=0;
+  
+queue.enqueue(async () => {
+    count++;  
+  console.log(`Task ${count} start`);  
+  await new Promise(resolve => setTimeout(resolve, 100)); 
+  console.log(`Task ${count} end`);  
+});
+ 
+  
+queue.enqueue(async () => {
+    count++;  
+  console.log(`Task ${count} start`);  
+  await new Promise(resolve => setTimeout(resolve, 20)); 
+  console.log(`Task ${count} end`);  
+});
+ 
+  
+queue.enqueue(() => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    console.log(`Task ${count} end`);  
+    return 11;
+  });
+      
+  
+queue.enqueue(() => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    console.log(`Task ${count} end`);  
+    return 11;
+  });   
+  
+  queue.enqueue(() => {
+      count++;  
+      console.log(`Task ${count} start`);  
+      console.log(`Task ${count} end`);  
+      return 11;
+    });   
+  
+    queue.enqueue(() => {
+        count++;  
+        console.log(`Task ${count} start`);  
+        console.log(`Task ${count} end`);  
+        return 11;
+      });   
+  
+      queue.enqueue(() => {
+          count++;  
+          console.log(`Task ${count} start`);  
+          console.log(`Task ${count} end`);  
+          return 11;
+        });   
+  
+        queue.enqueue(() => {
+            count++;  
+            console.log(`Task ${count} start`);  
+            console.log(`Task ${count} end`);  
+            return 11;
+          });
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+  
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+  
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+  
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+  
+queue.enqueue(() => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    console.log(`Task ${count} end`);  
+    return 11;
+  });   
+  
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+            
+