multi-tasks 1.2.7 → 2.0.0

package/examples/example3-dynamicIncreasing/run.js

@@ -1,36 +1,36 @@
-let fs = require('fs');
-let pathutil = require('path');
-let makeDir = require('../utils/makedir');
-let RunnerProducer = require('./RunnerProducer');//switch to require from node_modules in your project
-let RunnerConsumer = require('./RunnerConsumer');//switch to require from node_modules in your project
-
-let thisroot = pathutil.parse(__filename).dir;
-let taskId = `workers_of_${Math.random()}`;
-let multi_task_parent_folder = pathutil.resolve(thisroot, `../../example2_dynamicly_increasing`);
-
-let producer = new RunnerProducer(taskId, multi_task_parent_folder)
-let consumer = new RunnerConsumer();
-let master = require('../index').master;
-
-console.log(producer.getTaskName())
-//return;
-
-let NUMBER_OF_WORKS = 5;//LOCK NUMBER OF WORKERS
-
-let taskConfig = {
-    NUMBER_OF_WORKS,
-    dynamicTasksFolderNew: pathutil.resolve(multi_task_parent_folder, `./dynamicTasks-NEW`),
-    dynamicTasksFolderSucc: pathutil.resolve(multi_task_parent_folder, `./dynamicTasks-SUCC`),
-    dynamicTasksFolderFail: pathutil.resolve(multi_task_parent_folder, `./dynamicTasks-FAIL`),
-}
-makeDir.sync(taskConfig.dynamicTasksFolderNew)
-makeDir.sync(taskConfig.dynamicTasksFolderSucc)
-makeDir.sync(taskConfig.dynamicTasksFolderFail)
-
-
-producer.setTaskTrunkInfo(taskConfig);
-consumer.setTaskTrunkInfo(taskConfig)
-master.run(producer, consumer, {
-    quotaOfEachTask:1, //<--Lock as one
-    numberOfWorkers:NUMBER_OF_WORKS
+let fs = require('fs');
+let pathutil = require('path');
+let makeDir = require('../utils/makedir');
+let RunnerProducer = require('./RunnerProducer');//switch to require from node_modules in your project
+let RunnerConsumer = require('./RunnerConsumer');//switch to require from node_modules in your project
+
+let thisroot = pathutil.parse(__filename).dir;
+let taskId = `workers_of_${Math.random()}`;
+let multi_task_parent_folder = pathutil.resolve(thisroot, `../../example2_dynamicly_increasing`);
+
+let producer = new RunnerProducer(taskId, multi_task_parent_folder)
+let consumer = new RunnerConsumer();
+let master = require('../index').master;
+
+console.log(producer.getTaskName())
+//return;
+
+let NUMBER_OF_WORKS = 5;//LOCK NUMBER OF WORKERS
+
+let taskConfig = {
+    NUMBER_OF_WORKS,
+    dynamicTasksFolderNew: pathutil.resolve(multi_task_parent_folder, `./dynamicTasks-NEW`),
+    dynamicTasksFolderSucc: pathutil.resolve(multi_task_parent_folder, `./dynamicTasks-SUCC`),
+    dynamicTasksFolderFail: pathutil.resolve(multi_task_parent_folder, `./dynamicTasks-FAIL`),
+}
+makeDir.sync(taskConfig.dynamicTasksFolderNew)
+makeDir.sync(taskConfig.dynamicTasksFolderSucc)
+makeDir.sync(taskConfig.dynamicTasksFolderFail)
+
+
+producer.setTaskTrunkInfo(taskConfig);
+consumer.setTaskTrunkInfo(taskConfig)
+master.run(producer, consumer, {
+    quotaOfEachTask:1, //<--Lock as one
+    numberOfWorkers:NUMBER_OF_WORKS
 });

package/examples/example3-dynamicIncreasing/util.js

@@ -1,46 +1,46 @@
-const md5 = require('md5');
-const fs = require('fs');
-let pathutil = require('path');
-
-let getFileName = (moduleName, requestVersion)=>{
-    let fullname = `${moduleName}@${requestVersion}`;
-    let fullnameMd5 = md5(fullname);
-    return fullnameMd5;
-};
-let createNewTask = (moduleName, requestVersion, dynamicTasksFolderNew)=>{
-    let fullnameMd5 = getFileName(moduleName, requestVersion);
-    fs.writeFileSync(pathutil.resolve(dynamicTasksFolderNew, `./${fullnameMd5}.json`), JSON.stringify({
-        info:{
-            moduleName,
-            requestVersion,
-            fullnameMd5,
-        }
-    }));
-};
-function shuffleAnArray(array) {
-    if(array.length === 0 || array.length === 1) return array;
-    let currentIndex = array.length,  randomIndex;
-  
-    // While there remain elements to shuffle.
-    while (currentIndex > 0) {
-  
-      // Pick a remaining element.
-      randomIndex = Math.floor(Math.random() * currentIndex);
-      currentIndex--;
-
-      // And swap it with the current element.
-      let tmp = array[currentIndex];
-      array[currentIndex] = array[randomIndex];
-      array[randomIndex] = tmp;
-  
-    //   [array[currentIndex], array[randomIndex]] = [
-    //     array[randomIndex], array[currentIndex]];
-    }
-  
-    return array;
-  }
-module.exports = {
-    createNewTask,
-    getFileName,
-    shuffleAnArray
+const md5 = require('md5');
+const fs = require('fs');
+let pathutil = require('path');
+
+let getFileName = (moduleName, requestVersion)=>{
+    let fullname = `${moduleName}@${requestVersion}`;
+    let fullnameMd5 = md5(fullname);
+    return fullnameMd5;
+};
+let createNewTask = (moduleName, requestVersion, dynamicTasksFolderNew)=>{
+    let fullnameMd5 = getFileName(moduleName, requestVersion);
+    fs.writeFileSync(pathutil.resolve(dynamicTasksFolderNew, `./${fullnameMd5}.json`), JSON.stringify({
+        info:{
+            moduleName,
+            requestVersion,
+            fullnameMd5,
+        }
+    }));
+};
+function shuffleAnArray(array) {
+    if(array.length === 0 || array.length === 1) return array;
+    let currentIndex = array.length,  randomIndex;
+  
+    // While there remain elements to shuffle.
+    while (currentIndex > 0) {
+  
+      // Pick a remaining element.
+      randomIndex = Math.floor(Math.random() * currentIndex);
+      currentIndex--;
+
+      // And swap it with the current element.
+      let tmp = array[currentIndex];
+      array[currentIndex] = array[randomIndex];
+      array[randomIndex] = tmp;
+  
+    //   [array[currentIndex], array[randomIndex]] = [
+    //     array[randomIndex], array[currentIndex]];
+    }
+  
+    return array;
+  }
+module.exports = {
+    createNewTask,
+    getFileName,
+    shuffleAnArray
 };

package/index.js

@@ -1,37 +1,4 @@
-const SuperProducer = require('./workers/SuperProducer');
-const SuperConsumer = require('./workers/SuperConsumer');
-const master = require('./workers/master');
-const multiTasks = (config)=>{
-    if(!config.tasks){
-        throw 'Please provide tasks data';
-    };
-    if(!config.processTask){
-        throw 'Please provide processTask function';
-    };
-    class TestProducer extends SuperProducer {      
-        // constructor
-        setTaskTrunkInfo(trunkInfo){
-            this.trunkInfo = trunkInfo;
-        }
-        getAllActions() {
-            return Promise.resolve(config.tasks);//return all your tasks as an array
-        }
-    }
-    class TestConsumer extends SuperConsumer {  
-        processTask(taskProp){
-            console.log(`processing data:`, taskProp);
-            let result = config.processTask(taskProp);
-            return Promise.resolve(result);
-        }
-    }
-
-    master.start(TestProducer, TestConsumer, {
-        ...config
-    });
-};
-module.exports = {
-    SuperProducer,
-    SuperConsumer,
-    master,
-    multiTasks
+const multiTasks = require('./workers');
+module.exports = {
+    multiTasks
 }

package/monitors/cli.js

@@ -0,0 +1,28 @@
+const cliProgress = require('cli-progress');
+
+// create new container
+const multibar = new cliProgress.MultiBar({
+    clearOnComplete: false,
+    hideCursor: true,
+    format: ' {bar} | {filename} | {value}/{total}',
+}, cliProgress.Presets.shades_grey);
+
+// add bars
+const b1 = multibar.create(200, 0);
+const b2 = multibar.create(1000, 0);
+
+// control bars
+b1.increment();
+b2.update(20, {filename: "test1.txt"});
+b1.update(20, {filename: "helloworld.txt"});
+
+b2.setTotal(2000);
+let v = 200;
+setInterval(()=>{
+    v=v+20
+    b2.update(v, {filename: "2222.txt"});
+
+    // stop all bars
+    
+    //multibar.stop();
+}, 300)

package/package.json

@@ -1,22 +1,23 @@
-{
-  "name": "multi-tasks",
-  "version": "1.2.7",
-  "description": "",
-  "main": "index.js",
-  "directories": {
-    "example": "example"
-  },
-  "scripts": {
-    "test": "jest"
-  },
-  "keywords": [
-    "multitask",
-    "worker",
-    "multithread"
-  ],
-  "author": "zhanglei923@gmail.com",
-  "license": "MIT",
-  "devDependencies": {
-    "jest": "^29.7.0"
-  }
-}
+{
+  "name": "multi-tasks",
+  "version": "2.0.0",
+  "description": "",
+  "main": "index.js",
+  "directories": {
+    "example": "example"
+  },
+  "scripts": {
+    "test": "jest"
+  },
+  "keywords": [
+    "multitask",
+    "worker",
+    "multithread"
+  ],
+  "author": "zhanglei923@gmail.com",
+  "license": "MIT",
+  "devDependencies": {
+    "cli-progress": "^3.12.0",
+    "jest": "^29.7.0"
+  }
+}

package/test/splitter.test.js

@@ -1,17 +1,17 @@
-let splitter = require('../workers/taskSplitter');
-
-let createArray = (len)=>{  
-  let alltasks = []
-  for(let i=0;i<len;i++){
-      alltasks.push(i)
-  }
-  return alltasks;
-}
-
-test('test taskSplitter', async () => {
-    
-  expect(splitter.split(createArray(3), 5).length).toBe(1);
-  expect(splitter.split(createArray(16), 5).length).toBe(4);
-  expect(splitter.split(createArray(101), 1).length).toBe(101);
-  expect(splitter.split(createArray(101), 33).length).toBe(4);
+let splitter = require('../workers/taskSplitter');
+
+let createArray = (len)=>{  
+  let alltasks = []
+  for(let i=0;i<len;i++){
+      alltasks.push(i)
+  }
+  return alltasks;
+}
+
+test('test taskSplitter', async () => {
+    
+  expect(splitter.split(createArray(3), 5).length).toBe(1);
+  expect(splitter.split(createArray(16), 5).length).toBe(4);
+  expect(splitter.split(createArray(101), 1).length).toBe(101);
+  expect(splitter.split(createArray(101), 33).length).toBe(4);
 });

package/todo.txt

@@ -0,0 +1,4 @@
+Features:
+  - Continue execution from the point of interruption
+  - Re-run
+  - Show status in command line

package/utils/asyncQueue.js

@@ -0,0 +1,32 @@
+class AsyncQueue {  
+    constructor() {  
+      this.queue = [];  
+      this.isProcessing = false;  
+    }  
+    enqueue(task) {      
+      this.queue.push(task);  
+      
+      if (!this.isProcessing) {  
+        this.processNext();  
+      } 
+    }  
+    async processNext() {  
+      this.isProcessing = true;  
+      if (this.queue.length === 0) {  
+        this.isProcessing = false;  
+        return;  
+      }  
+      const task = this.queue.shift();  
+      let result = task();
+      if(!result instanceof Promise) result = Promise.resolve(result);
+    
+      try {  
+        await result; 
+      } catch (error) {  
+        console.error('Error processing task:', error);  
+      }  
+      this.processNext(); // 处理完一个任务后，继续处理下一个任务  
+    }  
+  }  
+
+module.exports = AsyncQueue;

package/utils/backward-v1.js

@@ -0,0 +1,11 @@
+function v1tov2(config) {      
+    if(config.multi_task_parent_folder) config.taskRootFolder = config.multi_task_parent_folder;//backward compatible
+    delete config.multi_task_parent_folder;
+    
+    if(config.numberOfWorks && !config.numberOfWorkers) config.numberOfWorkers = config.numberOfWorks; //Backward compatible with the wrong variable name "numberOfWorks"
+
+
+    return config;
+}  
+
+module.exports = {v1tov2};

package/utils/makedir.js

@@ -1,24 +1,55 @@
-const fs = require('fs');
-const path = require('path');
-
-function sync(directoryPath) {
-    directoryPath = directoryPath.replace(/\\/g, '/');
-
-    const directories = directoryPath.split('/');
-    
-    let currentPath = '';
-    for (let directory of directories) {
-        currentPath = path.join(currentPath, directory);
-        if (!fs.existsSync(currentPath)) {
-            try {
-                fs.mkdirSync(currentPath);
-            } catch (err) {
-                if (err.code !== 'EEXIST') {
-                    throw err;
-                }
-            }
-        }
-    }
-}
-
-module.exports = {sync};
+const fs = require('fs');
+const path = require('path');
+
+function sync(directoryPath) {
+    directoryPath = directoryPath.replace(/\\/g, '/');
+
+    let isAbsPath = false;
+    if(directoryPath.startsWith('/')){
+        isAbsPath = true;
+    }
+
+    const directories = directoryPath.split('/');
+    
+    let currentPath = isAbsPath ? '/':'';
+    for (let directory of directories) {
+        currentPath = path.join(currentPath, directory);
+        if (!fs.existsSync(currentPath)) {
+            try {
+                fs.mkdirSync(currentPath);
+            } catch (err) {
+                if (err.code !== 'EEXIST') {
+                    throw err;
+                }
+            }
+        }
+    }
+}
+function getDirectFiles(dir, callback) {  
+    let childrenfiles = [];
+    if(!fs.existsSync(dir)){
+        console.log('FATAL: directory is missing:', dir);
+        return process.exit(1);        
+    }
+    fs.readdir(dir, (err, files) => {  
+        if (err) {  
+            console.error("Could not list the directory.", err);  
+            return callback([]);
+        }
+        files.forEach(file => {  
+            const filePath = path.join(dir, file);  
+            //console.log(filePath)
+            try{
+                let stats = fs.statSync(filePath);
+                if (stats.isFile()) {  
+                    childrenfiles.push(file); 
+                }  
+            }catch(e){
+                //
+            }
+        });
+        callback(childrenfiles);
+    });  
+}  
+
+module.exports = {sync, getDirectFiles};

package/utils/randoms.js

@@ -0,0 +1,15 @@
+
+const fmtDigit = (n)=>{
+    return n > 9 ? "" + n: "0" + n;
+}
+const getDateTimeTxt = ()=>{
+    let d=new Date();
+    let YYYY=d.getFullYear()
+    let MM=fmtDigit(d.getMonth()+1);
+    let DD=fmtDigit(d.getDate());
+    let HH=fmtDigit(d.getHours());
+    let mm=fmtDigit(d.getMinutes());
+    let ss=fmtDigit(d.getSeconds());
+    return `${YYYY}-${MM}-${DD}_${HH}-${mm}-${ss}`;
+};
+module.exports = {getDateTimeTxt};

package/utils/test/test_asyncqueue.js

@@ -0,0 +1,116 @@
+const AsyncQueue = require(`../asyncQueue`);
+const queue = new AsyncQueue();  
+
+let count=0;
+  
+queue.enqueue(async () => {
+    count++;  
+  console.log(`Task ${count} start`);  
+  await new Promise(resolve => setTimeout(resolve, 100)); 
+  console.log(`Task ${count} end`);  
+});
+ 
+  
+queue.enqueue(async () => {
+    count++;  
+  console.log(`Task ${count} start`);  
+  await new Promise(resolve => setTimeout(resolve, 20)); 
+  console.log(`Task ${count} end`);  
+});
+ 
+  
+queue.enqueue(() => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    console.log(`Task ${count} end`);  
+    return 11;
+  });
+      
+  
+queue.enqueue(() => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    console.log(`Task ${count} end`);  
+    return 11;
+  });   
+  
+  queue.enqueue(() => {
+      count++;  
+      console.log(`Task ${count} start`);  
+      console.log(`Task ${count} end`);  
+      return 11;
+    });   
+  
+    queue.enqueue(() => {
+        count++;  
+        console.log(`Task ${count} start`);  
+        console.log(`Task ${count} end`);  
+        return 11;
+      });   
+  
+      queue.enqueue(() => {
+          count++;  
+          console.log(`Task ${count} start`);  
+          console.log(`Task ${count} end`);  
+          return 11;
+        });   
+  
+        queue.enqueue(() => {
+            count++;  
+            console.log(`Task ${count} start`);  
+            console.log(`Task ${count} end`);  
+            return 11;
+          });
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+  
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+  
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+  
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+  
+queue.enqueue(() => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    console.log(`Task ${count} end`);  
+    return 11;
+  });   
+  
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+            
+  

package/utils/test_makedir.js

@@ -1,3 +1,3 @@
-let makedir = require('./makedir');
-makedir.sync('c:/aaa/bb')
+let makedir = require('./makedir');
+makedir.sync('c:/aaa/bb')
 makedir.sync('c:/cc/dd')