multi-tasks 2.0.1 → 2.0.3

package/test/test_resume/step2_resume.js

@@ -0,0 +1,21 @@
+let multiTasks = require('../../index').multiTasks;
+
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    return 'a result which is not a promise'; 
+};
+
+//Step3, run!
+multiTasks.resume({
+    processTask, 
+    taskFolder: `./tmp/test_resume`, //a directory to store progress and results files, you can check the progress here
+    numberOfWorkers: 3, //Assign how many workers are working in parallel
+    //autoCloseAfterCompletion: true,
+    shouldTerminate:(info)=>{
+        //if(info.taskCount > 30) return true;
+    },
+    onFinish: (report)=>{
+        console.log('finish callback', report);
+    }
+});

package/test/test_resume/test.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+node step1_start_and_crash
+node step2_resume

package/todo.txt

@@ -1,4 +1,4 @@
-Features:
-  - Continue execution from the point of interruption
-  - Re-run
+Features:
+  - Continue execution from the point of interruption
+  - Re-run
   - Show status in command line

package/utils/asyncQueue.js

@@ -1,32 +1,36 @@
-class AsyncQueue {  
-    constructor() {  
-      this.queue = [];  
-      this.isProcessing = false;  
-    }  
-    enqueue(task) {      
-      this.queue.push(task);  
-      
-      if (!this.isProcessing) {  
-        this.processNext();  
-      } 
-    }  
-    async processNext() {  
-      this.isProcessing = true;  
-      if (this.queue.length === 0) {  
-        this.isProcessing = false;  
-        return;  
-      }  
-      const task = this.queue.shift();  
-      let result = task();
-      if(!result instanceof Promise) result = Promise.resolve(result);
-    
-      try {  
-        await result; 
-      } catch (error) {  
-        console.error('Error processing task:', error);  
-      }  
-      this.processNext(); // 处理完一个任务后，继续处理下一个任务  
-    }  
-  }  
-
+class AsyncQueue {  
+    constructor() {  
+      this.queue = [];  
+      this.isProcessing = false;  
+    }  
+    enqueue(task) {      
+      this.queue.push(task);  
+      
+      if (!this.isProcessing) {  
+        this.processNext();  
+      } 
+    }  
+    async processNext() {  
+      this.isProcessing = true;  
+      if (this.queue.length === 0) {  
+        this.isProcessing = false;  
+        return;  
+      }  
+      const task = this.queue.shift();  
+      let result = task();
+      let isPromise = result instanceof Promise;
+      //console.log('r=q',isPromise)
+      if(!isPromise) {
+        console.error('[ERR]Queue functions must return a promise object');  
+        throw 'Queue functions must return a promise object';
+      }    
+      try {  
+        await result; 
+      } catch (error) {  
+        console.error('Error processing task:', error);  
+      }  
+      this.processNext();
+    }  
+  }  
+
 module.exports = AsyncQueue;

package/utils/backward-v1.js

@@ -1,11 +1,11 @@
-function v1tov2(config) {      
-    if(config.multi_task_parent_folder) config.taskRootFolder = config.multi_task_parent_folder;//backward compatible
-    delete config.multi_task_parent_folder;
-    
-    if(config.numberOfWorks && !config.numberOfWorkers) config.numberOfWorkers = config.numberOfWorks; //Backward compatible with the wrong variable name "numberOfWorks"
-
-
-    return config;
-}  
-
+function v1tov2(config) {      
+    if(config.multi_task_parent_folder) config.taskRootFolder = config.multi_task_parent_folder;//backward compatible
+    delete config.multi_task_parent_folder;
+    
+    if(config.numberOfWorks && !config.numberOfWorkers) config.numberOfWorkers = config.numberOfWorks; //Backward compatible with the wrong variable name "numberOfWorks"
+
+
+    return config;
+}  
+
 module.exports = {v1tov2};

package/utils/makedir.js

@@ -1,51 +1,41 @@
-const fs = require('fs');
-const path = require('path');
-
-function sync(directoryPath) {
-    directoryPath = directoryPath.replace(/\\/g, '/');
-
-    let isAbsPath = false;
-    if(directoryPath.startsWith('/')){
-        isAbsPath = true;
-    }
-
-    const directories = directoryPath.split('/');
-    
-    let currentPath = isAbsPath ? '/':'';
-    for (let directory of directories) {
-        currentPath = path.join(currentPath, directory);
-        if (!fs.existsSync(currentPath)) {
-            try {
-                fs.mkdirSync(currentPath);
-            } catch (err) {
-                if (err.code !== 'EEXIST') {
-                    throw err;
-                }
-            }
-        }
-    }
-}
-function getDirectFiles(dir, callback) {  
-    let childrenfiles = [];
-    if(!fs.existsSync(dir)){
-        console.log('FATAL: directory is missing:', dir);
-        return process.exit(1);        
-    }
-    fs.readdir(dir, (err, files) => {  
-        if (err) {  
-            console.error("Could not list the directory.", err);  
-            return callback([]);
-        }
-        files.forEach(file => {  
-            const filePath = path.join(dir, file);  
-            //console.log(filePath)
-                let stats = fs.statSync(filePath);
-                if (stats.isFile()) {  
-                    childrenfiles.push(file); 
-                }  
-        });
-        callback(childrenfiles);
-    });  
-}  
-
+const fs = require('fs');
+const path = require('path');
+
+function sync(directoryPath) {
+    directoryPath = directoryPath.replace(/\\/g, '/');
+
+    let isAbsPath = false;
+    if(directoryPath.startsWith('/')){
+        isAbsPath = true;
+    }
+
+    fs.mkdirSync(directoryPath, { recursive: true });
+}
+function getDirectFiles(dir, callback) {  
+    let childrenfiles = [];
+    if(!fs.existsSync(dir)){
+        console.log('FATAL: directory is missing:', dir);
+        return process.exit(1);        
+    }
+    fs.readdir(dir, (err, files) => {  
+        if (err) {  
+            console.error("Could not list the directory.", err);  
+            return callback([]);
+        }
+        files.forEach(file => {  
+            const filePath = path.join(dir, file);  
+            //console.log(filePath)
+            try{
+                let stats = fs.statSync(filePath);
+                if (stats.isFile()) {  
+                    childrenfiles.push(file); 
+                }  
+            }catch(e){
+                //
+            }
+        });
+        callback(childrenfiles);
+    });  
+}  
+
 module.exports = {sync, getDirectFiles};

package/utils/randoms.js

@@ -1,15 +1,21 @@
-
-const fmtDigit = (n)=>{
-    return n > 9 ? "" + n: "0" + n;
-}
-const getDateTimeTxt = ()=>{
-    let d=new Date();
-    let YYYY=d.getFullYear()
-    let MM=fmtDigit(d.getMonth()+1);
-    let DD=fmtDigit(d.getDate());
-    let HH=fmtDigit(d.getHours());
-    let mm=fmtDigit(d.getMinutes());
-    let ss=fmtDigit(d.getSeconds());
-    return `${YYYY}-${MM}-${DD}_${HH}-${mm}-${ss}`;
-};
-module.exports = {getDateTimeTxt};
+
+const fmtDigit = (n)=>{
+    return n > 9 ? "" + n: "0" + n;
+}
+const getDateTimeTxt = ()=>{
+    let d=new Date();
+    let YYYY=d.getFullYear()
+    let MM=fmtDigit(d.getMonth()+1);
+    let DD=fmtDigit(d.getDate());
+    let HH=fmtDigit(d.getHours());
+    let mm=fmtDigit(d.getMinutes());
+    let ss=fmtDigit(d.getSeconds());
+    return `${YYYY}-${MM}-${DD}_${HH}-${mm}-${ss}`;
+};
+const get36Id = ()=>{
+    let num1 = (Math.random().toString().match(/\d/g).join('')*1).toString(36);  
+    let num2 = Date.now().toString(36);
+    let str = num2+num1;
+    return str;
+};
+module.exports = {getDateTimeTxt, get36Id};

package/utils/test/test_asyncqueue.js

@@ -1,116 +1,116 @@
-const AsyncQueue = require(`../asyncQueue`);
-const queue = new AsyncQueue();  
-
-let count=0;
-  
-queue.enqueue(async () => {
-    count++;  
-  console.log(`Task ${count} start`);  
-  await new Promise(resolve => setTimeout(resolve, 100)); 
-  console.log(`Task ${count} end`);  
-});
- 
-  
-queue.enqueue(async () => {
-    count++;  
-  console.log(`Task ${count} start`);  
-  await new Promise(resolve => setTimeout(resolve, 20)); 
-  console.log(`Task ${count} end`);  
-});
- 
-  
-queue.enqueue(() => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    console.log(`Task ${count} end`);  
-    return 11;
-  });
-      
-  
-queue.enqueue(() => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    console.log(`Task ${count} end`);  
-    return 11;
-  });   
-  
-  queue.enqueue(() => {
-      count++;  
-      console.log(`Task ${count} start`);  
-      console.log(`Task ${count} end`);  
-      return 11;
-    });   
-  
-    queue.enqueue(() => {
-        count++;  
-        console.log(`Task ${count} start`);  
-        console.log(`Task ${count} end`);  
-        return 11;
-      });   
-  
-      queue.enqueue(() => {
-          count++;  
-          console.log(`Task ${count} start`);  
-          console.log(`Task ${count} end`);  
-          return 11;
-        });   
-  
-        queue.enqueue(() => {
-            count++;  
-            console.log(`Task ${count} start`);  
-            console.log(`Task ${count} end`);  
-            return 11;
-          });
-  
-queue.enqueue(async () => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    await new Promise(resolve => setTimeout(resolve, 20)); 
-    console.log(`Task ${count} end`);  
-  });
-   
-  
-  
-queue.enqueue(async () => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    await new Promise(resolve => setTimeout(resolve, 20)); 
-    console.log(`Task ${count} end`);  
-  });
-   
-  
-  
-queue.enqueue(async () => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    await new Promise(resolve => setTimeout(resolve, 20)); 
-    console.log(`Task ${count} end`);  
-  });
-   
-  
-  
-queue.enqueue(async () => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    await new Promise(resolve => setTimeout(resolve, 20)); 
-    console.log(`Task ${count} end`);  
-  });
-   
-  
-queue.enqueue(() => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    console.log(`Task ${count} end`);  
-    return 11;
-  });   
-  
-  
-queue.enqueue(async () => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    await new Promise(resolve => setTimeout(resolve, 20)); 
-    console.log(`Task ${count} end`);  
-  });
-   
-            
+const AsyncQueue = require(`../asyncQueue`);
+const queue = new AsyncQueue();  
+
+let count=0;
+  
+queue.enqueue(async () => {
+    count++;  
+  console.log(`Task ${count} start`);  
+  await new Promise(resolve => setTimeout(resolve, 100)); 
+  console.log(`Task ${count} end`);  
+});
+ 
+  
+queue.enqueue(async () => {
+    count++;  
+  console.log(`Task ${count} start`);  
+  await new Promise(resolve => setTimeout(resolve, 20)); 
+  console.log(`Task ${count} end`);  
+});
+ 
+  
+queue.enqueue(() => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    console.log(`Task ${count} end`);  
+    return 11;
+  });
+      
+  
+queue.enqueue(() => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    console.log(`Task ${count} end`);  
+    return 11;
+  });   
+  
+  queue.enqueue(() => {
+      count++;  
+      console.log(`Task ${count} start`);  
+      console.log(`Task ${count} end`);  
+      return 11;
+    });   
+  
+    queue.enqueue(() => {
+        count++;  
+        console.log(`Task ${count} start`);  
+        console.log(`Task ${count} end`);  
+        return 11;
+      });   
+  
+      queue.enqueue(() => {
+          count++;  
+          console.log(`Task ${count} start`);  
+          console.log(`Task ${count} end`);  
+          return 11;
+        });   
+  
+        queue.enqueue(() => {
+            count++;  
+            console.log(`Task ${count} start`);  
+            console.log(`Task ${count} end`);  
+            return 11;
+          });
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+  
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+  
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+  
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+  
+queue.enqueue(() => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    console.log(`Task ${count} end`);  
+    return 11;
+  });   
+  
+  
+queue.enqueue(async () => {
+    count++;  
+    console.log(`Task ${count} start`);  
+    await new Promise(resolve => setTimeout(resolve, 20)); 
+    console.log(`Task ${count} end`);  
+  });
+   
+            
   

package/utils/test_makedir.js

@@ -1,3 +1,3 @@
-let makedir = require('./makedir');
-makedir.sync('c:/aaa/bb')
+let makedir = require('./makedir');
+makedir.sync('c:/aaa/bb')
 makedir.sync('c:/cc/dd')