multi-tasks 1.2.6 → 1.2.8

package/index.js

@@ -1,37 +1,37 @@
-const SuperProducer = require('./workers/SuperProducer');
-const SuperConsumer = require('./workers/SuperConsumer');
-const master = require('./workers/master');
-const multiTasks = (config)=>{
-    if(!config.tasks){
-        throw 'Please provide tasks data';
-    };
-    if(!config.processTask){
-        throw 'Please provide processTask function';
-    };
-    class TestProducer extends SuperProducer {      
-        // constructor
-        setTaskTrunkInfo(trunkInfo){
-            this.trunkInfo = trunkInfo;
-        }
-        getAllActions() {
-            return Promise.resolve(config.tasks);//return all your tasks as an array
-        }
-    }
-    class TestConsumer extends SuperConsumer {  
-        processTask(taskProp){
-            console.log(`processing data:`, taskProp);
-            let result = config.processTask(taskProp);
-            return Promise.resolve(result);
-        }
-    }
-
-    master.start(TestProducer, TestConsumer, {
-        ...config
-    });
-};
-module.exports = {
-    SuperProducer,
-    SuperConsumer,
-    master,
-    multiTasks
+const SuperProducer = require('./workers/SuperProducer');
+const SuperConsumer = require('./workers/SuperConsumer');
+const master = require('./workers/master');
+const multiTasks = (config)=>{
+    if(!config.tasks){
+        throw 'Please provide tasks data';
+    };
+    if(!config.processTask){
+        throw 'Please provide processTask function';
+    };
+    class TestProducer extends SuperProducer {      
+        // constructor
+        setTaskTrunkInfo(trunkInfo){
+            this.trunkInfo = trunkInfo;
+        }
+        getAllActions() {
+            return Promise.resolve(config.tasks);//return all your tasks as an array
+        }
+    }
+    class TestConsumer extends SuperConsumer {  
+        processTask(taskProp){
+            console.log(`processing data:`, taskProp);
+            let result = config.processTask(taskProp);
+            return Promise.resolve(result);
+        }
+    }
+
+    master.start(TestProducer, TestConsumer, {
+        ...config
+    });
+};
+module.exports = {
+    SuperProducer,
+    SuperConsumer,
+    master,
+    multiTasks
 }

package/package.json

@@ -1,22 +1,22 @@
-{
-  "name": "multi-tasks",
-  "version": "1.2.6",
-  "description": "",
-  "main": "index.js",
-  "directories": {
-    "example": "example"
-  },
-  "scripts": {
-    "test": "jest"
-  },
-  "keywords": [
-    "multitask",
-    "worker",
-    "multithread"
-  ],
-  "author": "zhanglei923@gmail.com",
-  "license": "MIT",
-  "devDependencies": {
-    "jest": "^29.7.0"
-  }
-}
+{
+  "name": "multi-tasks",
+  "version": "1.2.8",
+  "description": "",
+  "main": "index.js",
+  "directories": {
+    "example": "example"
+  },
+  "scripts": {
+    "test": "jest"
+  },
+  "keywords": [
+    "multitask",
+    "worker",
+    "multithread"
+  ],
+  "author": "zhanglei923@gmail.com",
+  "license": "MIT",
+  "devDependencies": {
+    "jest": "^29.7.0"
+  }
+}

package/test/splitter.test.js

@@ -1,17 +1,17 @@
-let splitter = require('../workers/taskSplitter');
-
-let createArray = (len)=>{  
-  let alltasks = []
-  for(let i=0;i<len;i++){
-      alltasks.push(i)
-  }
-  return alltasks;
-}
-
-test('test taskSplitter', async () => {
-    
-  expect(splitter.split(createArray(3), 5).length).toBe(1);
-  expect(splitter.split(createArray(16), 5).length).toBe(4);
-  expect(splitter.split(createArray(101), 1).length).toBe(101);
-  expect(splitter.split(createArray(101), 33).length).toBe(4);
+let splitter = require('../workers/taskSplitter');
+
+let createArray = (len)=>{  
+  let alltasks = []
+  for(let i=0;i<len;i++){
+      alltasks.push(i)
+  }
+  return alltasks;
+}
+
+test('test taskSplitter', async () => {
+    
+  expect(splitter.split(createArray(3), 5).length).toBe(1);
+  expect(splitter.split(createArray(16), 5).length).toBe(4);
+  expect(splitter.split(createArray(101), 1).length).toBe(101);
+  expect(splitter.split(createArray(101), 33).length).toBe(4);
 });

package/utils/makedir.js

@@ -1,24 +1,29 @@
-const fs = require('fs');
-const path = require('path');
-
-function sync(directoryPath) {
-    directoryPath = directoryPath.replace(/\\/g, '/');
-
-    const directories = directoryPath.split('/');
-    
-    let currentPath = '';
-    for (let directory of directories) {
-        currentPath = path.join(currentPath, directory);
-        if (!fs.existsSync(currentPath)) {
-            try {
-                fs.mkdirSync(currentPath);
-            } catch (err) {
-                if (err.code !== 'EEXIST') {
-                    throw err;
-                }
-            }
-        }
-    }
-}
-
+const fs = require('fs');
+const path = require('path');
+
+function sync(directoryPath) {
+    directoryPath = directoryPath.replace(/\\/g, '/');
+
+    let isAbsPath = false;
+    if(directoryPath.startsWith('/')){
+        isAbsPath = true;
+    }
+
+    const directories = directoryPath.split('/');
+    
+    let currentPath = isAbsPath ? '/':'';
+    for (let directory of directories) {
+        currentPath = path.join(currentPath, directory);
+        if (!fs.existsSync(currentPath)) {
+            try {
+                fs.mkdirSync(currentPath);
+            } catch (err) {
+                if (err.code !== 'EEXIST') {
+                    throw err;
+                }
+            }
+        }
+    }
+}
+
 module.exports = {sync};

package/utils/test_makedir.js

@@ -1,3 +1,3 @@
-let makedir = require('./makedir');
-makedir.sync('c:/aaa/bb')
+let makedir = require('./makedir');
+makedir.sync('c:/aaa/bb')
 makedir.sync('c:/cc/dd')

package/workers/SuperConsumer.js

@@ -1,106 +1,106 @@
-let fs = require('fs');
-let pathutil = require('path');
-
-class SuperConsumer {
-    // constructor
-    constructor(mainid, subid) {
-        this.mainid = mainid;
-        this.subid = subid;
-        this.results = [];
-    }
-
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    _run (mainTaskInfo, subTaskInfo, actions, results, errors, callback){
-        if(actions.length === 0) return callback({
-                                            finishedAt: new Date()*1,
-                                            results,
-                                            errors,
-                                            subTaskInfo
-                                        });
-        let taskProp = actions.shift();
-    
-        if(this.processTask){
-            let result;
-            let hasException = false;
-            try{
-                result = this.processTask(taskProp);//rewrite this method in subclass
-            }catch(e){
-                hasException = true;
-                result = {taskProp, exception: 'Exception:' + e.toString()}
-            }
-            if(typeof result === 'undefined') result = {};
-            let ispromise = result instanceof Promise;
-            if(!ispromise){result = Promise.resolve(result);}
-            result.then((taskResult)=>{
-                if(!hasException){
-                    results.push({taskProp, taskResult});
-                }else{
-                    errors.push(taskResult);
-                }
-            }).catch((e)=>{
-                errors.push({taskProp, exception: 'Exception:' + e.toString()});
-            }).finally(()=>{
-                this._run(mainTaskInfo, subTaskInfo, actions, results, errors, callback);
-            });            
-        }else{
-            this._run(mainTaskInfo, subTaskInfo, actions, results, errors, callback);
-        }    
-    };
-    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
-        let results = [];
-        let errors = [];
-        let actions = subTaskInfo.actions;
-        mainTaskInfo.defination = this.trunkInfo.defination;
-        this._run(mainTaskInfo, subTaskInfo, actions, results, errors, (result)=>{
-            callback(result)
-        })
-
-    }
-    runTasks (mainTaskInfo, subTaskInfo, message){
-        // console.log(`mainTaskInfo:`, mainTaskInfo)
-        // console.log(`subTaskInfo:`, subTaskInfo)
-        // console.log(`message:`, message)
-
-        this.mainTaskInfo = mainTaskInfo;
-        this.subTaskInfo = subTaskInfo;
-
-        return new Promise((resolve, reject)=>{
-            let mainid = mainTaskInfo.masterTaskId;
-            let subid = subTaskInfo.subid;
-            let taskfolderNew = mainTaskInfo.taskFolderNew;
-            let taskFolderResult = mainTaskInfo.taskFolderResult;
-            let taskFolderResultFailed = mainTaskInfo.taskFolderResultFailed;            
-            this.mvTask(mainid, subid,'new','running', (taskFile)=>{//mark as running
-                let taskContent = fs.readFileSync(taskFile, 'utf8');
-                let taskInfo = JSON.parse(taskContent);
-                this.run(mainTaskInfo, subTaskInfo, taskInfo, (taskReport)=>{//invoke
-                    let {results, errors} = taskReport;
-                    let hasError = false;
-                    if(errors &&  errors.length > 0) hasError = true;
-                    let folder = !hasError ? taskFolderResult : taskFolderResultFailed;
-                    let resultfile = pathutil.resolve(folder, `${subid}.json`)
-                    fs.writeFileSync(resultfile, JSON.stringify(taskReport));
-                    this.mvTask(mainid, subid,'running',(hasError ? 'finished_with_errors':'finished'), ()=>{//mark as finished
-                        resolve({
-                            mainid, 
-                            subid,
-                            ...message
-                        })
-                    });
-                });            
-            });
-        });
-    }
-    mvTask (mainid, subid, from, to, callback) {
-        let __thisdir = this.mainTaskInfo.progressRootFolder;
-        let oldPath = pathutil.resolve(__thisdir, `./${from}/${subid}.task.json`);
-        let newPath = pathutil.resolve(__thisdir, `./${to}/${subid}.task.json`);
-        fs.rename(oldPath, newPath, (err) => {
-            if (err) throw err;
-            callback(newPath);
-        });
-    }
-}
+let fs = require('fs');
+let pathutil = require('path');
+
+class SuperConsumer {
+    // constructor
+    constructor(mainid, subid) {
+        this.mainid = mainid;
+        this.subid = subid;
+        this.results = [];
+    }
+
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    _run (mainTaskInfo, subTaskInfo, actions, results, errors, callback){
+        if(actions.length === 0) return callback({
+                                            finishedAt: new Date()*1,
+                                            results,
+                                            errors,
+                                            subTaskInfo
+                                        });
+        let taskProp = actions.shift();
+    
+        if(this.processTask){
+            let result;
+            let hasException = false;
+            try{
+                result = this.processTask(taskProp);//rewrite this method in subclass
+            }catch(e){
+                hasException = true;
+                result = {taskProp, exception: 'Exception:' + e.toString()}
+            }
+            if(typeof result === 'undefined') result = {};
+            let ispromise = result instanceof Promise;
+            if(!ispromise){result = Promise.resolve(result);}
+            result.then((taskResult)=>{
+                if(!hasException){
+                    results.push({taskProp, taskResult});
+                }else{
+                    errors.push(taskResult);
+                }
+            }).catch((e)=>{
+                errors.push({taskProp, exception: 'Exception:' + e.toString()});
+            }).finally(()=>{
+                this._run(mainTaskInfo, subTaskInfo, actions, results, errors, callback);
+            });            
+        }else{
+            this._run(mainTaskInfo, subTaskInfo, actions, results, errors, callback);
+        }    
+    };
+    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
+        let results = [];
+        let errors = [];
+        let actions = subTaskInfo.actions;
+        mainTaskInfo.defination = this.trunkInfo.defination;
+        this._run(mainTaskInfo, subTaskInfo, actions, results, errors, (result)=>{
+            callback(result)
+        })
+
+    }
+    runTasks (mainTaskInfo, subTaskInfo, message){
+        // console.log(`mainTaskInfo:`, mainTaskInfo)
+        // console.log(`subTaskInfo:`, subTaskInfo)
+        // console.log(`message:`, message)
+
+        this.mainTaskInfo = mainTaskInfo;
+        this.subTaskInfo = subTaskInfo;
+
+        return new Promise((resolve, reject)=>{
+            let mainid = mainTaskInfo.masterTaskId;
+            let subid = subTaskInfo.subid;
+            let taskfolderNew = mainTaskInfo.taskFolderNew;
+            let taskFolderResult = mainTaskInfo.taskFolderResult;
+            let taskFolderResultFailed = mainTaskInfo.taskFolderResultFailed;            
+            this.mvTask(mainid, subid,'new','running', (taskFile)=>{//mark as running
+                let taskContent = fs.readFileSync(taskFile, 'utf8');
+                let taskInfo = JSON.parse(taskContent);
+                this.run(mainTaskInfo, subTaskInfo, taskInfo, (taskReport)=>{//invoke
+                    let {results, errors} = taskReport;
+                    let hasError = false;
+                    if(errors &&  errors.length > 0) hasError = true;
+                    let folder = !hasError ? taskFolderResult : taskFolderResultFailed;
+                    let resultfile = pathutil.resolve(folder, `${subid}.json`)
+                    fs.writeFileSync(resultfile, JSON.stringify(taskReport));
+                    this.mvTask(mainid, subid,'running',(hasError ? 'finished_with_errors':'finished'), ()=>{//mark as finished
+                        resolve({
+                            mainid, 
+                            subid,
+                            ...message
+                        })
+                    });
+                });            
+            });
+        });
+    }
+    mvTask (mainid, subid, from, to, callback) {
+        let __thisdir = this.mainTaskInfo.progressRootFolder;
+        let oldPath = pathutil.resolve(__thisdir, `./${from}/${subid}.task.json`);
+        let newPath = pathutil.resolve(__thisdir, `./${to}/${subid}.task.json`);
+        fs.rename(oldPath, newPath, (err) => {
+            if (err) throw err;
+            callback(newPath);
+        });
+    }
+}
 module.exports = SuperConsumer;

package/workers/SuperProducer.js

@@ -1,97 +1,97 @@
-let fs = require('fs');
-let pathutil = require('path');
-let makeDir = require('../utils/makedir');
-let splitter = require('./taskSplitter');
-let fmtdigit = (n)=>{
-    return n > 9 ? "" + n: "0" + n;
-}
-let getDateTimeTxt = ()=>{
-    let d=new Date();
-    let YYYY=d.getFullYear()
-    let MM=fmtdigit(d.getMonth()+1);
-    let DD=fmtdigit(d.getDate());
-    let HH=fmtdigit(d.getHours());
-    let mm=fmtdigit(d.getMinutes());
-    let ss=fmtdigit(d.getSeconds());
-    return `${YYYY}-${MM}-${DD}_${HH}-${mm}-${ss}`;
-};
-class SuperProducer {
-    // constructor
-    constructor(taskname, parentfolder) {
-        this.taskname = taskname;
-        this.parentfolder = parentfolder;
-
-    }
-    
-    getTaskName(){
-        return this.taskname;
-    }
-    getAllActions (){
-        //rewrite this method and return all your actions need to run
-        return new Promise();
-    }
-    generateTasks (quota, callback){
-        if(typeof quota === 'undefined' || !quota)quota=10;
-        let dateTimeText = getDateTimeTxt();//moment().format('YYYY-MM-DD_HH-mm-ss');
-        let masterTaskId = `${this.taskname}_${dateTimeText}`;
-        console.log('masterTaskId=',masterTaskId);
-        this.taskfolder = pathutil.resolve(this.parentfolder, masterTaskId)
-        let taskFolder = this.taskfolder;
-        //let taskfolder = pathutil.parse(__filename).dir;
-        //let taskFolder = pathutil.resolve(taskfolder, `./tasks/${masterTaskId}`);
-        let progressRootFolder = pathutil.resolve(taskFolder, './progress');
-        let resultsRootFolder = pathutil.resolve(taskFolder, './results');
-
-        let taskFolderNew = pathutil.resolve(progressRootFolder, './new');
-        let taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
-        let taskFolderFinished = pathutil.resolve(progressRootFolder, './finished');
-        let taskFolderFailed= pathutil.resolve(progressRootFolder, './finished_with_errors');
-
-        let taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
-        let taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
-        //let mainRptFolder = pathutil.resolve(taskfolder, './report');
-    
-        //makeDir.sync(mainRptFolder);
-        makeDir.sync(taskFolderNew);
-        makeDir.sync(taskFolderRunning);
-        makeDir.sync(taskFolderFinished);
-        makeDir.sync(taskFolderFailed);
-        makeDir.sync(taskFolderResult);
-        makeDir.sync(taskFolderResultFailed);
-
-        this.getAllActions().then((allactions)=>{
-
-            let allSubTasks = splitter.split(allactions, quota);
-
-        
-            console.log('allSubTasks=', allSubTasks.length)
-
-            let allSubTasksId = [];
-    
-            allSubTasks.forEach((info, i)=>{
-                let subid = i;
-                info.subid = subid;
-                allSubTasksId.push(subid+'');
-                let taskfilepath = pathutil.resolve(taskFolderNew, `./${info.subid}.task.json`);
-                fs.writeFileSync(taskfilepath, JSON.stringify(info));
-            });
-    
-            callback({
-                masterTaskId,
-                allSubTasksId,
-                //mainRptFolder,
-                progressRootFolder,
-                resultsRootFolder,
-
-                taskFolder,
-                taskFolderNew,
-                taskFolderRunning,
-                taskFolderFinished,
-                taskFolderResult,
-                taskFolderResultFailed,
-            });
-        });
-    }
-}
-
+let fs = require('fs');
+let pathutil = require('path');
+let makeDir = require('../utils/makedir');
+let splitter = require('./taskSplitter');
+let fmtdigit = (n)=>{
+    return n > 9 ? "" + n: "0" + n;
+}
+let getDateTimeTxt = ()=>{
+    let d=new Date();
+    let YYYY=d.getFullYear()
+    let MM=fmtdigit(d.getMonth()+1);
+    let DD=fmtdigit(d.getDate());
+    let HH=fmtdigit(d.getHours());
+    let mm=fmtdigit(d.getMinutes());
+    let ss=fmtdigit(d.getSeconds());
+    return `${YYYY}-${MM}-${DD}_${HH}-${mm}-${ss}`;
+};
+class SuperProducer {
+    // constructor
+    constructor(taskname, parentfolder) {
+        this.taskname = taskname;
+        this.parentfolder = parentfolder;
+
+    }
+    
+    getTaskName(){
+        return this.taskname;
+    }
+    getAllActions (){
+        //rewrite this method and return all your actions need to run
+        return new Promise();
+    }
+    generateTasks (quota, callback){
+        if(typeof quota === 'undefined' || !quota)quota=10;
+        let dateTimeText = getDateTimeTxt();//moment().format('YYYY-MM-DD_HH-mm-ss');
+        let masterTaskId = `${this.taskname}_${dateTimeText}`;
+        console.log('masterTaskId=',masterTaskId);
+        this.taskfolder = pathutil.resolve(this.parentfolder, masterTaskId)
+        let taskFolder = this.taskfolder;
+        //let taskfolder = pathutil.parse(__filename).dir;
+        //let taskFolder = pathutil.resolve(taskfolder, `./tasks/${masterTaskId}`);
+        let progressRootFolder = pathutil.resolve(taskFolder, './progress');
+        let resultsRootFolder = pathutil.resolve(taskFolder, './results');
+
+        let taskFolderNew = pathutil.resolve(progressRootFolder, './new');
+        let taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
+        let taskFolderFinished = pathutil.resolve(progressRootFolder, './finished');
+        let taskFolderFailed= pathutil.resolve(progressRootFolder, './finished_with_errors');
+
+        let taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
+        let taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
+        //let mainRptFolder = pathutil.resolve(taskfolder, './report');
+    
+        //makeDir.sync(mainRptFolder);
+        makeDir.sync(taskFolderNew);
+        makeDir.sync(taskFolderRunning);
+        makeDir.sync(taskFolderFinished);
+        makeDir.sync(taskFolderFailed);
+        makeDir.sync(taskFolderResult);
+        makeDir.sync(taskFolderResultFailed);
+
+        this.getAllActions().then((allactions)=>{
+
+            let allSubTasks = splitter.split(allactions, quota);
+
+        
+            console.log('allSubTasks=', allSubTasks.length)
+
+            let allSubTasksId = [];
+    
+            allSubTasks.forEach((info, i)=>{
+                let subid = i;
+                info.subid = subid;
+                allSubTasksId.push(subid+'');
+                let taskfilepath = pathutil.resolve(taskFolderNew, `./${info.subid}.task.json`);
+                fs.writeFileSync(taskfilepath, JSON.stringify(info));
+            });
+    
+            callback({
+                masterTaskId,
+                allSubTasksId,
+                //mainRptFolder,
+                progressRootFolder,
+                resultsRootFolder,
+
+                taskFolder,
+                taskFolderNew,
+                taskFolderRunning,
+                taskFolderFinished,
+                taskFolderResult,
+                taskFolderResultFailed,
+            });
+        });
+    }
+}
+
 module.exports = SuperProducer;