multi-tasks 1.2.7 → 2.0.0

package/workers/Multithread.js

@@ -0,0 +1,175 @@
+var fs = require('fs');
+var pathutil = require('path');
+const cluster = require('cluster');
+const numCPUs = require('os').cpus().length;
+
+let MSG_REQUEST_NEW_TASK = 'request_new_task';
+let MSG_NO_TASK_FOUND = 'no_task_found';
+
+const TaskMgr = require('./TaskMgr');
+const WorkerMgr = require('./WorkerMgr');
+const helper = require('./helper');
+
+let fmtdigit = (n)=>{
+    return n > 9 ? "" + n: "0" + n;
+};
+let getDateTimeTxt = ()=>{
+    let d=new Date();
+    let YYYY=d.getFullYear()
+    let MM=fmtdigit(d.getMonth()+1);
+    let DD=fmtdigit(d.getDate());
+    let HH=fmtdigit(d.getHours());
+    let mm=fmtdigit(d.getMinutes());
+    let ss=fmtdigit(d.getSeconds());
+    //return moment().format('YYYY-MM-DD HH:mm:ss');
+    return `${YYYY}-${MM}-${DD} ${HH}:${mm}:${ss}`;
+};
+
+let USER_CONFIG;
+
+let taskCount = 0;
+
+const start = (config)=>{    
+    USER_CONFIG = config;//this will remain user defined functions
+    let isMaster = (cluster.isPrimary || cluster.isMaster);
+
+    if (isMaster) {//init
+        console.log('[Master]: Start!!');
+        let PID = process.pid;
+        let numOfWorkers = config.numberOfWorkers;
+        console.log(`  numCPUs=${numCPUs}`);
+        console.log(`  numOfWorkers=${numOfWorkers}`);
+        console.log(`  PID=${PID}`);
+
+        const AsyncQueue = require(`../utils/asyncQueue`);
+        const queue = new AsyncQueue();  
+
+
+        require('./asMaster').init(config).then((updated_config)=>{
+            console.log(`  new tasks=${TaskMgr.getTasks('new').length}`);
+            config = updated_config;
+            for (let i = 0; i < numOfWorkers; i++) {
+                let worker = cluster.fork();
+                WorkerMgr.addWorker(worker);
+                worker.on('message', function(message) {
+                    if(message === MSG_REQUEST_NEW_TASK){
+                        queue.enqueue(TaskMgr.updateNewTasks);//use queue to make sure no I/O conflict
+                        queue.enqueue(async () => {
+                            TaskMgr.popNewTask().then((subTask)=>{
+                                if(!subTask){
+                                    //no tasks left
+                                    console.log('[Master]: no tasks left!');
+                                    //console.log('workers', cluster.workers);
+                                    if(config.autoCloseAfterCompletion){
+                                        worker.kill();
+                                    }else{//waiting for new tasks
+                                        worker.send(MSG_NO_TASK_FOUND,()=>{}, {
+                                            keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
+                                        });
+                                    }
+                                    //
+                                }else{
+                                    //send new tasks to the worker
+                                    worker.send({
+                                        taskCount,
+                                        startTimestamp: new Date()*1,
+                                        config,
+                                        subTask
+                                    },()=>{}, {
+                                        keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
+                                    });
+                                    taskCount++;
+                                }
+                            });
+                        });
+                    }  
+                });
+            }
+        });
+        cluster.on('exit', (worker, code, signal) => { 
+            let allkilled = true; 
+            for (const worker of Object.values(cluster.workers)) {
+                allkilled = allkilled && worker.killed;
+            }
+            if(allkilled){
+                if(USER_CONFIG.onFinish){
+                    USER_CONFIG.onFinish({});
+                    delete USER_CONFIG.onFinish;
+                }
+                setTimeout(()=>{
+                    console.log('[Master]: all workers are done, process exit.')
+                    process.exit(0);
+                }, 1000);
+            }
+        }); 
+    }else{
+        subWorker();
+    }
+};
+function subWorker(){
+    let worker = cluster.worker;
+    //console.log(`>[w]=${process.pid} started`);
+    process.on('message', function(message) {
+        //
+    });
+    worker.on('message', function(message) {
+        if(message === MSG_NO_TASK_FOUND){//wait for new task
+            console.log(`  (Worker #${worker.id}): waiting`);
+            setTimeout(()=>{
+                worker.send(MSG_REQUEST_NEW_TASK);//request new task
+            }, 5000 + Math.random()*5000);
+        }else
+        if(message.subTask){//new task
+            let {config, subTask, startTimestamp, taskCount} = message;
+            TaskMgr.load(config);
+            //console.log('ssss', subTask)
+            if(subTask){
+                subTask.index = taskCount;//for v1 backward
+                subTask.taskCount = taskCount;
+                console.log(`[Task Start]: "${subTask.subid}"`, `pid=${process.pid}`, getDateTimeTxt());
+                runTask(config, subTask, message).then((report)=>{
+                    let endTimestamp = new Date()*1;
+                    let cost = startTimestamp - endTimestamp;
+    
+                    console.log(`[Task Finished] "${config.masterTaskId}"."${subTask.subid}".`, getDateTimeTxt(), `cost=${cost}ms`)
+                }).then(()=>{
+                    worker.send(MSG_REQUEST_NEW_TASK);//request new task
+                });
+            }else{
+                console.log('Can not find: message.subTask');
+                throw 'can not find subTask!'
+            }
+        }
+    });
+    worker.send(MSG_REQUEST_NEW_TASK);//request new task
+
+};
+function runTask(config, subTask, message){
+    return new Promise((resolve)=>{ 
+        let resultPromise = Promise.resolve({});
+        let resultData;
+        let errors = [];
+        helper.load(config);
+        try{
+            resultPromise = USER_CONFIG.processTask(subTask, helper);//rewrite this method in subclass
+        }catch(e){
+            errors.push({subTask, exception: 'Exception:' + e.toString()});
+        }
+        let ispromise = resultPromise instanceof Promise;
+        if(!ispromise){resultPromise = Promise.resolve(resultPromise);}//convert data to promise
+        resultPromise.then((data)=>{
+            resultData = data;
+        }).catch((ex)=>{
+            if(typeof ex !== 'string') ex = ex.toString();
+            errors.push({ex});
+        }).finally(()=>{
+            let hasError = errors.length > 0;
+            if(hasError)console.log('hasError=====', hasError, errors)
+            TaskMgr.saveTaskResult(subTask.subid, hasError, hasError ? errors : resultData);
+            TaskMgr.mvTask(subTask.subid, 'running',(hasError ? 'finished_with_errors':'finished'), (newPath)=>{
+                resolve();
+            });   
+        });
+    });
+};
+module.exports = {start};

package/workers/TaskMgr.js

@@ -0,0 +1,186 @@
+const fs = require('fs');
+const pathutil = require('path');
+const makeDir = require('../utils/makedir');
+const randomsUtil = require('../utils/randoms');
+
+let taskCounter = 0;
+
+let taskFolder;
+let progressRootFolder;
+let resultsRootFolder;
+
+let taskFolderNew;
+let taskFolderRunning;
+let taskFolderFinished;;
+let taskFolderFailed;
+
+let taskFolderResult;
+let taskFolderResultFailed;
+
+let allSubTasks = {
+    new:[],
+    running:[],
+    finished:[],
+    finished_with_errors:[]
+};
+
+let main_config;
+
+const TaskMgr = {
+    load:(config)=>{
+        main_config = config;
+        return TaskMgr;
+    },
+    initFolders:()=>{
+        let {taskRootFolder, masterTaskId} = main_config;
+        taskFolder = pathutil.resolve(taskRootFolder, masterTaskId);
+        if(fs.existsSync(taskFolder)){
+            let files = fs.readdirSync(taskFolder);
+            if(files.length > 0){
+                console.log('[FATAL] taskRootFolder already exist and not empty, delete it and try again: ', taskFolder);
+                return process.exit(0);
+            }
+        };
+        //let taskfolder = pathutil.parse(__filename).dir;
+        //let taskFolder = pathutil.resolve(taskfolder, `./tasks/${masterTaskId}`);
+        progressRootFolder = pathutil.resolve(taskFolder, './progress');
+        resultsRootFolder = pathutil.resolve(taskFolder, './results');
+
+        taskFolderNew = pathutil.resolve(progressRootFolder, './new');
+        taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
+        taskFolderFinished = pathutil.resolve(progressRootFolder, './finished');
+        taskFolderFailed= pathutil.resolve(progressRootFolder, './finished_with_errors');
+
+        taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
+        taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
+        //let mainRptFolder = pathutil.resolve(taskfolder, './report');
+    
+        //makeDir.sync(mainRptFolder);
+        makeDir.sync(taskFolderNew);
+        makeDir.sync(taskFolderRunning);
+        makeDir.sync(taskFolderFinished);
+        makeDir.sync(taskFolderFailed);
+        makeDir.sync(taskFolderResult);
+        makeDir.sync(taskFolderResultFailed);
+
+        main_config = {
+            ...main_config,
+            taskFolder,
+            progressRootFolder,
+            resultsRootFolder,
+
+            taskFolderNew,
+            taskFolderRunning,
+            taskFolderFinished,
+            taskFolderFailed,
+            taskFolderResult,
+            taskFolderResultFailed,
+        };
+
+        //for restart a failed task
+        let main_config2 = JSON.parse(JSON.stringify(main_config));
+        delete main_config2.tasks;//will load from folder 'new', so no need the early tasks data
+        fs.writeFileSync(pathutil.resolve(taskFolder, 'task_config.json'), JSON.stringify(main_config2));
+        return main_config;
+    },
+    createNewTask:(task)=>{
+        const {masterTaskId, taskFolderNew} = main_config;
+        const subid = randomsUtil.getDateTimeTxt() + Math.random();
+
+        task.masterTaskId = masterTaskId;
+        task.subid = `${masterTaskId}-${subid}`;
+        let taskfilepath = pathutil.resolve(taskFolderNew, `./${task.subid}.task.json`);
+        fs.writeFileSync(taskfilepath, JSON.stringify(task));
+    },
+    loadNewTasks:()=>{
+        const {masterTaskId, taskFolderNew} = main_config;
+        return new Promise((resolve)=>{
+            makeDir.getDirectFiles(taskFolderNew, (taskFiles)=>{
+                taskFiles.forEach((filename)=>{
+                    let fpath = pathutil.resolve(taskFolderNew, filename);
+                    let fcontent;
+                    try{
+                        fcontent = fs.readFileSync(fpath, 'utf-8');
+                    }catch(e){
+                        console.log('[WARN]', `file not found`, fpath)
+                    }
+                    if(fcontent){
+                        let task = JSON.parse(fcontent);
+                        allSubTasks.new.push(task);
+                    }
+                });
+                resolve()
+            });
+        });
+    },
+    updateNewTasks:()=>{
+        let p;
+        if(allSubTasks.new.length === 0){
+            p = TaskMgr.loadNewTasks();//load again
+        }else{
+            p = Promise.resolve();
+        }
+        return p;
+    },
+    popNewTask:()=>{
+        return new Promise((resolve)=>{
+            let newtask = allSubTasks.new.shift();
+            if(newtask){
+                TaskMgr.mvTask(newtask.subid, 'new', 'running', (newPath)=>{ 
+                    resolve(newtask);
+                });
+            }else{
+                resolve(null);
+            }
+        });
+    },
+    getTask:(type, id)=>{
+        let result;
+        let arr = allSubTasks[type];
+        for(let i=0;i<arr.length;i++){
+            let task = arr[i];
+            if(task.subid === id){
+                result = task;
+                break;
+            } 
+        }
+        return result;
+    },
+    getTasks:(type)=>{
+        return allSubTasks[type];
+    },
+    getTaskFiles:(type, callback)=>{
+        let rootdir = main_config.progressRootFolder;
+        let targetFolder = pathutil.resolve(rootdir, `./${type}`);
+        makeDir.getDirectFiles(targetFolder, (files)=>{
+            callback(files);
+        });
+    },
+    mvTask: (subid, from, to, callback)=> {
+        //console.log('moooooov=======', subid, from, to)
+        let {progressRootFolder} = main_config;
+        let oldPath = pathutil.resolve(progressRootFolder, `./${from}/${subid}.task.json`);
+        let newPath = pathutil.resolve(progressRootFolder, `./${to}/${subid}.task.json`);
+        try{
+            fs.rename(oldPath, newPath, (err) => {
+                //if (err) throw err;
+                callback(newPath);
+            });
+        }catch(e){
+            callback(newPath);
+        }
+    },
+    saveTaskResult:(subid, hasError, data)=>{        
+        let {taskFolderResult, taskFolderResultFailed} = main_config;
+        let folder = hasError ? taskFolderResultFailed : taskFolderResult;
+        let fpath = pathutil.resolve(folder, `${subid}.json`);
+        let fcontent;
+        if(typeof data === 'undefined') {
+            fcontent = 'undefined';
+        }else{
+            fcontent = JSON.stringify(data);
+        };
+        fs.writeFileSync(fpath, fcontent);
+    }
+};
+module.exports = TaskMgr;

package/workers/WorkerMgr.js

@@ -0,0 +1,31 @@
+const fs = require('fs');
+const pathutil = require('path');
+const makeDir = require('../utils/makedir');
+
+const BUSY = 1;
+const IDLE = 0;
+
+let workers = {};
+let status = {};
+
+const mgr = {
+    addWorker:(worker)=>{
+        let id = `${worker.id}`;
+        if(workers[id] || status[id]){
+            throw 'FATAL: duplicated worker!';
+        }
+        status[id] = {isBusy: true};
+        console.log('add worker', worker.id);
+        workers[id] = worker;
+    },
+    getWorker:(id)=>{
+        return workers[`${id}`];
+    },
+    setBusy:(id, isBusy)=>{
+
+    },
+    isBusy:(id)=>{
+
+    }
+};
+module.exports = mgr;

package/workers/asMaster.js

@@ -0,0 +1,71 @@
+var fs = require('fs');
+var pathutil = require('path');
+const numCPUs = require('os').cpus().length;
+
+let MSG_REQUEST_TASKID = 'request_taskid';
+
+const TaskMgr = require('./TaskMgr');
+const Multithread = require('./Multithread');
+
+
+const fmtDigit = (n)=>{
+    return n > 9 ? "" + n: "0" + n;
+}
+const getDateTimeTxt = ()=>{
+    let d=new Date();
+    let YYYY=d.getFullYear()
+    let MM=fmtDigit(d.getMonth()+1);
+    let DD=fmtDigit(d.getDate());
+    let HH=fmtDigit(d.getHours());
+    let mm=fmtDigit(d.getMinutes());
+    let ss=fmtDigit(d.getSeconds());
+    return `${YYYY}-${MM}-${DD}_${HH}-${mm}-${ss}`;
+};
+
+const init = (config)=>{
+    if(typeof config === 'undefined') config = {}; 
+    if(!config.tasks){ throw 'Please provide tasks data!';  };
+    if(!config.processTask){ throw 'Please provide processTask function!'; };
+    if(!config.tasks){ throw 'Must assign initial tasks!'; };
+
+    if(typeof config.numberOfWorkers === 'undefined') config.numberOfWorkers = numCPUs - 1;
+
+    let {taskRootFolder, taskName} = config;
+
+    if(taskName && !taskName.match(/[a-zA-Z0-9_-]/)){ throw 'Error: taskName can only contain letters and numbers'; }
+    if(!taskName){ taskName = 'multitasks'; }
+
+    let {round, pow, random} = Math;
+    let dateTimeText = getDateTimeTxt();//moment().format('YYYY-MM-DD_HH-mm-ss');
+    let randomNum = round((random() * pow(10,8)),8).toString(36);
+    let masterTaskId = config.taskId ? config.taskId : `${taskName}${dateTimeText}--${randomNum}`;
+
+    console.log(`  masterTaskId===${masterTaskId}`);
+
+    config.taskRootFolder = taskRootFolder;
+    config.masterTaskId = masterTaskId;
+
+
+    TaskMgr.load(config);
+    config = TaskMgr.initFolders(config);
+
+    config.tasks.forEach((task)=>{
+        TaskMgr.createNewTask(task);
+    });
+
+    // let tasks = TaskMgr.getTasks('new')
+    // let task =  tasks.shift();
+    // console.log(tasks)
+    // TaskMgr.mvTask(task.subid, 'new', 'finished',()=>{})
+
+    //to reload is for to reStart a collapsed task
+    return new Promise((resolve)=>{
+        TaskMgr.loadNewTasks().then(()=>{
+            console.log('******start multi-thread')
+            resolve(config);
+        });
+    });
+};
+module.exports = {
+    init
+};

package/workers/asWorker.js

@@ -0,0 +1,14 @@
+var fs = require('fs');
+var pathutil = require('path');
+const numCPUs = require('os').cpus().length;
+
+let MSG_REQUEST_TASKID = 'request_taskid';
+
+const TaskMgr = require('./TaskMgr');
+
+const init = (config, worker)=>{
+
+};
+module.exports = {
+    init
+};

package/workers/helper.js

@@ -0,0 +1,22 @@
+let fs = require('fs');
+let pathutil = require('path');
+
+let TaskMgr = require('./TaskMgr');
+
+let myconfig;
+const load = (config)=>{
+    myconfig = config;
+};
+const createNewTasks = (tasks)=>{
+
+    if (!Array.isArray(tasks)) tasks = [tasks];
+    TaskMgr.load(myconfig);
+    tasks.forEach((task)=>{
+        console.log('in a  helper, ck', task)
+        TaskMgr.createNewTask(task);
+    });
+};
+module.exports = {
+    load,
+    createNewTasks
+};

package/workers/index.js

@@ -0,0 +1,8 @@
+const Multithread = require('./Multithread');
+const backwardv1 = require('../utils/backward-v1')
+
+const multiTasks = (config)=>{
+    config = backwardv1.v1tov2(config);
+    Multithread.start(config);
+};
+module.exports = multiTasks;

package/workers/SuperConsumer.js

@@ -1,106 +0,0 @@
-let fs = require('fs');
-let pathutil = require('path');
-
-class SuperConsumer {
-    // constructor
-    constructor(mainid, subid) {
-        this.mainid = mainid;
-        this.subid = subid;
-        this.results = [];
-    }
-
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    _run (mainTaskInfo, subTaskInfo, actions, results, errors, callback){
-        if(actions.length === 0) return callback({
-                                            finishedAt: new Date()*1,
-                                            results,
-                                            errors,
-                                            subTaskInfo
-                                        });
-        let taskProp = actions.shift();
-    
-        if(this.processTask){
-            let result;
-            let hasException = false;
-            try{
-                result = this.processTask(taskProp);//rewrite this method in subclass
-            }catch(e){
-                hasException = true;
-                result = {taskProp, exception: 'Exception:' + e.toString()}
-            }
-            if(typeof result === 'undefined') result = {};
-            let ispromise = result instanceof Promise;
-            if(!ispromise){result = Promise.resolve(result);}
-            result.then((taskResult)=>{
-                if(!hasException){
-                    results.push({taskProp, taskResult});
-                }else{
-                    errors.push(taskResult);
-                }
-            }).catch((e)=>{
-                errors.push({taskProp, exception: 'Exception:' + e.toString()});
-            }).finally(()=>{
-                this._run(mainTaskInfo, subTaskInfo, actions, results, errors, callback);
-            });            
-        }else{
-            this._run(mainTaskInfo, subTaskInfo, actions, results, errors, callback);
-        }    
-    };
-    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
-        let results = [];
-        let errors = [];
-        let actions = subTaskInfo.actions;
-        mainTaskInfo.defination = this.trunkInfo.defination;
-        this._run(mainTaskInfo, subTaskInfo, actions, results, errors, (result)=>{
-            callback(result)
-        })
-
-    }
-    runTasks (mainTaskInfo, subTaskInfo, message){
-        // console.log(`mainTaskInfo:`, mainTaskInfo)
-        // console.log(`subTaskInfo:`, subTaskInfo)
-        // console.log(`message:`, message)
-
-        this.mainTaskInfo = mainTaskInfo;
-        this.subTaskInfo = subTaskInfo;
-
-        return new Promise((resolve, reject)=>{
-            let mainid = mainTaskInfo.masterTaskId;
-            let subid = subTaskInfo.subid;
-            let taskfolderNew = mainTaskInfo.taskFolderNew;
-            let taskFolderResult = mainTaskInfo.taskFolderResult;
-            let taskFolderResultFailed = mainTaskInfo.taskFolderResultFailed;            
-            this.mvTask(mainid, subid,'new','running', (taskFile)=>{//mark as running
-                let taskContent = fs.readFileSync(taskFile, 'utf8');
-                let taskInfo = JSON.parse(taskContent);
-                this.run(mainTaskInfo, subTaskInfo, taskInfo, (taskReport)=>{//invoke
-                    let {results, errors} = taskReport;
-                    let hasError = false;
-                    if(errors &&  errors.length > 0) hasError = true;
-                    let folder = !hasError ? taskFolderResult : taskFolderResultFailed;
-                    let resultfile = pathutil.resolve(folder, `${subid}.json`)
-                    fs.writeFileSync(resultfile, JSON.stringify(taskReport));
-                    this.mvTask(mainid, subid,'running',(hasError ? 'finished_with_errors':'finished'), ()=>{//mark as finished
-                        resolve({
-                            mainid, 
-                            subid,
-                            ...message
-                        })
-                    });
-                });            
-            });
-        });
-    }
-    mvTask (mainid, subid, from, to, callback) {
-        let __thisdir = this.mainTaskInfo.progressRootFolder;
-        let oldPath = pathutil.resolve(__thisdir, `./${from}/${subid}.task.json`);
-        let newPath = pathutil.resolve(__thisdir, `./${to}/${subid}.task.json`);
-        fs.rename(oldPath, newPath, (err) => {
-            if (err) throw err;
-            callback(newPath);
-        });
-    }
-}
-module.exports = SuperConsumer;