multi-tasks 2.0.1 → 2.0.3

package/workers/Multithread.js

@@ -1,177 +1,197 @@
-var fs = require('fs');
-var pathutil = require('path');
-const cluster = require('cluster');
-const numCPUs = require('os').cpus().length;
-
-let MSG_REQUEST_NEW_TASK = 'request_new_task';
-let MSG_NO_TASK_FOUND = 'no_task_found';
-
-const TaskMgr = require('./TaskMgr');
-const WorkerMgr = require('./WorkerMgr');
-const helper = require('./helper');
-
-let fmtdigit = (n)=>{
-    return n > 9 ? "" + n: "0" + n;
-};
-let getDateTimeTxt = ()=>{
-    let d=new Date();
-    let YYYY=d.getFullYear()
-    let MM=fmtdigit(d.getMonth()+1);
-    let DD=fmtdigit(d.getDate());
-    let HH=fmtdigit(d.getHours());
-    let mm=fmtdigit(d.getMinutes());
-    let ss=fmtdigit(d.getSeconds());
-    //return moment().format('YYYY-MM-DD HH:mm:ss');
-    return `${YYYY}-${MM}-${DD} ${HH}:${mm}:${ss}`;
-};
-
-let USER_CONFIG;
-
-let taskCount = 0;
-
-const start = (config)=>{    
-    USER_CONFIG = config;//this will remain user defined functions
-    let isMaster = (cluster.isPrimary || cluster.isMaster);
-
-    if (isMaster) {//init
-        console.log('[Master]: Start!!');
-        let PID = process.pid;
-        let numOfWorkers = config.numberOfWorkers;
-        console.log(`  numCPUs=${numCPUs}`);
-        console.log(`  numOfWorkers=${numOfWorkers}`);
-        console.log(`  PID=${PID}`);
-
-        const AsyncQueue = require(`../utils/asyncQueue`);
-        const queue = new AsyncQueue();  
-
-
-        require('./asMaster').init(config).then((updated_config)=>{
-            console.log(`  new tasks=${TaskMgr.getTasks('new').length}`);
-            config = updated_config;
-            for (let i = 0; i < numOfWorkers; i++) {
-                let worker = cluster.fork();
-                WorkerMgr.addWorker(worker);
-                worker.on('message', function(message) {
-                    if(message === MSG_REQUEST_NEW_TASK){
-                        queue.enqueue(async ()=>{
-                            return TaskMgr.updateNewTasks();
-                        });//use queue to make sure no I/O conflict
-                        queue.enqueue(async () => {
-                            return TaskMgr.popNewTask().then((subTask)=>{
-                                if(!subTask){
-                                    //no tasks left
-                                    console.log('[Master]: no tasks left!');
-                                    //console.log('workers', cluster.workers);
-                                    if(config.autoCloseAfterCompletion){
-                                        worker.kill();
-                                    }else{//waiting for new tasks
-                                        worker.send(MSG_NO_TASK_FOUND,()=>{}, {
-                                            keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
-                                        });
-                                    }
-                                    //
-                                }else{
-                                    //send new tasks to the worker
-                                    worker.send({
-                                        taskCount,
-                                        startTimestamp: new Date()*1,
-                                        config,
-                                        subTask
-                                    },()=>{}, {
-                                        keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
-                                    });
-                                    taskCount++;
-                                }
-                            });
-                        });
-                    }  
-                });
-            }
-        });
-        cluster.on('exit', (worker, code, signal) => { 
-            let allkilled = true; 
-            for (const worker of Object.values(cluster.workers)) {
-                allkilled = allkilled && worker.killed;
-            }
-            if(allkilled){
-                if(USER_CONFIG.onFinish){
-                    USER_CONFIG.onFinish({});
-                    delete USER_CONFIG.onFinish;
-                }
-                setTimeout(()=>{
-                    console.log('[Master]: all workers are done, process exit.')
-                    process.exit(0);
-                }, 10);
-            }
-        }); 
-    }else{
-        subWorker();
-    }
-};
-function subWorker(){
-    let worker = cluster.worker;
-    //console.log(`>[w]=${process.pid} started`);
-    process.on('message', function(message) {
-        //
-    });
-    worker.on('message', function(message) {
-        if(message === MSG_NO_TASK_FOUND){//wait for new task
-            console.log(`  (Worker #${worker.id}): waiting`);
-            setTimeout(()=>{
-                worker.send(MSG_REQUEST_NEW_TASK);//request new task
-            }, 5000 + Math.random()*5000);
-        }else
-        if(message.subTask){//new task
-            let {config, subTask, startTimestamp, taskCount} = message;
-            TaskMgr.load(config);
-            //console.log('ssss', subTask)
-            if(subTask){
-                subTask.index = taskCount;//for v1 backward
-                subTask.taskCount = taskCount;
-                console.log(`[Task Start]: "${subTask.subid}"`, `pid=${process.pid}`, getDateTimeTxt());
-                runTask(config, subTask, message).then((report)=>{
-                    let endTimestamp = new Date()*1;
-                    let cost = startTimestamp - endTimestamp;
-    
-                    console.log(`[Task Finished] "${config.masterTaskId}"."${subTask.subid}".`, getDateTimeTxt(), `cost=${cost}ms`)
-                }).then(()=>{
-                    worker.send(MSG_REQUEST_NEW_TASK);//request new task
-                });
-            }else{
-                console.log('Can not find: message.subTask');
-                throw 'can not find subTask!'
-            }
-        }
-    });
-    worker.send(MSG_REQUEST_NEW_TASK);//request new task
-
-};
-function runTask(config, subTask, message){
-    return new Promise((resolve)=>{ 
-        let resultPromise = Promise.resolve({});
-        let resultData;
-        let errors = [];
-        helper.load(config);
-        try{
-            resultPromise = USER_CONFIG.processTask(subTask, helper);//rewrite this method in subclass
-        }catch(e){
-            errors.push({subTask, exception: 'Exception:' + e.toString()});
-        }
-        let ispromise = resultPromise instanceof Promise;
-        if(!ispromise){resultPromise = Promise.resolve(resultPromise);}//convert data to promise
-        resultPromise.then((data)=>{
-            resultData = data;
-        }).catch((ex)=>{
-            if(typeof ex !== 'string') ex = ex.toString();
-            errors.push({ex});
-        }).finally(()=>{
-            let hasError = errors.length > 0;
-            if(hasError)console.log('hasError=====', hasError, errors)
-            TaskMgr.saveTaskResult(subTask.subid, hasError, hasError ? errors : resultData);
-            TaskMgr.mvTask(subTask.subid, 'running',(hasError ? 'finished_with_errors':'finished'), (newPath)=>{
-                resolve();
-            });   
-        });
-    });
-};
+var fs = require('fs');
+var pathutil = require('path');
+const cluster = require('cluster');
+const numCPUs = require('os').cpus().length;
+
+let MSG_REQUEST_NEW_TASK = 'request_new_task';
+let MSG_NO_TASK_FOUND = 'no_task_found';
+
+const asMaster = require('./asMaster');
+const TaskMgr = require('./TaskMgr');
+const WorkerMgr = require('./WorkerMgr');
+const helper = require('./helper');
+
+let fmtdigit = (n)=>{
+    return n > 9 ? "" + n: "0" + n;
+};
+let getDateTimeTxt = ()=>{
+    let d=new Date();
+    let YYYY=d.getFullYear()
+    let MM=fmtdigit(d.getMonth()+1);
+    let DD=fmtdigit(d.getDate());
+    let HH=fmtdigit(d.getHours());
+    let mm=fmtdigit(d.getMinutes());
+    let ss=fmtdigit(d.getSeconds());
+    //return moment().format('YYYY-MM-DD HH:mm:ss');
+    return `${YYYY}-${MM}-${DD} ${HH}:${mm}:${ss}`;
+};
+
+let USER_CONFIG;
+
+let taskCount = 0;
+
+let killAllWorkers = ()=>{  
+    for (const id in cluster.workers) {  
+        if (cluster.workers.hasOwnProperty(id)) {  
+            const worker = cluster.workers[id];  
+            console.log(`Killing worker ${worker.process.pid}`);  
+            killWorker(worker);
+        }  
+    }  
+};
+let killWorker = (worker)=>{
+    worker.disconnect(); 
+    //worker.process.kill(); 
+    worker.kill();
+};
+
+const start = (config)=>{    
+    USER_CONFIG = config;//this will remain user defined functions
+    let isMaster = (cluster.isPrimary || cluster.isMaster);
+
+    if (isMaster) {//init
+        console.log('[Master]: Start!!');
+        let PID = process.pid;
+        let numOfWorkers = config.numberOfWorkers;
+        console.log(`  numCPUs=${numCPUs}`);
+        console.log(`  numOfWorkers=${numOfWorkers}`);
+        console.log(`  PID=${PID}`);
+
+        const AsyncQueue = require(`../utils/asyncQueue`);
+        const queue = new AsyncQueue();  
+
+        let initPromise = config.__resume ? asMaster.resume(config) : asMaster.init(config);
+        initPromise.then((updated_config)=>{
+            console.log(`  new tasks=${TaskMgr.getTasks('new').length}`);
+            config = updated_config;
+            for (let i = 0; i < numOfWorkers; i++) {
+                let worker = cluster.fork();
+                WorkerMgr.addWorker(worker);
+                worker.on('message', function(message) {
+                    if(message === MSG_REQUEST_NEW_TASK){
+                        queue.enqueue(async ()=>{
+                            return TaskMgr.updateNewTasks();
+                        });//use queue to make sure no I/O conflict
+                        queue.enqueue(async () => {
+                            return TaskMgr.popNewTask().then((subTask)=>{
+                                if(!subTask){
+                                    //no tasks left
+                                    console.log('[Master]: no tasks left!');
+                                    //console.log('workers', cluster.workers);
+                                    if(config.autoCloseAfterCompletion){
+                                        killWorker(worker);
+                                    }else{//waiting for new tasks
+                                        worker.send(MSG_NO_TASK_FOUND,()=>{}, {
+                                            keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
+                                        });
+                                    }
+                                }else{
+                                    let terminateData = {taskCount,subTask,config, id: worker.id};
+                                    if(USER_CONFIG.shouldTerminate && USER_CONFIG.shouldTerminate(terminateData)===true) {
+                                        killAllWorkers();
+                                        return process.exit(0);
+                                    }
+                                    //send new tasks to the worker
+                                    worker.send({
+                                        taskCount,
+                                        startTimestamp: new Date()*1,
+                                        config,
+                                        subTask
+                                    },()=>{}, {
+                                        keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
+                                    });
+                                    taskCount++;
+                                }
+                            });
+                        });
+                    }  
+                });
+            }
+        });
+        cluster.on('exit', (worker, code, signal) => { 
+            let allkilled = true; 
+            for (const worker of Object.values(cluster.workers)) {
+                allkilled = allkilled && worker.killed;
+            }
+            if(allkilled){
+                if(USER_CONFIG.onFinish){
+                    USER_CONFIG.onFinish({});
+                    delete USER_CONFIG.onFinish;
+                }
+                setTimeout(()=>{
+                    console.log('[Master]: all workers are done, process exit.')
+                    process.exit(0);
+                }, 10);
+            }
+        }); 
+    }else{
+        subWorker();
+    }
+};
+function subWorker(){
+    let worker = cluster.worker;
+    //console.log(`>[w]=${process.pid} started`);
+    process.on('message', function(message) {
+        //
+    });
+    worker.on('message', function(message) {
+        if(message === MSG_NO_TASK_FOUND){//wait for new task
+            console.log(`  (Worker #${worker.id}): waiting`);
+            setTimeout(()=>{
+                worker.send(MSG_REQUEST_NEW_TASK);//request new task
+            }, 5000 + Math.random()*5000);
+        }else
+        if(message.subTask){//new task
+            let {config, subTask, startTimestamp, taskCount} = message;
+            TaskMgr.load(config);
+            //console.log('ssss', subTask)
+            if(subTask){
+                subTask.index = taskCount;//for v1 backward
+                subTask.taskCount = taskCount;
+                console.log(`[Task Start]: "${subTask.subid}"`, `pid=${process.pid}`, getDateTimeTxt());
+                runTask(config, subTask, message).then((report)=>{
+                    let endTimestamp = new Date()*1;
+                    let cost = startTimestamp - endTimestamp;
+    
+                    console.log(`[Task Finished] "${config.masterTaskId}"."${subTask.subid}".`, getDateTimeTxt(), `cost=${cost}ms`)
+                }).then(()=>{
+                    worker.send(MSG_REQUEST_NEW_TASK);//request new task
+                });
+            }else{
+                console.log('Can not find: message.subTask');
+                throw 'can not find subTask!'
+            }
+        }
+    });
+    worker.send(MSG_REQUEST_NEW_TASK);//request new task
+
+};
+function runTask(config, subTask, message){
+    return new Promise((resolve)=>{ 
+        let resultPromise = Promise.resolve({});
+        let resultData;
+        let errors = [];
+        helper.load(config);
+        try{
+            resultPromise = USER_CONFIG.processTask(subTask, helper);//rewrite this method in subclass
+        }catch(e){
+            errors.push({subTask, exception: 'Exception:' + e.toString()});
+        }
+        let ispromise = resultPromise instanceof Promise;
+        if(!ispromise){resultPromise = Promise.resolve(resultPromise);}//convert data to promise
+        resultPromise.then((data)=>{
+            resultData = data;
+        }).catch((ex)=>{
+            if(typeof ex !== 'string') ex = ex.toString();
+            errors.push({ex});
+        }).finally(()=>{
+            let hasError = errors.length > 0;
+            if(hasError)console.log('hasError=====', hasError, errors)
+            TaskMgr.saveTaskResult(subTask.subid, hasError, hasError ? errors : resultData);
+            TaskMgr.mvTask(subTask.subid, 'running',(hasError ? 'finished_with_errors':'finished'), (newPath)=>{
+                resolve();
+            });   
+        });
+    });
+};
 module.exports = {start};