multi-tasks 1.2.6 → 1.2.8

package/workers/master.js

@@ -1,171 +1,171 @@
-var fs = require('fs');
-var pathutil = require('path');
-const cluster = require('cluster');
-const numCPUs = require('os').cpus().length;
-const report_status = require('./report/report_status');
-
-let MSG_REQUEST_TASKID = 'request_taskid';
-/**
- * masterId, the id of scanning flow
- *   -master, a master is a host of child_process computing
- *   -masterTaskId, id of a master, will rename to masterId 
- *     -task, a child computing process
- */
-let fmtdigit = (n)=>{
-  return n > 9 ? "" + n: "0" + n;
-}
- let getDateTimeTxt = ()=>{
-    let d=new Date();
-    let YYYY=d.getFullYear()
-    let MM=fmtdigit(d.getMonth()+1);
-    let DD=fmtdigit(d.getDate());
-    let HH=fmtdigit(d.getHours());
-    let mm=fmtdigit(d.getMinutes());
-    let ss=fmtdigit(d.getSeconds());
-    //return moment().format('YYYY-MM-DD HH:mm:ss');
-    return `${YYYY}-${MM}-${DD} ${HH}:${mm}:${ss}`;
-};
-let workidlist = new Set();
-let addWorker = (wid)=>{
-  wid = wid+'';
-  workidlist.add(wid);
-};
-let removeWorker = (wid)=>{
-  wid = wid+'';
-  workidlist.delete(wid);
-};
-let hasWorker = ()=>{
-  return workidlist.size !== 0;
-}
-let run = (currentProducer, currentConsumer, opt)=>{
-    let defaultCPUs = numCPUs - 1;
-    if(typeof opt === 'undefined') opt = {};    
-    let masterId = currentProducer.getTaskName();
-    if(typeof opt.quotaOfEachTask === 'undefined') opt.quotaOfEachTask = 1;
-    if(opt.numberOfWorks && !opt.numberOfWorkers) opt.numberOfWorkers = opt.numberOfWorks; //Backward compatible with the wrong variable name
-    if(typeof opt.numberOfWorkers === 'undefined') opt.numberOfWorkers = defaultCPUs;
-    console.log('master1, masterId=', masterId)
-
-
-    if(!masterId) throw `***Must assign masterId, pls use run.sh to run!***`
-
-    console.log('start', getDateTimeTxt())
-
-    if (cluster.isMaster) {//init
-      let numWorks = opt.numberOfWorkers;
-      console.log(`numCPUs ${numCPUs}`);
-      console.log(`numWorks ${numWorks}`);
-      console.log(`Master=${process.pid}`);
-
-      currentProducer.generateTasks(opt.quotaOfEachTask, (mainTaskInfo)=>{
-          //get ready
-          taskStack = mainTaskInfo.allSubTasksId;
-          let totalTaskNum = taskStack.length; 
-          console.log( 'tasks=', mainTaskInfo.masterTaskId);
-          console.log( 'totalTaskNum=', totalTaskNum);
-
-          for (let i = 0; i < numWorks; i++) {
-            let worker = cluster.fork();
-            addWorker(worker.id)
-            worker.on('message', function(message) {
-              //console.log(`(master).${process.pid} got: '${JSON.stringify(message)}' from w.${worker.process.pid}`);
-              if(message === MSG_REQUEST_TASKID){
-                if(taskStack.length > 0){     
-                  let subTaskId = taskStack.shift();
-                  let _sendMsg = ()=>{
-                    worker.send({
-                      startTimestamp: new Date()*1,
-                      mainTaskInfo,
-                      subTaskId,
-                      totalTaskNum
-                    },()=>{
-                      //handler
-                    }, {
-                      keepOpen: (typeof opt.keepOpen === 'undefined') ? false : opt.keepOpen
-                    })
-                  };
-                  _sendMsg();
-                }else{
-                  removeWorker(worker.id)
-                  worker.kill();
-                  console.log('no task left.', getDateTimeTxt());
-                  if(!hasWorker()){
-                    console.log('all done!')
-                    let taskFolder = mainTaskInfo.taskFolder;
-                    let dataForNextMission = {
-                      ...mainTaskInfo
-                    };
-                    let dataPath = pathutil.resolve(taskFolder, `./extractTaskData-${masterId}.json`);
-                    fs.writeFileSync(dataPath, JSON.stringify(dataForNextMission))
-                    console.log('dataForNextMission=', dataPath);
-                    if(opt.onFinish) {
-                      opt.taskFolder = taskFolder;
-                      let report = report_status.getReport(opt);
-                      opt.onFinish(report);
-                    }
-                  };
-                }
-              }      
-            });
-          }
-      });
-      
-
-    } else {
-      let worker = cluster.worker;
-      //console.log(`>[w]=${process.pid} started`);
-      process.on('message', function(message) {});
-      worker.on('message', function(message) {
-          if(message.subTaskId){
-              let mainTaskInfo = message.mainTaskInfo;
-              //console.log(message);
-              let subTaskFilePath = pathutil.resolve(mainTaskInfo.taskFolderNew, `./${message.subTaskId}.task.json`);
-              let subTaskInfo = JSON.parse(fs.readFileSync(subTaskFilePath));
-              //let subTaskInfo = message.subTaskInfo;
-              let subid = subTaskInfo.subid;
-              if(mainTaskInfo){
-                    console.log(`[Task Start]: "${subid}"`, `${subid}/${message.totalTaskNum}`, `pid=${process.pid}`, getDateTimeTxt());
-                  //fs.writeFileSync(pathutil.resolve(mainTaskInfo.taskFolderFinished, `./${subid}`), subTaskInfo.tasksLen);
-                  if(0){//mock
-                    setTimeout(()=>{
-                      worker.send(MSG_REQUEST_TASKID);//request new task
-                    }, 100);
-                  }else{//real
-                    currentConsumer.runTasks(mainTaskInfo, subTaskInfo, message).then((report)=>{
-                      let endTimestamp = new Date()*1;
-                      let cost = report.startTimestamp - endTimestamp;
-      
-                        console.log(`[Task Finished] "${report.mainid}"."${report.subid}".`, getDateTimeTxt(), `cost=${cost}ms`)
-                        worker.send(MSG_REQUEST_TASKID);//request new task
-                    });
-                  }
-              }else{
-                  console.log('Can not find: message.subTaskInfo')
-              }
-          }
-      });
-      worker.send(MSG_REQUEST_TASKID);//request new task
-
-    }
-};
-let start = (ProducerClass, ConsumerClass, taskConfig)=>{  
-  let taskId = Math.random()+'';
-  let {multi_task_parent_folder} = taskConfig;
-  
-  let producer = new ProducerClass(`workers_of_${taskId}`, multi_task_parent_folder)
-  let consumer = new ConsumerClass();
-
-  console.log(producer.getTaskName())
-  //return;
-
-  //console.log(resp)
-  producer.setTaskTrunkInfo(taskConfig);
-  consumer.setTaskTrunkInfo(taskConfig);
-  run(producer, consumer, {
-      ...taskConfig
-  });
-}
-module.exports = {
-  run,
-  start
+var fs = require('fs');
+var pathutil = require('path');
+const cluster = require('cluster');
+const numCPUs = require('os').cpus().length;
+const report_status = require('./report/report_status');
+
+let MSG_REQUEST_TASKID = 'request_taskid';
+/**
+ * masterId, the id of scanning flow
+ *   -master, a master is a host of child_process computing
+ *   -masterTaskId, id of a master, will rename to masterId 
+ *     -task, a child computing process
+ */
+let fmtdigit = (n)=>{
+  return n > 9 ? "" + n: "0" + n;
+}
+ let getDateTimeTxt = ()=>{
+    let d=new Date();
+    let YYYY=d.getFullYear()
+    let MM=fmtdigit(d.getMonth()+1);
+    let DD=fmtdigit(d.getDate());
+    let HH=fmtdigit(d.getHours());
+    let mm=fmtdigit(d.getMinutes());
+    let ss=fmtdigit(d.getSeconds());
+    //return moment().format('YYYY-MM-DD HH:mm:ss');
+    return `${YYYY}-${MM}-${DD} ${HH}:${mm}:${ss}`;
+};
+let workidlist = new Set();
+let addWorker = (wid)=>{
+  wid = wid+'';
+  workidlist.add(wid);
+};
+let removeWorker = (wid)=>{
+  wid = wid+'';
+  workidlist.delete(wid);
+};
+let hasWorker = ()=>{
+  return workidlist.size !== 0;
+}
+let run = (currentProducer, currentConsumer, opt)=>{
+    let defaultCPUs = numCPUs - 1;
+    if(typeof opt === 'undefined') opt = {};    
+    let masterId = currentProducer.getTaskName();
+    if(typeof opt.quotaOfEachTask === 'undefined') opt.quotaOfEachTask = 1;
+    if(opt.numberOfWorks && !opt.numberOfWorkers) opt.numberOfWorkers = opt.numberOfWorks; //Backward compatible with the wrong variable name
+    if(typeof opt.numberOfWorkers === 'undefined') opt.numberOfWorkers = defaultCPUs;
+    console.log('master1, masterId=', masterId)
+
+
+    if(!masterId) throw `***Must assign masterId, pls use run.sh to run!***`
+
+    console.log('start', getDateTimeTxt())
+
+    if (cluster.isMaster) {//init
+      let numWorks = opt.numberOfWorkers;
+      console.log(`numCPUs ${numCPUs}`);
+      console.log(`numWorks ${numWorks}`);
+      console.log(`Master=${process.pid}`);
+
+      currentProducer.generateTasks(opt.quotaOfEachTask, (mainTaskInfo)=>{
+          //get ready
+          taskStack = mainTaskInfo.allSubTasksId;
+          let totalTaskNum = taskStack.length; 
+          console.log( 'tasks=', mainTaskInfo.masterTaskId);
+          console.log( 'totalTaskNum=', totalTaskNum);
+
+          for (let i = 0; i < numWorks; i++) {
+            let worker = cluster.fork();
+            addWorker(worker.id)
+            worker.on('message', function(message) {
+              //console.log(`(master).${process.pid} got: '${JSON.stringify(message)}' from w.${worker.process.pid}`);
+              if(message === MSG_REQUEST_TASKID){
+                if(taskStack.length > 0){     
+                  let subTaskId = taskStack.shift();
+                  let _sendMsg = ()=>{
+                    worker.send({
+                      startTimestamp: new Date()*1,
+                      mainTaskInfo,
+                      subTaskId,
+                      totalTaskNum
+                    },()=>{
+                      //handler
+                    }, {
+                      keepOpen: (typeof opt.keepOpen === 'undefined') ? false : opt.keepOpen
+                    })
+                  };
+                  _sendMsg();
+                }else{
+                  removeWorker(worker.id)
+                  worker.kill();
+                  console.log('no task left.', getDateTimeTxt());
+                  if(!hasWorker()){
+                    console.log('all done!')
+                    let taskFolder = mainTaskInfo.taskFolder;
+                    let dataForNextMission = {
+                      ...mainTaskInfo
+                    };
+                    let dataPath = pathutil.resolve(taskFolder, `./extractTaskData-${masterId}.json`);
+                    fs.writeFileSync(dataPath, JSON.stringify(dataForNextMission))
+                    console.log('dataForNextMission=', dataPath);
+                    if(opt.onFinish) {
+                      opt.taskFolder = taskFolder;
+                      let report = report_status.getReport(opt);
+                      opt.onFinish(report);
+                    }
+                  };
+                }
+              }      
+            });
+          }
+      });
+      
+
+    } else {
+      let worker = cluster.worker;
+      //console.log(`>[w]=${process.pid} started`);
+      process.on('message', function(message) {});
+      worker.on('message', function(message) {
+          if(message.subTaskId){
+              let mainTaskInfo = message.mainTaskInfo;
+              //console.log(message);
+              let subTaskFilePath = pathutil.resolve(mainTaskInfo.taskFolderNew, `./${message.subTaskId}.task.json`);
+              let subTaskInfo = JSON.parse(fs.readFileSync(subTaskFilePath));
+              //let subTaskInfo = message.subTaskInfo;
+              let subid = subTaskInfo.subid;
+              if(mainTaskInfo){
+                    console.log(`[Task Start]: "${subid}"`, `${subid}/${message.totalTaskNum}`, `pid=${process.pid}`, getDateTimeTxt());
+                  //fs.writeFileSync(pathutil.resolve(mainTaskInfo.taskFolderFinished, `./${subid}`), subTaskInfo.tasksLen);
+                  if(0){//mock
+                    setTimeout(()=>{
+                      worker.send(MSG_REQUEST_TASKID);//request new task
+                    }, 100);
+                  }else{//real
+                    currentConsumer.runTasks(mainTaskInfo, subTaskInfo, message).then((report)=>{
+                      let endTimestamp = new Date()*1;
+                      let cost = report.startTimestamp - endTimestamp;
+      
+                        console.log(`[Task Finished] "${report.mainid}"."${report.subid}".`, getDateTimeTxt(), `cost=${cost}ms`)
+                        worker.send(MSG_REQUEST_TASKID);//request new task
+                    });
+                  }
+              }else{
+                  console.log('Can not find: message.subTaskInfo')
+              }
+          }
+      });
+      worker.send(MSG_REQUEST_TASKID);//request new task
+
+    }
+};
+let start = (ProducerClass, ConsumerClass, taskConfig)=>{  
+  let taskId = Math.random()+'';
+  let {multi_task_parent_folder} = taskConfig;
+  
+  let producer = new ProducerClass(`workers_of_${taskId}`, multi_task_parent_folder)
+  let consumer = new ConsumerClass();
+
+  console.log(producer.getTaskName())
+  //return;
+
+  //console.log(resp)
+  producer.setTaskTrunkInfo(taskConfig);
+  consumer.setTaskTrunkInfo(taskConfig);
+  run(producer, consumer, {
+      ...taskConfig
+  });
+}
+module.exports = {
+  run,
+  start
 };

package/workers/report/report_status.js

@@ -1,52 +1,52 @@
-var fs = require('fs');
-var pathutil = require('path');
-
-function countDirectChildFiles(directoryPath) {
-    const files = fs.readdirSync(directoryPath);
-    const directChildFiles = files.filter(file => {
-      const filePath = pathutil.join(directoryPath, file);
-      return fs.statSync(filePath).isFile();
-    });
-    return directChildFiles;
-};
-
-let getReport = (taskConfig)=>{
-    let {taskFolder} = taskConfig;
-    let folder_result_errors = pathutil.resolve(taskFolder, './results/errors');
-    let folder_result_succ = pathutil.resolve(taskFolder, './results/succ');
-
-    let folder_new = pathutil.resolve(taskFolder, './progress/new');
-    let folder_running = pathutil.resolve(taskFolder, './progress/running');
-    let folder_finished = pathutil.resolve(taskFolder, './progress/finished');
-    let folder_finished_with_errors = pathutil.resolve(taskFolder, './progress/finished_with_errors');
-
-    let files_new = countDirectChildFiles(folder_new);
-    let files_running = countDirectChildFiles(folder_running);
-    let files_finished = countDirectChildFiles(folder_finished);
-    let files_finished_with_errors = countDirectChildFiles(folder_finished_with_errors);
-
-    let report = {
-        taskConfig,
-        task_folders:{
-            folder_new,
-            folder_running,
-            folder_finished,
-            folder_finished_with_errors
-        },
-        result_folders:{
-            folder_result_errors,
-            folder_result_succ
-        },
-        tasks:{
-            new: files_new.length,
-            running: files_running.length,
-            finished: files_finished.length,
-            finished_with_errors: files_finished_with_errors.length,
-        }
-    };
-    return report;
-};
-
-module.exports = {
-    getReport
+var fs = require('fs');
+var pathutil = require('path');
+
+function countDirectChildFiles(directoryPath) {
+    const files = fs.readdirSync(directoryPath);
+    const directChildFiles = files.filter(file => {
+      const filePath = pathutil.join(directoryPath, file);
+      return fs.statSync(filePath).isFile();
+    });
+    return directChildFiles;
+};
+
+let getReport = (taskConfig)=>{
+    let {taskFolder} = taskConfig;
+    let folder_result_errors = pathutil.resolve(taskFolder, './results/errors');
+    let folder_result_succ = pathutil.resolve(taskFolder, './results/succ');
+
+    let folder_new = pathutil.resolve(taskFolder, './progress/new');
+    let folder_running = pathutil.resolve(taskFolder, './progress/running');
+    let folder_finished = pathutil.resolve(taskFolder, './progress/finished');
+    let folder_finished_with_errors = pathutil.resolve(taskFolder, './progress/finished_with_errors');
+
+    let files_new = countDirectChildFiles(folder_new);
+    let files_running = countDirectChildFiles(folder_running);
+    let files_finished = countDirectChildFiles(folder_finished);
+    let files_finished_with_errors = countDirectChildFiles(folder_finished_with_errors);
+
+    let report = {
+        taskConfig,
+        task_folders:{
+            folder_new,
+            folder_running,
+            folder_finished,
+            folder_finished_with_errors
+        },
+        result_folders:{
+            folder_result_errors,
+            folder_result_succ
+        },
+        tasks:{
+            new: files_new.length,
+            running: files_running.length,
+            finished: files_finished.length,
+            finished_with_errors: files_finished_with_errors.length,
+        }
+    };
+    return report;
+};
+
+module.exports = {
+    getReport
 }

package/workers/taskSplitter.js

@@ -1,30 +1,30 @@
-let split = (list, quota)=>{
-    let totalFilesLen = list.length
-    console.log('total:', totalFilesLen,'actions');
-    let allSubTasks = [];
-    let actions = [];
-    let currentIdx = 0;
-    let subid = 0;
-    list.forEach((act)=>{
-        currentIdx++;
-        actions.push(act)
-        if(actions.length>=quota){
-            allSubTasks.push({
-                actions,
-                tasksLen: actions.length
-            });
-            actions = [];
-            subid++;
-        }else if(totalFilesLen === currentIdx){
-            allSubTasks.push({
-                actions,
-                tasksLen: actions.length
-            });
-        }
-        //console.log(totalFilesLen,'-', currentIdx)
-    });
-    return allSubTasks;
-}
-module.exports = {
-    split
+let split = (list, quota)=>{
+    let totalFilesLen = list.length
+    console.log('total:', totalFilesLen,'actions');
+    let allSubTasks = [];
+    let actions = [];
+    let currentIdx = 0;
+    let subid = 0;
+    list.forEach((act)=>{
+        currentIdx++;
+        actions.push(act)
+        if(actions.length>=quota){
+            allSubTasks.push({
+                actions,
+                tasksLen: actions.length
+            });
+            actions = [];
+            subid++;
+        }else if(totalFilesLen === currentIdx){
+            allSubTasks.push({
+                actions,
+                tasksLen: actions.length
+            });
+        }
+        //console.log(totalFilesLen,'-', currentIdx)
+    });
+    return allSubTasks;
+}
+module.exports = {
+    split
 }

package/workers/test/MyConsumer.js

@@ -1,25 +1,25 @@
-let SuperConsumer = require('../SuperConsumer');
-class TestConsumer extends SuperConsumer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
-        //do job
-        // console.log(`mainTaskInfo:`, mainTaskInfo)
-        // console.log(`subTaskInfo:`, subTaskInfo)
-        // console.log(`taskInfo:`, taskInfo)
-        //return;
-        //rewrite this
-        let succ = (Math.random()*1000)%3===0?true:false;
-        let result = {
-            succ
-        }        
-        setTimeout(()=>{
-            callback(result);
-        }, Math.random()*1000)
-    }
-  }
+let SuperConsumer = require('../SuperConsumer');
+class TestConsumer extends SuperConsumer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
+        //do job
+        // console.log(`mainTaskInfo:`, mainTaskInfo)
+        // console.log(`subTaskInfo:`, subTaskInfo)
+        // console.log(`taskInfo:`, taskInfo)
+        //return;
+        //rewrite this
+        let succ = (Math.random()*1000)%3===0?true:false;
+        let result = {
+            succ
+        }        
+        setTimeout(()=>{
+            callback(result);
+        }, Math.random()*1000)
+    }
+  }
 module.exports = TestConsumer;

package/workers/test/MyProducer.js

@@ -1,26 +1,26 @@
-let SuperProducer = require('../SuperProducer');
-class TestProducer extends SuperProducer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    getAllActions() {
-        return new Promise((resolve, reject)=>{
-            setTimeout(()=>{
-
-
-                let alltasks = []
-                for(let i=0;i<1673;i++){
-                    alltasks.push({
-                        xxx:111,
-                        zzz:222
-                    })
-                }
-                resolve(alltasks);
-            }, 3000)
-        });
-    }
-  }
+let SuperProducer = require('../SuperProducer');
+class TestProducer extends SuperProducer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    getAllActions() {
+        return new Promise((resolve, reject)=>{
+            setTimeout(()=>{
+
+
+                let alltasks = []
+                for(let i=0;i<1673;i++){
+                    alltasks.push({
+                        xxx:111,
+                        zzz:222
+                    })
+                }
+                resolve(alltasks);
+            }, 3000)
+        });
+    }
+  }
 module.exports = TestProducer;

package/workers/test/test_master.js

@@ -1,12 +1,12 @@
-let MyProducer = require('./MyProducer')
-let MyConsumer = require('./MyConsumer')
-let producer = new MyProducer(`atasknametest`, `D:\\tmp`)
-let consumer = new MyConsumer()
-let master = require('../master')
-
-
-master.run(producer, consumer, {
-
-    quotaOfEachTask: 3
-
+let MyProducer = require('./MyProducer')
+let MyConsumer = require('./MyConsumer')
+let producer = new MyProducer(`atasknametest`, `D:\\tmp`)
+let consumer = new MyConsumer()
+let master = require('../master')
+
+
+master.run(producer, consumer, {
+
+    quotaOfEachTask: 3
+
 })

package/workers/test/test_producer.js

@@ -1,4 +1,4 @@
-let TestProducer = require('./MyProducer')
-let producer = new TestProducer(`tasknametest`, `D:\\tmp`)
-
+let TestProducer = require('./MyProducer')
+let producer = new TestProducer(`tasknametest`, `D:\\tmp`)
+
 producer.generateTasks()

package/workers/test/test_splitter.js

@@ -1,7 +1,7 @@
-let splitter = require('../taskSplitter');
-let alltasks = []
-for(let i=0;i<16;i++){
-    alltasks.push(i)
-}
-let subtasks = splitter.split(alltasks, 5);
+let splitter = require('../taskSplitter');
+let alltasks = []
+for(let i=0;i<16;i++){
+    alltasks.push(i)
+}
+let subtasks = splitter.split(alltasks, 5);
 console.log(subtasks)