multi-tasks 1.2.8 → 2.0.0

package/workers/SuperProducer.js

@@ -1,97 +0,0 @@
-let fs = require('fs');
-let pathutil = require('path');
-let makeDir = require('../utils/makedir');
-let splitter = require('./taskSplitter');
-let fmtdigit = (n)=>{
-    return n > 9 ? "" + n: "0" + n;
-}
-let getDateTimeTxt = ()=>{
-    let d=new Date();
-    let YYYY=d.getFullYear()
-    let MM=fmtdigit(d.getMonth()+1);
-    let DD=fmtdigit(d.getDate());
-    let HH=fmtdigit(d.getHours());
-    let mm=fmtdigit(d.getMinutes());
-    let ss=fmtdigit(d.getSeconds());
-    return `${YYYY}-${MM}-${DD}_${HH}-${mm}-${ss}`;
-};
-class SuperProducer {
-    // constructor
-    constructor(taskname, parentfolder) {
-        this.taskname = taskname;
-        this.parentfolder = parentfolder;
-
-    }
-    
-    getTaskName(){
-        return this.taskname;
-    }
-    getAllActions (){
-        //rewrite this method and return all your actions need to run
-        return new Promise();
-    }
-    generateTasks (quota, callback){
-        if(typeof quota === 'undefined' || !quota)quota=10;
-        let dateTimeText = getDateTimeTxt();//moment().format('YYYY-MM-DD_HH-mm-ss');
-        let masterTaskId = `${this.taskname}_${dateTimeText}`;
-        console.log('masterTaskId=',masterTaskId);
-        this.taskfolder = pathutil.resolve(this.parentfolder, masterTaskId)
-        let taskFolder = this.taskfolder;
-        //let taskfolder = pathutil.parse(__filename).dir;
-        //let taskFolder = pathutil.resolve(taskfolder, `./tasks/${masterTaskId}`);
-        let progressRootFolder = pathutil.resolve(taskFolder, './progress');
-        let resultsRootFolder = pathutil.resolve(taskFolder, './results');
-
-        let taskFolderNew = pathutil.resolve(progressRootFolder, './new');
-        let taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
-        let taskFolderFinished = pathutil.resolve(progressRootFolder, './finished');
-        let taskFolderFailed= pathutil.resolve(progressRootFolder, './finished_with_errors');
-
-        let taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
-        let taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
-        //let mainRptFolder = pathutil.resolve(taskfolder, './report');
-    
-        //makeDir.sync(mainRptFolder);
-        makeDir.sync(taskFolderNew);
-        makeDir.sync(taskFolderRunning);
-        makeDir.sync(taskFolderFinished);
-        makeDir.sync(taskFolderFailed);
-        makeDir.sync(taskFolderResult);
-        makeDir.sync(taskFolderResultFailed);
-
-        this.getAllActions().then((allactions)=>{
-
-            let allSubTasks = splitter.split(allactions, quota);
-
-        
-            console.log('allSubTasks=', allSubTasks.length)
-
-            let allSubTasksId = [];
-    
-            allSubTasks.forEach((info, i)=>{
-                let subid = i;
-                info.subid = subid;
-                allSubTasksId.push(subid+'');
-                let taskfilepath = pathutil.resolve(taskFolderNew, `./${info.subid}.task.json`);
-                fs.writeFileSync(taskfilepath, JSON.stringify(info));
-            });
-    
-            callback({
-                masterTaskId,
-                allSubTasksId,
-                //mainRptFolder,
-                progressRootFolder,
-                resultsRootFolder,
-
-                taskFolder,
-                taskFolderNew,
-                taskFolderRunning,
-                taskFolderFinished,
-                taskFolderResult,
-                taskFolderResultFailed,
-            });
-        });
-    }
-}
-
-module.exports = SuperProducer;

package/workers/master.js

@@ -1,171 +0,0 @@
-var fs = require('fs');
-var pathutil = require('path');
-const cluster = require('cluster');
-const numCPUs = require('os').cpus().length;
-const report_status = require('./report/report_status');
-
-let MSG_REQUEST_TASKID = 'request_taskid';
-/**
- * masterId, the id of scanning flow
- *   -master, a master is a host of child_process computing
- *   -masterTaskId, id of a master, will rename to masterId 
- *     -task, a child computing process
- */
-let fmtdigit = (n)=>{
-  return n > 9 ? "" + n: "0" + n;
-}
- let getDateTimeTxt = ()=>{
-    let d=new Date();
-    let YYYY=d.getFullYear()
-    let MM=fmtdigit(d.getMonth()+1);
-    let DD=fmtdigit(d.getDate());
-    let HH=fmtdigit(d.getHours());
-    let mm=fmtdigit(d.getMinutes());
-    let ss=fmtdigit(d.getSeconds());
-    //return moment().format('YYYY-MM-DD HH:mm:ss');
-    return `${YYYY}-${MM}-${DD} ${HH}:${mm}:${ss}`;
-};
-let workidlist = new Set();
-let addWorker = (wid)=>{
-  wid = wid+'';
-  workidlist.add(wid);
-};
-let removeWorker = (wid)=>{
-  wid = wid+'';
-  workidlist.delete(wid);
-};
-let hasWorker = ()=>{
-  return workidlist.size !== 0;
-}
-let run = (currentProducer, currentConsumer, opt)=>{
-    let defaultCPUs = numCPUs - 1;
-    if(typeof opt === 'undefined') opt = {};    
-    let masterId = currentProducer.getTaskName();
-    if(typeof opt.quotaOfEachTask === 'undefined') opt.quotaOfEachTask = 1;
-    if(opt.numberOfWorks && !opt.numberOfWorkers) opt.numberOfWorkers = opt.numberOfWorks; //Backward compatible with the wrong variable name
-    if(typeof opt.numberOfWorkers === 'undefined') opt.numberOfWorkers = defaultCPUs;
-    console.log('master1, masterId=', masterId)
-
-
-    if(!masterId) throw `***Must assign masterId, pls use run.sh to run!***`
-
-    console.log('start', getDateTimeTxt())
-
-    if (cluster.isMaster) {//init
-      let numWorks = opt.numberOfWorkers;
-      console.log(`numCPUs ${numCPUs}`);
-      console.log(`numWorks ${numWorks}`);
-      console.log(`Master=${process.pid}`);
-
-      currentProducer.generateTasks(opt.quotaOfEachTask, (mainTaskInfo)=>{
-          //get ready
-          taskStack = mainTaskInfo.allSubTasksId;
-          let totalTaskNum = taskStack.length; 
-          console.log( 'tasks=', mainTaskInfo.masterTaskId);
-          console.log( 'totalTaskNum=', totalTaskNum);
-
-          for (let i = 0; i < numWorks; i++) {
-            let worker = cluster.fork();
-            addWorker(worker.id)
-            worker.on('message', function(message) {
-              //console.log(`(master).${process.pid} got: '${JSON.stringify(message)}' from w.${worker.process.pid}`);
-              if(message === MSG_REQUEST_TASKID){
-                if(taskStack.length > 0){     
-                  let subTaskId = taskStack.shift();
-                  let _sendMsg = ()=>{
-                    worker.send({
-                      startTimestamp: new Date()*1,
-                      mainTaskInfo,
-                      subTaskId,
-                      totalTaskNum
-                    },()=>{
-                      //handler
-                    }, {
-                      keepOpen: (typeof opt.keepOpen === 'undefined') ? false : opt.keepOpen
-                    })
-                  };
-                  _sendMsg();
-                }else{
-                  removeWorker(worker.id)
-                  worker.kill();
-                  console.log('no task left.', getDateTimeTxt());
-                  if(!hasWorker()){
-                    console.log('all done!')
-                    let taskFolder = mainTaskInfo.taskFolder;
-                    let dataForNextMission = {
-                      ...mainTaskInfo
-                    };
-                    let dataPath = pathutil.resolve(taskFolder, `./extractTaskData-${masterId}.json`);
-                    fs.writeFileSync(dataPath, JSON.stringify(dataForNextMission))
-                    console.log('dataForNextMission=', dataPath);
-                    if(opt.onFinish) {
-                      opt.taskFolder = taskFolder;
-                      let report = report_status.getReport(opt);
-                      opt.onFinish(report);
-                    }
-                  };
-                }
-              }      
-            });
-          }
-      });
-      
-
-    } else {
-      let worker = cluster.worker;
-      //console.log(`>[w]=${process.pid} started`);
-      process.on('message', function(message) {});
-      worker.on('message', function(message) {
-          if(message.subTaskId){
-              let mainTaskInfo = message.mainTaskInfo;
-              //console.log(message);
-              let subTaskFilePath = pathutil.resolve(mainTaskInfo.taskFolderNew, `./${message.subTaskId}.task.json`);
-              let subTaskInfo = JSON.parse(fs.readFileSync(subTaskFilePath));
-              //let subTaskInfo = message.subTaskInfo;
-              let subid = subTaskInfo.subid;
-              if(mainTaskInfo){
-                    console.log(`[Task Start]: "${subid}"`, `${subid}/${message.totalTaskNum}`, `pid=${process.pid}`, getDateTimeTxt());
-                  //fs.writeFileSync(pathutil.resolve(mainTaskInfo.taskFolderFinished, `./${subid}`), subTaskInfo.tasksLen);
-                  if(0){//mock
-                    setTimeout(()=>{
-                      worker.send(MSG_REQUEST_TASKID);//request new task
-                    }, 100);
-                  }else{//real
-                    currentConsumer.runTasks(mainTaskInfo, subTaskInfo, message).then((report)=>{
-                      let endTimestamp = new Date()*1;
-                      let cost = report.startTimestamp - endTimestamp;
-      
-                        console.log(`[Task Finished] "${report.mainid}"."${report.subid}".`, getDateTimeTxt(), `cost=${cost}ms`)
-                        worker.send(MSG_REQUEST_TASKID);//request new task
-                    });
-                  }
-              }else{
-                  console.log('Can not find: message.subTaskInfo')
-              }
-          }
-      });
-      worker.send(MSG_REQUEST_TASKID);//request new task
-
-    }
-};
-let start = (ProducerClass, ConsumerClass, taskConfig)=>{  
-  let taskId = Math.random()+'';
-  let {multi_task_parent_folder} = taskConfig;
-  
-  let producer = new ProducerClass(`workers_of_${taskId}`, multi_task_parent_folder)
-  let consumer = new ConsumerClass();
-
-  console.log(producer.getTaskName())
-  //return;
-
-  //console.log(resp)
-  producer.setTaskTrunkInfo(taskConfig);
-  consumer.setTaskTrunkInfo(taskConfig);
-  run(producer, consumer, {
-      ...taskConfig
-  });
-}
-module.exports = {
-  run,
-  start
-};

package/workers/report/report_status.js

@@ -1,52 +0,0 @@
-var fs = require('fs');
-var pathutil = require('path');
-
-function countDirectChildFiles(directoryPath) {
-    const files = fs.readdirSync(directoryPath);
-    const directChildFiles = files.filter(file => {
-      const filePath = pathutil.join(directoryPath, file);
-      return fs.statSync(filePath).isFile();
-    });
-    return directChildFiles;
-};
-
-let getReport = (taskConfig)=>{
-    let {taskFolder} = taskConfig;
-    let folder_result_errors = pathutil.resolve(taskFolder, './results/errors');
-    let folder_result_succ = pathutil.resolve(taskFolder, './results/succ');
-
-    let folder_new = pathutil.resolve(taskFolder, './progress/new');
-    let folder_running = pathutil.resolve(taskFolder, './progress/running');
-    let folder_finished = pathutil.resolve(taskFolder, './progress/finished');
-    let folder_finished_with_errors = pathutil.resolve(taskFolder, './progress/finished_with_errors');
-
-    let files_new = countDirectChildFiles(folder_new);
-    let files_running = countDirectChildFiles(folder_running);
-    let files_finished = countDirectChildFiles(folder_finished);
-    let files_finished_with_errors = countDirectChildFiles(folder_finished_with_errors);
-
-    let report = {
-        taskConfig,
-        task_folders:{
-            folder_new,
-            folder_running,
-            folder_finished,
-            folder_finished_with_errors
-        },
-        result_folders:{
-            folder_result_errors,
-            folder_result_succ
-        },
-        tasks:{
-            new: files_new.length,
-            running: files_running.length,
-            finished: files_finished.length,
-            finished_with_errors: files_finished_with_errors.length,
-        }
-    };
-    return report;
-};
-
-module.exports = {
-    getReport
-}

package/workers/taskSplitter.js

@@ -1,30 +0,0 @@
-let split = (list, quota)=>{
-    let totalFilesLen = list.length
-    console.log('total:', totalFilesLen,'actions');
-    let allSubTasks = [];
-    let actions = [];
-    let currentIdx = 0;
-    let subid = 0;
-    list.forEach((act)=>{
-        currentIdx++;
-        actions.push(act)
-        if(actions.length>=quota){
-            allSubTasks.push({
-                actions,
-                tasksLen: actions.length
-            });
-            actions = [];
-            subid++;
-        }else if(totalFilesLen === currentIdx){
-            allSubTasks.push({
-                actions,
-                tasksLen: actions.length
-            });
-        }
-        //console.log(totalFilesLen,'-', currentIdx)
-    });
-    return allSubTasks;
-}
-module.exports = {
-    split
-}

package/workers/test/MyConsumer.js

@@ -1,25 +0,0 @@
-let SuperConsumer = require('../SuperConsumer');
-class TestConsumer extends SuperConsumer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
-        //do job
-        // console.log(`mainTaskInfo:`, mainTaskInfo)
-        // console.log(`subTaskInfo:`, subTaskInfo)
-        // console.log(`taskInfo:`, taskInfo)
-        //return;
-        //rewrite this
-        let succ = (Math.random()*1000)%3===0?true:false;
-        let result = {
-            succ
-        }        
-        setTimeout(()=>{
-            callback(result);
-        }, Math.random()*1000)
-    }
-  }
-module.exports = TestConsumer;

package/workers/test/MyProducer.js

@@ -1,26 +0,0 @@
-let SuperProducer = require('../SuperProducer');
-class TestProducer extends SuperProducer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    getAllActions() {
-        return new Promise((resolve, reject)=>{
-            setTimeout(()=>{
-
-
-                let alltasks = []
-                for(let i=0;i<1673;i++){
-                    alltasks.push({
-                        xxx:111,
-                        zzz:222
-                    })
-                }
-                resolve(alltasks);
-            }, 3000)
-        });
-    }
-  }
-module.exports = TestProducer;

package/workers/test/test_master.js

@@ -1,12 +0,0 @@
-let MyProducer = require('./MyProducer')
-let MyConsumer = require('./MyConsumer')
-let producer = new MyProducer(`atasknametest`, `D:\\tmp`)
-let consumer = new MyConsumer()
-let master = require('../master')
-
-
-master.run(producer, consumer, {
-
-    quotaOfEachTask: 3
-
-})

package/workers/test/test_producer.js

@@ -1,4 +0,0 @@
-let TestProducer = require('./MyProducer')
-let producer = new TestProducer(`tasknametest`, `D:\\tmp`)
-
-producer.generateTasks()

package/workers/test/test_splitter.js

@@ -1,7 +0,0 @@
-let splitter = require('../taskSplitter');
-let alltasks = []
-for(let i=0;i<16;i++){
-    alltasks.push(i)
-}
-let subtasks = splitter.split(alltasks, 5);
-console.log(subtasks)