multi-tasks 1.0.0

package/example/RunnerConsumer.js

@@ -0,0 +1,32 @@
+let SuperConsumer = require('multi-task').SuperConsumer;
+
+
+let _do = (mainTaskInfo, actions, callback)=>{
+    if(actions.length === 0) return callback({data:'hahaha'+Math.random()});
+    let act = actions.shift();
+
+    console.log('act', act)
+    _do(mainTaskInfo, actions, callback);
+};
+
+
+class TestConsumer extends SuperConsumer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
+        let actions = subTaskInfo.actions;
+        mainTaskInfo.defination = this.trunkInfo.defination;
+        _do(mainTaskInfo, actions, (result)=>{
+            callback(result)
+        })
+
+    }
+  }
+module.exports = TestConsumer;

package/example/RunnerProducer.js

@@ -0,0 +1,29 @@
+let SuperProducer = require('multi-task').SuperProducer;
+class TestProducer extends SuperProducer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    getAllActions() {
+
+        return new Promise((resolve, reject)=>{
+            let trunkInfo = this.trunkInfo;
+            //console.log(this.trunkInfo)
+           //throw repos.length+'ff'+repos2.length
+            //here, IGNORED_repoRegexStringList
+            let alltasks = []
+            for(let i=0;i<100;i++){
+                alltasks.push({
+                    num:i
+                })
+            }
+            resolve(alltasks);
+        })
+    }
+  }
+module.exports = TestProducer;

package/example/run.js

@@ -0,0 +1,26 @@
+let fs = require('fs');
+let _ = require('lodash');
+let pathutil = require('path');
+let RunnerProducer = require('./RunnerProducer')
+let RunnerConsumer = require('./RunnerConsumer')
+
+let taskId = Math.random()+'';
+let multi_task_parent_folder = pathutil.resolve(__dirname, `../../../tasks-search-keywords`);
+
+let producer = new RunnerProducer(`workers_of_${taskId}`, multi_task_parent_folder)
+let consumer = new RunnerConsumer();
+let master = require('multi-task').master;
+
+console.log(producer.getTaskName())
+//return;
+
+let taskConfig = {
+    multi_task_parent_folder
+}
+//console.log(resp)
+producer.setTaskTrunkInfo(taskConfig);
+consumer.setTaskTrunkInfo(taskConfig)
+master.run(producer, consumer, {
+    quotaOfEachTask:2,
+    numberOfWorks: 6
+});

package/index.js

@@ -0,0 +1,8 @@
+const SuperProducer = require('./workers/SuperProducer');
+const SuperConsumer = require('./workers/SuperConsumer');
+const master = require('./workers/master')
+module.exports = {
+    SuperProducer,
+    SuperConsumer,
+    master
+}

package/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "multi-tasks",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "directories": {
+    "example": "example"
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [
+    "multitask",
+    "worker",
+    "multithread"
+  ],
+  "author": "zhanglei923@gmail.com",
+  "license": "MIT"
+}

package/workers/SuperConsumer.js

@@ -0,0 +1,57 @@
+let fs = require('fs');
+let pathutil = require('path');
+class SuperConsumer {
+    // constructor
+    constructor(mainid, subid) {
+        this.mainid = mainid;
+        this.subid = subid;
+    }
+    run (taskInfo, callback){
+        //rewrite this
+        let succ = (Math.random()*1000)%3===0?true:false;
+        let result = {
+            succ
+        }
+        callback(result);
+    }
+    runTasks (mainTaskInfo, subTaskInfo, message){
+        // console.log(`mainTaskInfo:`, mainTaskInfo)
+        // console.log(`subTaskInfo:`, subTaskInfo)
+        // console.log(`message:`, message)
+
+        this.mainTaskInfo = mainTaskInfo;
+        this.subTaskInfo = subTaskInfo;
+
+        return new Promise((resolve, reject)=>{
+            let mainid = mainTaskInfo.masterTaskId;
+            let subid = subTaskInfo.subid;
+            let taskfolderNew = mainTaskInfo.taskFolderNew;
+            let taskFolderResult = mainTaskInfo.taskFolderResult;
+            this.mvTask(mainid, subid,'new','running', (taskFile)=>{//mark as running
+                let taskContent = fs.readFileSync(taskFile, 'utf8');
+                let taskInfo = JSON.parse(taskContent);
+                this.run(mainTaskInfo, subTaskInfo, taskInfo, (taskReport)=>{//invoke
+                    let resultfile = pathutil.resolve(taskFolderResult, `${subid}.json`)
+                    fs.writeFileSync(resultfile, JSON.stringify(taskReport));
+                    this.mvTask(mainid, subid,'running','finished', ()=>{//mark as finished
+                        resolve({
+                            mainid, 
+                            subid,
+                            ...message
+                        })
+                    });
+                });            
+            });
+        });
+    }
+    mvTask (mainid, subid, from, to, callback) {
+        let __thisdir = this.mainTaskInfo.taskFolder;
+        let oldPath = pathutil.resolve(__thisdir, `./${from}/${subid}.task.json`);
+        let newPath = pathutil.resolve(__thisdir, `./${to}/${subid}.task.json`);
+        fs.rename(oldPath, newPath, (err) => {
+            if (err) throw err;
+            callback(newPath);
+        });
+    }
+}
+module.exports = SuperConsumer;

package/workers/SuperProducer.js

@@ -0,0 +1,70 @@
+let fs = require('fs');
+let moment = require('moment');
+let pathutil = require('path');
+let makeDir = require('make-dir');
+let splitter = require('./taskSplitter');
+class SuperProducer {
+    // constructor
+    constructor(taskname, parentfolder) {
+        this.taskname = taskname;
+        this.parentfolder = parentfolder;
+
+    }
+    
+    getTaskName(){
+        return this.taskname;
+    }
+    getAllActions (){
+        //rewrite this method and return all your actions need to run
+        return new Promise();
+    }
+    generateTasks (quota, callback){
+        if(typeof quota === 'undefined' || !quota)quota=10;
+        let dateTimeText = moment().format('YYYY-MM-DD_HH-mm-ss');
+        let masterTaskId = `${this.taskname}_${dateTimeText}`;
+        console.log('masterTaskId=',masterTaskId);
+        this.taskfolder = pathutil.resolve(this.parentfolder, masterTaskId)
+        let taskFolder = this.taskfolder;
+        //let taskfolder = pathutil.parse(__filename).dir;
+        //let taskFolder = pathutil.resolve(taskfolder, `./tasks/${masterTaskId}`);
+        let taskFolderNew = pathutil.resolve(taskFolder, './new');
+        let taskFolderRunning = pathutil.resolve(taskFolder, './running');
+        let taskFolderFinished = pathutil.resolve(taskFolder, './finished');
+        let taskFolderResult = pathutil.resolve(taskFolder, './result');
+        //let mainRptFolder = pathutil.resolve(taskfolder, './report');
+    
+        //makeDir.sync(mainRptFolder);
+        makeDir.sync(taskFolderNew);
+        makeDir.sync(taskFolderRunning);
+        makeDir.sync(taskFolderFinished);
+        makeDir.sync(taskFolderResult);
+
+        this.getAllActions().then((allactions)=>{
+
+            let allSubTasks = splitter.split(allactions, quota);
+
+        
+            console.log('allSubTasks=', allSubTasks.length)
+    
+            allSubTasks.forEach((info, i)=>{
+                let subid = i;
+                info.subid = subid;
+                let taskfilepath = pathutil.resolve(taskFolderNew, `./${info.subid}.task.json`);
+                fs.writeFileSync(taskfilepath, JSON.stringify(info));
+            });
+    
+            callback({
+                masterTaskId,
+                allSubTasks,
+                //mainRptFolder,
+                taskFolder,
+                taskFolderNew,
+                taskFolderRunning,
+                taskFolderFinished,
+                taskFolderResult,
+            });
+        });
+    }
+}
+
+module.exports = SuperProducer;

package/workers/master.js

@@ -0,0 +1,128 @@
+var fs = require('fs');
+var pathutil = require('path');
+const cluster = require('cluster');
+const numCPUs = require('os').cpus().length;
+let moment = require('moment');
+
+
+let MSG_REQUEST_TASKID = 'request_taskid';
+/**
+ * masterId, the id of scanning flow
+ *   -master, a master is a host of child_process computing
+ *   -masterTaskId, id of a master, will rename to masterId 
+ *     -task, a child computing process
+ */
+
+ let getDateTimeTxt = ()=>{
+  return moment().format('YYYY-MM-DD HH:mm:ss');
+};
+let workidlist = new Set();
+let addWorker = (wid)=>{
+  wid = wid+'';
+  workidlist.add(wid);
+};
+let removeWorker = (wid)=>{
+  wid = wid+'';
+  workidlist.delete(wid);
+};
+let hasWorker = ()=>{
+  return workidlist.size !== 0;
+}
+let run = (currentProducer, currentConsumer, opt)=>{
+    if(typeof opt === 'undefined') opt = {
+      quotaOfEachTask: 10
+    };    
+    let masterId = currentProducer.getTaskName();
+    if(typeof opt.numberOfWorks === 'undefined') opt.numberOfWorks = numCPUs;
+    console.log('master1, masterId=', masterId)
+
+
+    if(!masterId) throw `***Must assign masterId, pls use run.sh to run!***`
+
+    console.log('start', getDateTimeTxt())
+
+    if (cluster.isMaster) {//init
+      let numWorks = opt.numberOfWorks;
+      console.log(`numCPUs ${numCPUs}`);
+      console.log(`numWorks ${numWorks}`);
+      console.log(`Master=${process.pid}`);
+
+      currentProducer.generateTasks(opt.quotaOfEachTask, (mainTaskInfo)=>{
+          //get ready
+          taskStack = mainTaskInfo.allSubTasks;
+          let totalTaskNum = taskStack.length; 
+          console.log( 'tasks=', mainTaskInfo.masterTaskId);
+          console.log( 'totalTaskNum=', totalTaskNum);
+
+          for (let i = 0; i < numWorks; i++) {
+            let worker = cluster.fork();
+            addWorker(worker.id)
+            worker.on('message', function(message) {
+              //console.log(`(master).${process.pid} got: '${JSON.stringify(message)}' from w.${worker.process.pid}`);
+              if(message === MSG_REQUEST_TASKID){
+                if(taskStack.length > 0){     
+                  let subTaskInfo = taskStack.shift();
+                  worker.send({
+                    startTimestamp: new Date()*1,
+                    mainTaskInfo,
+                    subTaskInfo,
+                    totalTaskNum
+                  })
+                }else{
+                  removeWorker(worker.id)
+                  worker.kill();
+                  console.log('no task left.', getDateTimeTxt());
+                  if(!hasWorker()){
+                    console.log('all done!')
+                    let taskFolder = mainTaskInfo.taskFolder;
+                    let dataForNextMission = {
+                      ...mainTaskInfo
+                    };
+                    let dataPath = pathutil.resolve(taskFolder, `./extractTaskData-${masterId}.json`);
+                    fs.writeFileSync(dataPath, JSON.stringify(dataForNextMission))
+                    console.log('dataForNextMission=', dataPath)
+                  };
+                }
+              }      
+            });
+          }
+      });
+      
+
+    } else {
+      let worker = cluster.worker;
+      //console.log(`>[w]=${process.pid} started`);
+      process.on('message', function(message) {});
+      worker.on('message', function(message) {
+          if(message.subTaskInfo){
+              let mainTaskInfo = message.mainTaskInfo;
+              let subTaskInfo = message.subTaskInfo;
+              let subid = subTaskInfo.subid;
+              if(mainTaskInfo){
+                    console.log(`[Task Start]: "${subid}"`, `${subid}/${message.totalTaskNum}`, `pid=${process.pid}`, getDateTimeTxt());
+                  //fs.writeFileSync(pathutil.resolve(mainTaskInfo.taskFolderFinished, `./${subid}`), subTaskInfo.tasksLen);
+                  if(0){//mock
+                    setTimeout(()=>{
+                      worker.send(MSG_REQUEST_TASKID);//request new task
+                    }, 100);
+                  }else{//real
+                    currentConsumer.runTasks(mainTaskInfo, subTaskInfo, message).then((report)=>{
+                      let endTimestamp = new Date()*1;
+                      let cost = report.startTimestamp - endTimestamp;
+      
+                        console.log(`[Task Finished] "${report.mainid}"."${report.subid}".`, getDateTimeTxt(), `cost=${cost}ms`)
+                        worker.send(MSG_REQUEST_TASKID);//request new task
+                    });
+                  }
+              }else{
+                  console.log('Can not find: message.subTaskInfo')
+              }
+          }
+      });
+      worker.send(MSG_REQUEST_TASKID);//request new task
+
+    }
+}
+module.exports = {
+  run
+};

package/workers/taskSplitter.js

@@ -0,0 +1,30 @@
+let split = (list, quota)=>{
+    let totalFilesLen = list.length
+    console.log('total:', totalFilesLen,'actions');
+    let allSubTasks = [];
+    let actions = [];
+    let currentIdx = 0;
+    let subid = 0;
+    list.forEach((act)=>{
+        currentIdx++;
+        actions.push(act)
+        if(actions.length>=quota){
+            allSubTasks.push({
+                actions,
+                tasksLen: actions.length
+            });
+            actions = [];
+            subid++;
+        }else if(totalFilesLen === currentIdx){
+            allSubTasks.push({
+                actions,
+                tasksLen: actions.length
+            });
+        }
+        //console.log(totalFilesLen,'-', currentIdx)
+    });
+    return allSubTasks;
+}
+module.exports = {
+    split
+}

package/workers/test/MyConsumer.js

@@ -0,0 +1,25 @@
+let SuperConsumer = require('../SuperConsumer');
+class TestConsumer extends SuperConsumer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
+        //do job
+        // console.log(`mainTaskInfo:`, mainTaskInfo)
+        // console.log(`subTaskInfo:`, subTaskInfo)
+        // console.log(`taskInfo:`, taskInfo)
+        //return;
+        //rewrite this
+        let succ = (Math.random()*1000)%3===0?true:false;
+        let result = {
+            succ
+        }        
+        setTimeout(()=>{
+            callback(result);
+        }, Math.random()*1000)
+    }
+  }
+module.exports = TestConsumer;

package/workers/test/MyProducer.js

@@ -0,0 +1,26 @@
+let SuperProducer = require('../SuperProducer');
+class TestProducer extends SuperProducer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    getAllActions() {
+        return new Promise((resolve, reject)=>{
+            setTimeout(()=>{
+
+
+                let alltasks = []
+                for(let i=0;i<1673;i++){
+                    alltasks.push({
+                        xxx:111,
+                        zzz:222
+                    })
+                }
+                resolve(alltasks);
+            }, 3000)
+        });
+    }
+  }
+module.exports = TestProducer;

package/workers/test/test_master.js

@@ -0,0 +1,12 @@
+let MyProducer = require('./MyProducer')
+let MyConsumer = require('./MyConsumer')
+let producer = new MyProducer(`atasknametest`, `D:\\tmp`)
+let consumer = new MyConsumer()
+let master = require('../master')
+
+
+master.run(producer, consumer, {
+
+    quotaOfEachTask: 3
+
+})

package/workers/test/test_producer.js

@@ -0,0 +1,4 @@
+let TestProducer = require('./MyProducer')
+let producer = new TestProducer(`tasknametest`, `D:\\tmp`)
+
+producer.generateTasks()

package/workers/test/test_splitter.js

@@ -0,0 +1,7 @@
+let splitter = require('../taskSplitter');
+let alltasks = []
+for(let i=0;i<16;i++){
+    alltasks.push(i)
+}
+let subtasks = splitter.split(alltasks, 5);
+console.log(subtasks)