multi-tasks 1.0.7 → 1.1.3

package/README.md

@@ -1,2 +1,39 @@
 # multi-tasks
 multi-tasks
+
+Install:
+
+```javascript
+npm install multi-tasks
+```
+
+How to use:
+
+```javascript
+let master = require('multi-tasks').master;
+let RunnerProducer;//See examples, please provide a Producer class to specify all tasks you have, multi-tasks will automatically split them and execute them in a multi-threaded manner
+let RunnerConsumer;//See examples, please provide a Consumer class to process a certain sub-task and return the processed data
+
+master.start(RunnerProducer, RunnerConsumer, {
+    multi_task_parent_folder: `C:/tmp-demo`,
+    quotaOfEachTask:2, //The number of tasks assigned to each worker
+    numberOfWorks: 6
+});
+```
+
+Have a try:
+
+```shell
+node example0/run
+node example1/run
+```
+
+Changelog:
+
+- 1.1.3 Simplified usage, see example0
+- 1.1.2 
+- 1.1.1 Rename files, updated changelog
+- 1.1.0 Simplified the usage of a customized Consumer, see example0
+- 1.0.8 Fix examples
+- 1.0.7 Remove moment
+- 1.0.6 Performance optimization

package/example0/RunnerConsumer.js

@@ -0,0 +1,18 @@
+let SuperConsumer = require('../index').SuperConsumer;
+
+class TestConsumer extends SuperConsumer {
+  
+    process(taskInfo, actionData){
+        console.log(`processing data:`, taskInfo, actionData);
+        return {
+          task:{            
+            ...taskInfo, 
+            ...actionData,
+          },
+          result: {
+            message: 'this is a result from a demo, random data=' + Math.random()
+          }
+        }
+    }
+  }
+module.exports = TestConsumer;

package/example0/RunnerProducer.js

@@ -0,0 +1,35 @@
+let SuperProducer = require('../index').SuperProducer;
+class TestProducer extends SuperProducer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    getAllActions() {
+        /**
+         * Generate all your tasks in this method, return as an array;
+         * These data will be allocated to multiple threads (workers) for running; 
+         * You can obtain each of the data and process it in the 'process' method of your consumer.
+         * 
+         * **/
+
+        return new Promise((resolve, reject)=>{
+            let trunkInfo = this.trunkInfo;
+            let alltasks = []
+            for(let i=0;i<200;i++){
+                let task_props = {
+                    index: i,
+                    name: `task-${i}`,
+                    description: `This prop is for a subtask`
+                };
+                alltasks.push(task_props);
+            }
+            resolve(alltasks);//return all your tasks as an array
+        })
+    }
+  }
+module.exports = TestProducer;

package/example0/run.js

@@ -0,0 +1,9 @@
+let RunnerProducer = require('./RunnerProducer');//You need to provide a Producer to return all subtasks, multi-tasks will automatically split them and execute them in a multi-threaded manner
+let RunnerConsumer = require('./RunnerConsumer');//You need to provide a Consumer to process a certain sub-task and return the processed data
+let master = require('../index').master;
+
+master.start(RunnerProducer, RunnerConsumer, {
+    multi_task_parent_folder: `C:/tmp-demo`,
+    quotaOfEachTask:2, //The number of tasks assigned to each worker
+    numberOfWorks: 6
+});

package/{example → example1}/RunnerConsumer.js

@@ -2,10 +2,18 @@ let SuperConsumer = require('../index').SuperConsumer;
 
 
 let _do = (mainTaskInfo, actions, callback)=>{
-    if(actions.length === 0) return callback({data:'hahaha'+Math.random()});
-    let act = actions.shift();
+    if(actions.length === 0) return callback({
+        data:'subtask finished, random data='+Math.random()
+    });
+    let actionData = actions.shift();
 
-    console.log('act', act)
+    //start to write your process code here
+
+    console.log('actionData', actionData)
+
+    //end of your process code.
+
+    //trigger next task
     _do(mainTaskInfo, actions, callback);
 };
 

package/example1/RunnerProducer.js

@@ -0,0 +1,35 @@
+let SuperProducer = require('../index').SuperProducer;
+class TestProducer extends SuperProducer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    getAllActions() {
+        /**
+         * Generate all your tasks in this method, return as an array;
+         * These data will be allocated to multiple threads (workers) for running; 
+         * You can obtain each of the data and process it in the 'process' method of your consumer.
+         * 
+         * **/
+
+        return new Promise((resolve, reject)=>{
+            let trunkInfo = this.trunkInfo;
+            let alltasks = []
+            for(let i=0;i<200;i++){
+                let task_props = {
+                    index: i,
+                    name: `task-${i}`,
+                    description: `This prop is for a subtask`
+                };
+                alltasks.push(task_props);
+            }
+            resolve(alltasks);//return all your tasks as an array
+        })
+    }
+  }
+module.exports = TestProducer;

package/{example → example1}/run.js

@@ -1,7 +1,5 @@
-let fs = require('fs');
-let pathutil = require('path');
-let RunnerProducer = require('./RunnerProducer')
-let RunnerConsumer = require('./RunnerConsumer')
+let RunnerProducer = require('./RunnerProducer');//switch to require from node_modules in your project
+let RunnerConsumer = require('./RunnerConsumer');//switch to require from node_modules in your project
 
 let taskId = Math.random()+'';
 let multi_task_parent_folder = `C:/tmp-demo`;

package/example2-dynamicIncreasing/RunnerConsumer.js

@@ -1,5 +1,4 @@
 let fs = require('fs');
-const jsonFormat = require('json-format');
 let pathutil = require('path');
 const { spawn, exec } = require('child_process');
 let SuperConsumer = require('../index').SuperConsumer;
@@ -120,7 +119,7 @@ let _do = (mainTaskInfo, actions, callback)=>{
                 console.log('processing...', process.pid, fpath);
                 processTask(moduleName, requestVersion, (succ, logJson)=>{
                     if(succ){      
-                        fs.writeFileSync(succfPath, jsonFormat(logJson));  
+                        fs.writeFileSync(succfPath, JSON.stringify(logJson));  
                         logJson.response.forEach((item)=>{                            
                             if(item.dependencies){
                                 for(let module2 in item.dependencies){
@@ -132,7 +131,7 @@ let _do = (mainTaskInfo, actions, callback)=>{
                         });
                     }else{
                         console.log('ERR! failed', moduleName, requestVersion);
-                        fs.writeFileSync(pathutil.resolve(dynamicTasksFolderFail, `${fullnameMd5}.json`), jsonFormat({
+                        fs.writeFileSync(pathutil.resolve(dynamicTasksFolderFail, `${fullnameMd5}.json`), JSON.stringify({
                             moduleName, requestVersion,
                             msg: logJson
                         }));

package/example2-dynamicIncreasing/RunnerProducer.js

@@ -1,7 +1,6 @@
 let SuperProducer = require('../index').SuperProducer;
 const fs = require('fs');
 let pathutil = require('path');
-const jsonFormat = require('json-format');
 
 let thisroot = pathutil.parse(__filename).dir;
 

package/example2-dynamicIncreasing/run.js

@@ -1,8 +1,8 @@
 let fs = require('fs');
 let pathutil = require('path');
 let makeDir = require('make-dir');
-let RunnerProducer = require('./RunnerProducer')
-let RunnerConsumer = require('./RunnerConsumer')
+let RunnerProducer = require('./RunnerProducer');//switch to require from node_modules in your project
+let RunnerConsumer = require('./RunnerConsumer');//switch to require from node_modules in your project
 
 let thisroot = pathutil.parse(__filename).dir;
 let taskId = `workers_of_${Math.random()}`;

package/example2-dynamicIncreasing/util.js

@@ -1,7 +1,6 @@
 const md5 = require('md5');
 const fs = require('fs');
 let pathutil = require('path');
-const jsonFormat = require('json-format');
 
 let getFileName = (moduleName, requestVersion)=>{
     let fullname = `${moduleName}@${requestVersion}`;
@@ -10,7 +9,7 @@ let getFileName = (moduleName, requestVersion)=>{
 };
 let createNewTask = (moduleName, requestVersion, dynamicTasksFolderNew)=>{
     let fullnameMd5 = getFileName(moduleName, requestVersion);
-    fs.writeFileSync(pathutil.resolve(dynamicTasksFolderNew, `./${fullnameMd5}.json`), jsonFormat({
+    fs.writeFileSync(pathutil.resolve(dynamicTasksFolderNew, `./${fullnameMd5}.json`), JSON.stringify({
         info:{
             moduleName,
             requestVersion,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "1.0.7",
+  "version": "1.1.3",
   "description": "",
   "main": "index.js",
   "directories": {

package/workers/SuperConsumer.js

@@ -1,18 +1,46 @@
 let fs = require('fs');
 let pathutil = require('path');
+
 class SuperConsumer {
     // constructor
     constructor(mainid, subid) {
         this.mainid = mainid;
         this.subid = subid;
+        this.results = [];
+    }
+
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
     }
-    run (taskInfo, callback){
-        //rewrite this
-        let succ = (Math.random()*1000)%3===0?true:false;
-        let result = {
-            succ
+    _run (mainTaskInfo, subTaskInfo, actions, results, callback){
+        if(actions.length === 0) return callback({
+                                            finishedAt: new Date()*1,
+                                            results,
+                                            subTaskInfo
+                                        });
+        let taskInfo = {
+            mainid: this.mainid,
+            subid: this.subid
+        };
+        let actionData = actions.shift();
+    
+        if(this.process){
+            let result = this.process(taskInfo, actionData);//rewrite this method in subclass
+            if(result){
+                results.push(result);
+            }
         }
-        callback(result);
+    
+        this._run(mainTaskInfo, subTaskInfo, actions, results, callback);
+    };
+    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
+        let results = [];
+        let actions = subTaskInfo.actions;
+        mainTaskInfo.defination = this.trunkInfo.defination;
+        this._run(mainTaskInfo, subTaskInfo, actions, results, (result)=>{
+            callback(result)
+        })
+
     }
     runTasks (mainTaskInfo, subTaskInfo, message){
         // console.log(`mainTaskInfo:`, mainTaskInfo)

package/workers/master.js

@@ -3,7 +3,6 @@ var pathutil = require('path');
 const cluster = require('cluster');
 const numCPUs = require('os').cpus().length;
 
-
 let MSG_REQUEST_TASKID = 'request_taskid';
 /**
  * masterId, the id of scanning flow
@@ -141,7 +140,26 @@ let run = (currentProducer, currentConsumer, opt)=>{
       worker.send(MSG_REQUEST_TASKID);//request new task
 
     }
+};
+let start = (ProducerClass, ConsumerClass, taskConfig)=>{  
+  let taskId = Math.random()+'';
+  let {multi_task_parent_folder} = taskConfig;
+  
+  let producer = new ProducerClass(`workers_of_${taskId}`, multi_task_parent_folder)
+  let consumer = new ConsumerClass();
+
+  console.log(producer.getTaskName())
+  //return;
+
+  //console.log(resp)
+  producer.setTaskTrunkInfo(taskConfig);
+  consumer.setTaskTrunkInfo(taskConfig);
+  run(producer, consumer, {
+      quotaOfEachTask:2,
+      numberOfWorks: 6
+  });
 }
 module.exports = {
-  run
+  run,
+  start
 };

package/example/RunnerProducer.js

@@ -1,29 +0,0 @@
-let SuperProducer = require('../index').SuperProducer;
-class TestProducer extends SuperProducer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    getAllActions() {
-
-        return new Promise((resolve, reject)=>{
-            let trunkInfo = this.trunkInfo;
-            //console.log(this.trunkInfo)
-           //throw repos.length+'ff'+repos2.length
-            //here, IGNORED_repoRegexStringList
-            let alltasks = []
-            for(let i=0;i<200;i++){
-                alltasks.push({
-                    num:i
-                })
-            }
-            resolve(alltasks);
-        })
-    }
-  }
-module.exports = TestProducer;