multi-tasks 1.0.7 → 1.1.2

package/README.md

@@ -1,2 +1,22 @@
 # multi-tasks
 multi-tasks
+
+Install:
+
+```javascript
+npm install multi-tasks
+```
+
+Have a try:
+
+```shell
+node example/run
+```
+
+Changelog:
+
+- 1.1.1 Rename files, updated changelog
+- 1.1.0 Simplified the usage of a customized Consumer, see example0
+- 1.0.8 Fix examples
+- 1.0.7 Remove moment
+- 1.0.6 Performance optimization

package/example0/RunnerConsumer.js

@@ -0,0 +1,16 @@
+let SuperConsumer = require('../index').SuperConsumer;
+
+class TestConsumer extends SuperConsumer {
+  
+    process(taskInfo, actionData){
+        console.log(`process data:`, actionData);
+        return {
+          result: {
+            message: 'this is a result from a demo, random=' + Math.random(),
+            ...taskInfo, 
+            ...actionData
+          }
+        }
+    }
+  }
+module.exports = TestConsumer;

package/example0/RunnerProducer.js

@@ -0,0 +1,35 @@
+let SuperProducer = require('../index').SuperProducer;
+class TestProducer extends SuperProducer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    getAllActions() {
+        /**
+         * Generate all your tasks in this method, return as an array;
+         * These data will be allocated to multiple threads (workers) for running; 
+         * You can obtain each of the data and process it in the 'process' method of your consumer.
+         * 
+         * **/
+
+        return new Promise((resolve, reject)=>{
+            let trunkInfo = this.trunkInfo;
+            let alltasks = []
+            for(let i=0;i<200;i++){
+                let task_props = {
+                    index: i,
+                    name: `task-${i}`,
+                    description: `This is a subtask's prop`
+                };
+                alltasks.push(task_props);
+            }
+            resolve(alltasks);//return all your tasks as an array
+        })
+    }
+  }
+module.exports = TestProducer;

package/{example → example0}/run.js

@@ -1,7 +1,5 @@
-let fs = require('fs');
-let pathutil = require('path');
-let RunnerProducer = require('./RunnerProducer')
-let RunnerConsumer = require('./RunnerConsumer')
+let RunnerProducer = require('./RunnerProducer');//switch to require from node_modules in your project
+let RunnerConsumer = require('./RunnerConsumer');//switch to require from node_modules in your project
 
 let taskId = Math.random()+'';
 let multi_task_parent_folder = `C:/tmp-demo`;

package/{example → example1}/RunnerConsumer.js

@@ -3,9 +3,14 @@ let SuperConsumer = require('../index').SuperConsumer;
 
 let _do = (mainTaskInfo, actions, callback)=>{
     if(actions.length === 0) return callback({data:'hahaha'+Math.random()});
-    let act = actions.shift();
+    let action = actions.shift();
+
+    //start to write your process code here
+
+    console.log('action', action)
+
+    //end of your process code.
 
-    console.log('act', act)
     _do(mainTaskInfo, actions, callback);
 };
 

package/example1/RunnerProducer.js

@@ -0,0 +1,35 @@
+let SuperProducer = require('../index').SuperProducer;
+class TestProducer extends SuperProducer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    getAllActions() {
+        /**
+         * Generate all your tasks in this method, return as an array;
+         * These data will be allocated to multiple threads (workers) for running; 
+         * You can obtain each of the data and process it in the 'process' method of your consumer.
+         * 
+         * **/
+
+        return new Promise((resolve, reject)=>{
+            let trunkInfo = this.trunkInfo;
+            let alltasks = []
+            for(let i=0;i<200;i++){
+                let task_props = {
+                    index: i,
+                    name: `task-${i}`,
+                    description: `This is a subtask's prop`
+                };
+                alltasks.push(task_props);
+            }
+            resolve(alltasks);//return all your tasks as an array
+        })
+    }
+  }
+module.exports = TestProducer;

package/example1/run.js

@@ -0,0 +1,23 @@
+let RunnerProducer = require('./RunnerProducer');//switch to require from node_modules in your project
+let RunnerConsumer = require('./RunnerConsumer');//switch to require from node_modules in your project
+
+let taskId = Math.random()+'';
+let multi_task_parent_folder = `C:/tmp-demo`;
+
+let producer = new RunnerProducer(`workers_of_${taskId}`, multi_task_parent_folder)
+let consumer = new RunnerConsumer();
+let master = require('../index').master;
+
+console.log(producer.getTaskName())
+//return;
+
+let taskConfig = {
+    multi_task_parent_folder
+}
+//console.log(resp)
+producer.setTaskTrunkInfo(taskConfig);
+consumer.setTaskTrunkInfo(taskConfig)
+master.run(producer, consumer, {
+    quotaOfEachTask:2,
+    numberOfWorks: 6
+});

package/example2-dynamicIncreasing/RunnerConsumer.js

@@ -1,5 +1,4 @@
 let fs = require('fs');
-const jsonFormat = require('json-format');
 let pathutil = require('path');
 const { spawn, exec } = require('child_process');
 let SuperConsumer = require('../index').SuperConsumer;
@@ -120,7 +119,7 @@ let _do = (mainTaskInfo, actions, callback)=>{
                 console.log('processing...', process.pid, fpath);
                 processTask(moduleName, requestVersion, (succ, logJson)=>{
                     if(succ){      
-                        fs.writeFileSync(succfPath, jsonFormat(logJson));  
+                        fs.writeFileSync(succfPath, JSON.stringify(logJson));  
                         logJson.response.forEach((item)=>{                            
                             if(item.dependencies){
                                 for(let module2 in item.dependencies){
@@ -132,7 +131,7 @@ let _do = (mainTaskInfo, actions, callback)=>{
                         });
                     }else{
                         console.log('ERR! failed', moduleName, requestVersion);
-                        fs.writeFileSync(pathutil.resolve(dynamicTasksFolderFail, `${fullnameMd5}.json`), jsonFormat({
+                        fs.writeFileSync(pathutil.resolve(dynamicTasksFolderFail, `${fullnameMd5}.json`), JSON.stringify({
                             moduleName, requestVersion,
                             msg: logJson
                         }));

package/example2-dynamicIncreasing/RunnerProducer.js

@@ -1,7 +1,6 @@
 let SuperProducer = require('../index').SuperProducer;
 const fs = require('fs');
 let pathutil = require('path');
-const jsonFormat = require('json-format');
 
 let thisroot = pathutil.parse(__filename).dir;
 

package/example2-dynamicIncreasing/run.js

@@ -1,8 +1,8 @@
 let fs = require('fs');
 let pathutil = require('path');
 let makeDir = require('make-dir');
-let RunnerProducer = require('./RunnerProducer')
-let RunnerConsumer = require('./RunnerConsumer')
+let RunnerProducer = require('./RunnerProducer');//switch to require from node_modules in your project
+let RunnerConsumer = require('./RunnerConsumer');//switch to require from node_modules in your project
 
 let thisroot = pathutil.parse(__filename).dir;
 let taskId = `workers_of_${Math.random()}`;

package/example2-dynamicIncreasing/util.js

@@ -1,7 +1,6 @@
 const md5 = require('md5');
 const fs = require('fs');
 let pathutil = require('path');
-const jsonFormat = require('json-format');
 
 let getFileName = (moduleName, requestVersion)=>{
     let fullname = `${moduleName}@${requestVersion}`;
@@ -10,7 +9,7 @@ let getFileName = (moduleName, requestVersion)=>{
 };
 let createNewTask = (moduleName, requestVersion, dynamicTasksFolderNew)=>{
     let fullnameMd5 = getFileName(moduleName, requestVersion);
-    fs.writeFileSync(pathutil.resolve(dynamicTasksFolderNew, `./${fullnameMd5}.json`), jsonFormat({
+    fs.writeFileSync(pathutil.resolve(dynamicTasksFolderNew, `./${fullnameMd5}.json`), JSON.stringify({
         info:{
             moduleName,
             requestVersion,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "1.0.7",
+  "version": "1.1.2",
   "description": "",
   "main": "index.js",
   "directories": {

package/workers/SuperConsumer.js

@@ -1,18 +1,44 @@
 let fs = require('fs');
 let pathutil = require('path');
+
 class SuperConsumer {
     // constructor
     constructor(mainid, subid) {
         this.mainid = mainid;
         this.subid = subid;
+        this.results = [];
+    }
+
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
     }
-    run (taskInfo, callback){
-        //rewrite this
-        let succ = (Math.random()*1000)%3===0?true:false;
-        let result = {
-            succ
+    _run (mainTaskInfo, actions, callback){
+        if(actions.length === 0) return callback({
+                                            finishedAt: new Date()*1,
+                                            results:this.results
+                                        });
+        let taskInfo = {
+            mainid: this.mainid,
+            subid: this.subid
+        };
+        let actionData = actions.shift();
+    
+        if(this.process){
+            let result = this.process(taskInfo, actionData);//rewrite this method in subclass
+            if(result){
+                this.results.push(result);
+            }
         }
-        callback(result);
+    
+        this._run(mainTaskInfo, actions, callback);
+    };
+    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
+        let actions = subTaskInfo.actions;
+        mainTaskInfo.defination = this.trunkInfo.defination;
+        this._run(mainTaskInfo, actions, (result)=>{
+            callback(result)
+        })
+
     }
     runTasks (mainTaskInfo, subTaskInfo, message){
         // console.log(`mainTaskInfo:`, mainTaskInfo)

package/example/RunnerProducer.js

@@ -1,29 +0,0 @@
-let SuperProducer = require('../index').SuperProducer;
-class TestProducer extends SuperProducer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    getAllActions() {
-
-        return new Promise((resolve, reject)=>{
-            let trunkInfo = this.trunkInfo;
-            //console.log(this.trunkInfo)
-           //throw repos.length+'ff'+repos2.length
-            //here, IGNORED_repoRegexStringList
-            let alltasks = []
-            for(let i=0;i<200;i++){
-                alltasks.push({
-                    num:i
-                })
-            }
-            resolve(alltasks);
-        })
-    }
-  }
-module.exports = TestProducer;