multi-tasks 1.2.7 → 2.0.0

package/examples/example2/RunnerConsumer.js

@@ -1,40 +1,40 @@
-let SuperConsumer = require('../../index').SuperConsumer;
-
-
-let _do = (mainTaskInfo, actions, callback)=>{
-    if(actions.length === 0) return callback({
-        data:'subtask finished, random data='+Math.random()
-    });
-    let actionData = actions.shift();
-
-    //start to write your process code here
-
-    console.log('actionData', actionData)
-
-    //end of your process code.
-
-    //trigger next task
-    _do(mainTaskInfo, actions, callback);
-};
-
-
-class TestConsumer extends SuperConsumer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
-        let actions = subTaskInfo.actions;
-        mainTaskInfo.defination = this.trunkInfo.defination;
-        _do(mainTaskInfo, actions, (result)=>{
-            callback(result)
-        })
-
-    }
-  }
+let SuperConsumer = require('../../index').SuperConsumer;
+
+
+let _do = (mainTaskInfo, actions, callback)=>{
+    if(actions.length === 0) return callback({
+        data:'subtask finished, random data='+Math.random()
+    });
+    let actionData = actions.shift();
+
+    //start to write your process code here
+
+    console.log('actionData', actionData)
+
+    //end of your process code.
+
+    //trigger next task
+    _do(mainTaskInfo, actions, callback);
+};
+
+
+class TestConsumer extends SuperConsumer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
+        let actions = subTaskInfo.actions;
+        mainTaskInfo.defination = this.trunkInfo.defination;
+        _do(mainTaskInfo, actions, (result)=>{
+            callback(result)
+        })
+
+    }
+  }
 module.exports = TestConsumer;

package/examples/example2/RunnerProducer.js

@@ -1,35 +1,35 @@
-let SuperProducer = require('../../index').SuperProducer;
-class TestProducer extends SuperProducer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    getAllActions() {
-        /**
-         * Generate all your tasks in this method, return as an array;
-         * These data will be allocated to multiple threads (workers) for running; 
-         * You can obtain each of the data and process it in the 'processTask' method of your consumer.
-         * 
-         * **/
-
-        return new Promise((resolve, reject)=>{
-            let trunkInfo = this.trunkInfo;
-            let alltasks = []
-            for(let i=0;i<200;i++){
-                let task_props = {
-                    index: i,
-                    name: `task-${i}`,
-                    description: `This prop is for a subtask`
-                };
-                alltasks.push(task_props);
-            }
-            resolve(alltasks);//return all your tasks as an array
-        })
-    }
-  }
+let SuperProducer = require('../../index').SuperProducer;
+class TestProducer extends SuperProducer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    getAllActions() {
+        /**
+         * Generate all your tasks in this method, return as an array;
+         * These data will be allocated to multiple threads (workers) for running; 
+         * You can obtain each of the data and process it in the 'processTask' method of your consumer.
+         * 
+         * **/
+
+        return new Promise((resolve, reject)=>{
+            let trunkInfo = this.trunkInfo;
+            let alltasks = []
+            for(let i=0;i<200;i++){
+                let task_props = {
+                    index: i,
+                    name: `task-${i}`,
+                    description: `This prop is for a subtask`
+                };
+                alltasks.push(task_props);
+            }
+            resolve(alltasks);//return all your tasks as an array
+        })
+    }
+  }
 module.exports = TestProducer;

package/examples/example2/run.js

@@ -1,23 +1,23 @@
-let RunnerProducer = require('./RunnerProducer');//switch to require from node_modules in your project
-let RunnerConsumer = require('./RunnerConsumer');//switch to require from node_modules in your project
-
-let taskId = Math.random()+'';
-let multi_task_parent_folder = `C:/tmp-demo`;
-
-let producer = new RunnerProducer(`workers_of_${taskId}`, multi_task_parent_folder)
-let consumer = new RunnerConsumer();
-let master = require('../../index').master;
-
-console.log(producer.getTaskName())
-//return;
-
-let taskConfig = {
-    multi_task_parent_folder
-}
-//console.log(resp)
-producer.setTaskTrunkInfo(taskConfig);
-consumer.setTaskTrunkInfo(taskConfig)
-master.run(producer, consumer, {
-    quotaOfEachTask:2,
-    numberOfWorkers: 6
+let RunnerProducer = require('./RunnerProducer');//switch to require from node_modules in your project
+let RunnerConsumer = require('./RunnerConsumer');//switch to require from node_modules in your project
+
+let taskId = Math.random()+'';
+let multi_task_parent_folder = `C:/tmp-demo`;
+
+let producer = new RunnerProducer(`workers_of_${taskId}`, multi_task_parent_folder)
+let consumer = new RunnerConsumer();
+let master = require('../../index').master;
+
+console.log(producer.getTaskName())
+//return;
+
+let taskConfig = {
+    multi_task_parent_folder
+}
+//console.log(resp)
+producer.setTaskTrunkInfo(taskConfig);
+consumer.setTaskTrunkInfo(taskConfig)
+master.run(producer, consumer, {
+    quotaOfEachTask:2,
+    numberOfWorkers: 6
 });

package/examples/example3-dynamicIncreasing/RunnerConsumer.js

@@ -1,176 +1,176 @@
-let fs = require('fs');
-let pathutil = require('path');
-const { spawn, exec } = require('child_process');
-let SuperConsumer = require('../index').SuperConsumer;
-let localUtil = require('./util');
-const makeDir = require('../utils/makedir');
-
-let thisroot = pathutil.parse(__filename).dir;
-let garbagedir = pathutil.resolve(thisroot,`./tmp-garbage`);
-makeDir.sync(garbagedir);
-
-let loadTaskFile = (dir)=>{
-    var results = []
-    var list = fs.readdirSync(dir);
-    list.forEach(function(file) {
-        var stat;
-        try{
-            stat = fs.statSync(pathutil.resolve(dir, file))
-        }catch(e){
-            //Must try-catch, the file maybe deleted by other process
-        }
-        if (stat && !stat.isDirectory()) results.push({
-            file, 
-            fpath: pathutil.resolve(dir, file)
-        });
-    })
-    //console.log(results);
-    return results;
-};
-let processTask = (moduleName, requestVersion, callback)=>{
-    let outputs = [];
-    let ls = spawn(`ls`, [], {cwd: garbagedir, shell: true});
-    ls.stdout.on('data', (data) => {
-        let stdout = data.toString();
-        outputs.push(stdout);
-        //console.log('[stdout]', stdout)
-    });    
-    ls.stderr.on('data', (data) => {
-        let stdout = data.toString();
-        //console.log('[stderr]', stdout)
-        outputs.push(stdout);
-    });    
-    ls.on('close', (code) => {
-        //console.log('[close]', code)
-        // if(code === 1) {
-        //     console.log('[error], checkout failed!!')
-        // }
-        let succ = true;
-        let logTxt = outputs.join('');
-        
-        if(logTxt.indexOf('ERR!')>=0 && logTxt.match(/ERR\!/g)) {
-            succ = false;
-        }
-        let logJson;
-        try{
-            logJson = JSON.parse(logTxt);
-        }catch(e){
-            console.log(e)
-            console.log(`at===`, moduleName, requestVersion)            
-            console.log(`logTxt===`, logTxt)
-            succ = false;
-        }
-        if(succ && logJson){
-            if(!Array.isArray(logJson)){
-                logJson = [logJson];
-            };
-            let data = {
-                response: logJson,
-                __info__: {
-                    moduleName,
-                    requestVersion,
-                }
-            };
-            callback(true, data);
-        }else{
-            callback(false, logTxt);
-        }
-    });
-
-}
-
-let _do = (mainTaskInfo, actions, callback)=>{
-    let{dynamicTasksFolderNew, dynamicTasksFolderSucc, dynamicTasksFolderFail} = mainTaskInfo.defination;
-    let tasks = loadTaskFile(dynamicTasksFolderNew);  
-    if(tasks.length === 0) return callback({data:'hahaha'+Math.random()});//stop!
-
-    tasks = localUtil.shuffleAnArray(tasks);//avoid file IO conflicts
-
-    let task;
-    let data;
-    while(!data){
-        task = tasks.shift();
-        try{
-            data = JSON.parse(fs.readFileSync(task.fpath, 'utf8'));  
-        }catch(e){
-            console.log('xxxx')
-        }
-    }
-
-    if(task && data){
-        let fpath = task.fpath;
-        { 
-            //processing...
-            let {moduleName, requestVersion, fullnameMd5} = data.info;            
-            let oldfPath = fpath;
-            let newfPath = pathutil.resolve(dynamicTasksFolderNew, `${fullnameMd5}.json`);
-            let succfPath = pathutil.resolve(dynamicTasksFolderSucc, `${fullnameMd5}.json`);
-            let errfPath = pathutil.resolve(dynamicTasksFolderFail, `${fullnameMd5}.json`);
-            if(fs.existsSync(succfPath) || fs.existsSync(errfPath)) {
-                console.log('exist...', process.pid, fullnameMd5);
-                if(fs.existsSync(newfPath))
-                try{
-                    fs.unlinkSync(newfPath);
-                }catch(e){
-                    console.log('remove failed', process.pid, newfPath, e);
-                }
-                _do(mainTaskInfo, actions, callback);
-            }else {
-                console.log('processing...', process.pid, fpath);
-                processTask(moduleName, requestVersion, (succ, logJson)=>{
-                    if(succ){      
-                        fs.writeFileSync(succfPath, JSON.stringify(logJson));  
-                        logJson.response.forEach((item)=>{                            
-                            if(item.dependencies){
-                                for(let module2 in item.dependencies){
-                                    let version2 = item.dependencies[module2];
-                                    console.log('found new task', module2, version2);
-                                    localUtil.createNewTask(module2, version2, dynamicTasksFolderNew);//find new one, create a new task file
-                                }
-                            }
-                        });
-                    }else{
-                        console.log('ERR! failed', moduleName, requestVersion);
-                        fs.writeFileSync(pathutil.resolve(dynamicTasksFolderFail, `${fullnameMd5}.json`), JSON.stringify({
-                            moduleName, requestVersion,
-                            msg: logJson
-                        }));
-                    }
-                    _do(mainTaskInfo, actions, callback);
-                });
-            }          
-            if(fs.existsSync(oldfPath)){
-                try{
-                    fs.unlinkSync(oldfPath);
-                }catch(e){
-                    console.log('remove failed', process.pid, oldfPath, e);
-                }
-            }            
-        }
-    }else{        
-        console.log('__done2', process.pid)
-        return callback({data:'hahaha'+Math.random()});//stop!
-    }
-};
-
-
-class TestConsumer extends SuperConsumer {  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
-        //console.log(`this.trunkInfo===`, this.trunkInfo)
-        let actions = subTaskInfo.actions;
-        mainTaskInfo.defination = this.trunkInfo;
-        _do(mainTaskInfo, actions, (result)=>{
-            callback(result)
-        })
-
-    }
-  }
+let fs = require('fs');
+let pathutil = require('path');
+const { spawn, exec } = require('child_process');
+let SuperConsumer = require('../index').SuperConsumer;
+let localUtil = require('./util');
+const makeDir = require('../utils/makedir');
+
+let thisroot = pathutil.parse(__filename).dir;
+let garbagedir = pathutil.resolve(thisroot,`./tmp-garbage`);
+makeDir.sync(garbagedir);
+
+let loadTaskFile = (dir)=>{
+    var results = []
+    var list = fs.readdirSync(dir);
+    list.forEach(function(file) {
+        var stat;
+        try{
+            stat = fs.statSync(pathutil.resolve(dir, file))
+        }catch(e){
+            //Must try-catch, the file maybe deleted by other process
+        }
+        if (stat && !stat.isDirectory()) results.push({
+            file, 
+            fpath: pathutil.resolve(dir, file)
+        });
+    })
+    //console.log(results);
+    return results;
+};
+let processTask = (moduleName, requestVersion, callback)=>{
+    let outputs = [];
+    let ls = spawn(`ls`, [], {cwd: garbagedir, shell: true});
+    ls.stdout.on('data', (data) => {
+        let stdout = data.toString();
+        outputs.push(stdout);
+        //console.log('[stdout]', stdout)
+    });    
+    ls.stderr.on('data', (data) => {
+        let stdout = data.toString();
+        //console.log('[stderr]', stdout)
+        outputs.push(stdout);
+    });    
+    ls.on('close', (code) => {
+        //console.log('[close]', code)
+        // if(code === 1) {
+        //     console.log('[error], checkout failed!!')
+        // }
+        let succ = true;
+        let logTxt = outputs.join('');
+        
+        if(logTxt.indexOf('ERR!')>=0 && logTxt.match(/ERR\!/g)) {
+            succ = false;
+        }
+        let logJson;
+        try{
+            logJson = JSON.parse(logTxt);
+        }catch(e){
+            console.log(e)
+            console.log(`at===`, moduleName, requestVersion)            
+            console.log(`logTxt===`, logTxt)
+            succ = false;
+        }
+        if(succ && logJson){
+            if(!Array.isArray(logJson)){
+                logJson = [logJson];
+            };
+            let data = {
+                response: logJson,
+                __info__: {
+                    moduleName,
+                    requestVersion,
+                }
+            };
+            callback(true, data);
+        }else{
+            callback(false, logTxt);
+        }
+    });
+
+}
+
+let _do = (mainTaskInfo, actions, callback)=>{
+    let{dynamicTasksFolderNew, dynamicTasksFolderSucc, dynamicTasksFolderFail} = mainTaskInfo.defination;
+    let tasks = loadTaskFile(dynamicTasksFolderNew);  
+    if(tasks.length === 0) return callback({data:'hahaha'+Math.random()});//stop!
+
+    tasks = localUtil.shuffleAnArray(tasks);//avoid file IO conflicts
+
+    let task;
+    let data;
+    while(!data){
+        task = tasks.shift();
+        try{
+            data = JSON.parse(fs.readFileSync(task.fpath, 'utf8'));  
+        }catch(e){
+            console.log('xxxx')
+        }
+    }
+
+    if(task && data){
+        let fpath = task.fpath;
+        { 
+            //processing...
+            let {moduleName, requestVersion, fullnameMd5} = data.info;            
+            let oldfPath = fpath;
+            let newfPath = pathutil.resolve(dynamicTasksFolderNew, `${fullnameMd5}.json`);
+            let succfPath = pathutil.resolve(dynamicTasksFolderSucc, `${fullnameMd5}.json`);
+            let errfPath = pathutil.resolve(dynamicTasksFolderFail, `${fullnameMd5}.json`);
+            if(fs.existsSync(succfPath) || fs.existsSync(errfPath)) {
+                console.log('exist...', process.pid, fullnameMd5);
+                if(fs.existsSync(newfPath))
+                try{
+                    fs.unlinkSync(newfPath);
+                }catch(e){
+                    console.log('remove failed', process.pid, newfPath, e);
+                }
+                _do(mainTaskInfo, actions, callback);
+            }else {
+                console.log('processing...', process.pid, fpath);
+                processTask(moduleName, requestVersion, (succ, logJson)=>{
+                    if(succ){      
+                        fs.writeFileSync(succfPath, JSON.stringify(logJson));  
+                        logJson.response.forEach((item)=>{                            
+                            if(item.dependencies){
+                                for(let module2 in item.dependencies){
+                                    let version2 = item.dependencies[module2];
+                                    console.log('found new task', module2, version2);
+                                    localUtil.createNewTask(module2, version2, dynamicTasksFolderNew);//find new one, create a new task file
+                                }
+                            }
+                        });
+                    }else{
+                        console.log('ERR! failed', moduleName, requestVersion);
+                        fs.writeFileSync(pathutil.resolve(dynamicTasksFolderFail, `${fullnameMd5}.json`), JSON.stringify({
+                            moduleName, requestVersion,
+                            msg: logJson
+                        }));
+                    }
+                    _do(mainTaskInfo, actions, callback);
+                });
+            }          
+            if(fs.existsSync(oldfPath)){
+                try{
+                    fs.unlinkSync(oldfPath);
+                }catch(e){
+                    console.log('remove failed', process.pid, oldfPath, e);
+                }
+            }            
+        }
+    }else{        
+        console.log('__done2', process.pid)
+        return callback({data:'hahaha'+Math.random()});//stop!
+    }
+};
+
+
+class TestConsumer extends SuperConsumer {  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
+        //console.log(`this.trunkInfo===`, this.trunkInfo)
+        let actions = subTaskInfo.actions;
+        mainTaskInfo.defination = this.trunkInfo;
+        _do(mainTaskInfo, actions, (result)=>{
+            callback(result)
+        })
+
+    }
+  }
 module.exports = TestConsumer;

package/examples/example3-dynamicIncreasing/RunnerProducer.js

@@ -1,49 +1,49 @@
-let SuperProducer = require('../index').SuperProducer;
-const fs = require('fs');
-let pathutil = require('path');
-
-let thisroot = pathutil.parse(__filename).dir;
-
-let localUtil = require('./util')
-
-
-let createInitialTasks = (info)=>{    
-    let succList = JSON.parse(fs.readFileSync(pathutil.resolve(thisroot,'../tmp/version_succList.json'), 'utf8'));
-    //succList = succList.slice(0, 20);
-    succList.forEach((item)=>{
-        let {moduleName, requestVersion} = item;
-        localUtil.createNewTask(moduleName, requestVersion, info.dynamicTasksFolderNew);
-    });
-};
-
-
-
-class TestProducer extends SuperProducer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    getAllActions() {
-        createInitialTasks(this.trunkInfo);
-        return new Promise((resolve, reject)=>{
-            let trunkInfo = this.trunkInfo;
-            let {NUMBER_OF_WORKS} = trunkInfo;
-            //console.log(this.trunkInfo)
-           //throw repos.length+'ff'+repos2.length
-            //here, IGNORED_repoRegexStringList
-            let alltasks = []
-            for(let i=0;i<NUMBER_OF_WORKS;i++){//the task was driven by files, not here
-                alltasks.push({
-                    num:i
-                })
-            }
-            resolve(alltasks);
-        })
-    }
-  }
+let SuperProducer = require('../index').SuperProducer;
+const fs = require('fs');
+let pathutil = require('path');
+
+let thisroot = pathutil.parse(__filename).dir;
+
+let localUtil = require('./util')
+
+
+let createInitialTasks = (info)=>{    
+    let succList = JSON.parse(fs.readFileSync(pathutil.resolve(thisroot,'../tmp/version_succList.json'), 'utf8'));
+    //succList = succList.slice(0, 20);
+    succList.forEach((item)=>{
+        let {moduleName, requestVersion} = item;
+        localUtil.createNewTask(moduleName, requestVersion, info.dynamicTasksFolderNew);
+    });
+};
+
+
+
+class TestProducer extends SuperProducer {
+  
+    // constructor
+    constructor(taskname, parentfolder) {
+        super(taskname, parentfolder)
+
+    }
+    setTaskTrunkInfo(trunkInfo){
+        this.trunkInfo = trunkInfo;
+    }
+    getAllActions() {
+        createInitialTasks(this.trunkInfo);
+        return new Promise((resolve, reject)=>{
+            let trunkInfo = this.trunkInfo;
+            let {NUMBER_OF_WORKS} = trunkInfo;
+            //console.log(this.trunkInfo)
+           //throw repos.length+'ff'+repos2.length
+            //here, IGNORED_repoRegexStringList
+            let alltasks = []
+            for(let i=0;i<NUMBER_OF_WORKS;i++){//the task was driven by files, not here
+                alltasks.push({
+                    num:i
+                })
+            }
+            resolve(alltasks);
+        })
+    }
+  }
 module.exports = TestProducer;