multi-tasks 2.0.0 → 2.0.2

package/README.md

@@ -90,13 +90,14 @@ let processTask = (task, helper)=>{
 }
 
 //Step3, run!
-multiTasks({
+multiTasks.start({
     tasks: alltasks, 
     processTask, 
     taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
     taskId: 'my-task',
     numberOfWorkers: 3, //Assign how many workers are working in parallel
     //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
+    shouldTerminate,//return true if you need to terminate the whole process
     onFinish: (report)=>{
         console.log('finish callback', report);
     }
@@ -113,6 +114,8 @@ node examples/example1/run
 
 Changelog:
 
+- 2.0.2 new feature: support shouldTerminate
+- 2.0.1 Avoid possible I/O conflicts.
 - 2.0.0 Rewritten with a new architecture to support dynamic tasks.
 - 1.2.8 Fix: create task folder failed on MacOS
 - 1.2.7 Small updates

package/examples/example0/run-readme.js

@@ -75,7 +75,7 @@ let processTask = (task, helper)=>{
 }
 
 //Step3, run!
-multiTasks({
+multiTasks.start({
     tasks: alltasks, 
     processTask, 
     taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here

package/examples/example0/run.js

@@ -40,7 +40,7 @@ let processTask = (task, helper)=>{
 }
 
 //Step3, run!
-multiTasks({
+multiTasks.start({
     tasks: alltasks, 
     processTask, 
     taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "2.0.0",
+  "version": "2.0.2",
   "description": "",
   "main": "index.js",
   "directories": {

package/utils/asyncQueue.js

@@ -18,14 +18,18 @@ class AsyncQueue {
       }  
       const task = this.queue.shift();  
       let result = task();
-      if(!result instanceof Promise) result = Promise.resolve(result);
-    
+      let isPromise = result instanceof Promise;
+      //console.log('r=q',isPromise)
+      if(!isPromise) {
+        console.error('[ERR]Queue functions must return a promise object');  
+        throw 'Queue functions must return a promise object';
+      }    
       try {  
         await result; 
       } catch (error) {  
         console.error('Error processing task:', error);  
       }  
-      this.processNext(); // 处理完一个任务后，继续处理下一个任务  
+      this.processNext();
     }  
   }  
 

package/workers/Multithread.js

@@ -29,6 +29,21 @@ let USER_CONFIG;
 
 let taskCount = 0;
 
+let killAllWorkers = ()=>{  
+    for (const id in cluster.workers) {  
+        if (cluster.workers.hasOwnProperty(id)) {  
+            const worker = cluster.workers[id];  
+            console.log(`Killing worker ${worker.process.pid}`);  
+            killWorker(worker);
+        }  
+    }  
+};
+let killWorker = (worker)=>{
+    worker.disconnect(); 
+    //worker.process.kill(); 
+    worker.kill();
+};
+
 const start = (config)=>{    
     USER_CONFIG = config;//this will remain user defined functions
     let isMaster = (cluster.isPrimary || cluster.isMaster);
@@ -53,15 +68,17 @@ const start = (config)=>{
                 WorkerMgr.addWorker(worker);
                 worker.on('message', function(message) {
                     if(message === MSG_REQUEST_NEW_TASK){
-                        queue.enqueue(TaskMgr.updateNewTasks);//use queue to make sure no I/O conflict
+                        queue.enqueue(async ()=>{
+                            return TaskMgr.updateNewTasks();
+                        });//use queue to make sure no I/O conflict
                         queue.enqueue(async () => {
-                            TaskMgr.popNewTask().then((subTask)=>{
+                            return TaskMgr.popNewTask().then((subTask)=>{
                                 if(!subTask){
                                     //no tasks left
                                     console.log('[Master]: no tasks left!');
                                     //console.log('workers', cluster.workers);
                                     if(config.autoCloseAfterCompletion){
-                                        worker.kill();
+                                        killWorker(worker);
                                     }else{//waiting for new tasks
                                         worker.send(MSG_NO_TASK_FOUND,()=>{}, {
                                             keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
@@ -69,6 +86,12 @@ const start = (config)=>{
                                     }
                                     //
                                 }else{
+
+                                    let terminateData = {taskCount,subTask,config, id: worker.id};
+                                    if(USER_CONFIG.shouldTerminate && USER_CONFIG.shouldTerminate(terminateData)===true) {
+                                        killAllWorkers();
+                                        return process.exit(0);
+                                    }
                                     //send new tasks to the worker
                                     worker.send({
                                         taskCount,
@@ -99,7 +122,7 @@ const start = (config)=>{
                 setTimeout(()=>{
                     console.log('[Master]: all workers are done, process exit.')
                     process.exit(0);
-                }, 1000);
+                }, 10);
             }
         }); 
     }else{

package/workers/TaskMgr.js

@@ -34,7 +34,7 @@ const TaskMgr = {
     initFolders:()=>{
         let {taskRootFolder, masterTaskId} = main_config;
         taskFolder = pathutil.resolve(taskRootFolder, masterTaskId);
-        if(fs.existsSync(taskFolder)){
+        if(fs.existsSync(taskFolder) && !(main_config.__resume || main_config.__restart)){
             let files = fs.readdirSync(taskFolder);
             if(files.length > 0){
                 console.log('[FATAL] taskRootFolder already exist and not empty, delete it and try again: ', taskFolder);

package/workers/index.js

@@ -1,8 +1,43 @@
+const pathUtil = require('path');
 const Multithread = require('./Multithread');
 const backwardv1 = require('../utils/backward-v1')
 
-const multiTasks = (config)=>{
+const start = (config)=>{
     config = backwardv1.v1tov2(config);
     Multithread.start(config);
 };
+const resume = (config)=>{
+    config = backwardv1.v1tov2(config);
+    let {taskFolder} = config;
+    if(!taskFolder){
+        console.log('[ERROR]', 'method resume need param "taskFolder"')
+        return;
+    }
+    let pathinfo = pathUtil.parse(taskFolder);
+    let taskId = pathinfo.name;
+    let taskRootFolder = pathinfo.dir;
+
+    config.taskId = taskId;
+    config.taskRootFolder = taskRootFolder;
+    config.__resume = true;
+
+    Multithread.start(config);
+};
+const restart = (config)=>{
+    config = backwardv1.v1tov2(config);
+    let {taskFolder} = config;
+    if(!taskFolder){
+        console.log('[ERROR]', 'method resume need param "taskFolder"')
+        return;
+    }
+};
+
+const multiTasks = (config)=>{
+    start(config);
+};
+
+multiTasks.start = start;
+multiTasks.restart = restart;
+multiTasks.resume = resume;
+
 module.exports = multiTasks;

package/examples/example3-dynamicIncreasing/RunnerConsumer.js

@@ -1,176 +0,0 @@
-let fs = require('fs');
-let pathutil = require('path');
-const { spawn, exec } = require('child_process');
-let SuperConsumer = require('../index').SuperConsumer;
-let localUtil = require('./util');
-const makeDir = require('../utils/makedir');
-
-let thisroot = pathutil.parse(__filename).dir;
-let garbagedir = pathutil.resolve(thisroot,`./tmp-garbage`);
-makeDir.sync(garbagedir);
-
-let loadTaskFile = (dir)=>{
-    var results = []
-    var list = fs.readdirSync(dir);
-    list.forEach(function(file) {
-        var stat;
-        try{
-            stat = fs.statSync(pathutil.resolve(dir, file))
-        }catch(e){
-            //Must try-catch, the file maybe deleted by other process
-        }
-        if (stat && !stat.isDirectory()) results.push({
-            file, 
-            fpath: pathutil.resolve(dir, file)
-        });
-    })
-    //console.log(results);
-    return results;
-};
-let processTask = (moduleName, requestVersion, callback)=>{
-    let outputs = [];
-    let ls = spawn(`ls`, [], {cwd: garbagedir, shell: true});
-    ls.stdout.on('data', (data) => {
-        let stdout = data.toString();
-        outputs.push(stdout);
-        //console.log('[stdout]', stdout)
-    });    
-    ls.stderr.on('data', (data) => {
-        let stdout = data.toString();
-        //console.log('[stderr]', stdout)
-        outputs.push(stdout);
-    });    
-    ls.on('close', (code) => {
-        //console.log('[close]', code)
-        // if(code === 1) {
-        //     console.log('[error], checkout failed!!')
-        // }
-        let succ = true;
-        let logTxt = outputs.join('');
-        
-        if(logTxt.indexOf('ERR!')>=0 && logTxt.match(/ERR\!/g)) {
-            succ = false;
-        }
-        let logJson;
-        try{
-            logJson = JSON.parse(logTxt);
-        }catch(e){
-            console.log(e)
-            console.log(`at===`, moduleName, requestVersion)            
-            console.log(`logTxt===`, logTxt)
-            succ = false;
-        }
-        if(succ && logJson){
-            if(!Array.isArray(logJson)){
-                logJson = [logJson];
-            };
-            let data = {
-                response: logJson,
-                __info__: {
-                    moduleName,
-                    requestVersion,
-                }
-            };
-            callback(true, data);
-        }else{
-            callback(false, logTxt);
-        }
-    });
-
-}
-
-let _do = (mainTaskInfo, actions, callback)=>{
-    let{dynamicTasksFolderNew, dynamicTasksFolderSucc, dynamicTasksFolderFail} = mainTaskInfo.defination;
-    let tasks = loadTaskFile(dynamicTasksFolderNew);  
-    if(tasks.length === 0) return callback({data:'hahaha'+Math.random()});//stop!
-
-    tasks = localUtil.shuffleAnArray(tasks);//avoid file IO conflicts
-
-    let task;
-    let data;
-    while(!data){
-        task = tasks.shift();
-        try{
-            data = JSON.parse(fs.readFileSync(task.fpath, 'utf8'));  
-        }catch(e){
-            console.log('xxxx')
-        }
-    }
-
-    if(task && data){
-        let fpath = task.fpath;
-        { 
-            //processing...
-            let {moduleName, requestVersion, fullnameMd5} = data.info;            
-            let oldfPath = fpath;
-            let newfPath = pathutil.resolve(dynamicTasksFolderNew, `${fullnameMd5}.json`);
-            let succfPath = pathutil.resolve(dynamicTasksFolderSucc, `${fullnameMd5}.json`);
-            let errfPath = pathutil.resolve(dynamicTasksFolderFail, `${fullnameMd5}.json`);
-            if(fs.existsSync(succfPath) || fs.existsSync(errfPath)) {
-                console.log('exist...', process.pid, fullnameMd5);
-                if(fs.existsSync(newfPath))
-                try{
-                    fs.unlinkSync(newfPath);
-                }catch(e){
-                    console.log('remove failed', process.pid, newfPath, e);
-                }
-                _do(mainTaskInfo, actions, callback);
-            }else {
-                console.log('processing...', process.pid, fpath);
-                processTask(moduleName, requestVersion, (succ, logJson)=>{
-                    if(succ){      
-                        fs.writeFileSync(succfPath, JSON.stringify(logJson));  
-                        logJson.response.forEach((item)=>{                            
-                            if(item.dependencies){
-                                for(let module2 in item.dependencies){
-                                    let version2 = item.dependencies[module2];
-                                    console.log('found new task', module2, version2);
-                                    localUtil.createNewTask(module2, version2, dynamicTasksFolderNew);//find new one, create a new task file
-                                }
-                            }
-                        });
-                    }else{
-                        console.log('ERR! failed', moduleName, requestVersion);
-                        fs.writeFileSync(pathutil.resolve(dynamicTasksFolderFail, `${fullnameMd5}.json`), JSON.stringify({
-                            moduleName, requestVersion,
-                            msg: logJson
-                        }));
-                    }
-                    _do(mainTaskInfo, actions, callback);
-                });
-            }          
-            if(fs.existsSync(oldfPath)){
-                try{
-                    fs.unlinkSync(oldfPath);
-                }catch(e){
-                    console.log('remove failed', process.pid, oldfPath, e);
-                }
-            }            
-        }
-    }else{        
-        console.log('__done2', process.pid)
-        return callback({data:'hahaha'+Math.random()});//stop!
-    }
-};
-
-
-class TestConsumer extends SuperConsumer {  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    run (mainTaskInfo, subTaskInfo, taskInfo, callback){
-        //console.log(`this.trunkInfo===`, this.trunkInfo)
-        let actions = subTaskInfo.actions;
-        mainTaskInfo.defination = this.trunkInfo;
-        _do(mainTaskInfo, actions, (result)=>{
-            callback(result)
-        })
-
-    }
-  }
-module.exports = TestConsumer;

package/examples/example3-dynamicIncreasing/RunnerProducer.js

@@ -1,49 +0,0 @@
-let SuperProducer = require('../index').SuperProducer;
-const fs = require('fs');
-let pathutil = require('path');
-
-let thisroot = pathutil.parse(__filename).dir;
-
-let localUtil = require('./util')
-
-
-let createInitialTasks = (info)=>{    
-    let succList = JSON.parse(fs.readFileSync(pathutil.resolve(thisroot,'../tmp/version_succList.json'), 'utf8'));
-    //succList = succList.slice(0, 20);
-    succList.forEach((item)=>{
-        let {moduleName, requestVersion} = item;
-        localUtil.createNewTask(moduleName, requestVersion, info.dynamicTasksFolderNew);
-    });
-};
-
-
-
-class TestProducer extends SuperProducer {
-  
-    // constructor
-    constructor(taskname, parentfolder) {
-        super(taskname, parentfolder)
-
-    }
-    setTaskTrunkInfo(trunkInfo){
-        this.trunkInfo = trunkInfo;
-    }
-    getAllActions() {
-        createInitialTasks(this.trunkInfo);
-        return new Promise((resolve, reject)=>{
-            let trunkInfo = this.trunkInfo;
-            let {NUMBER_OF_WORKS} = trunkInfo;
-            //console.log(this.trunkInfo)
-           //throw repos.length+'ff'+repos2.length
-            //here, IGNORED_repoRegexStringList
-            let alltasks = []
-            for(let i=0;i<NUMBER_OF_WORKS;i++){//the task was driven by files, not here
-                alltasks.push({
-                    num:i
-                })
-            }
-            resolve(alltasks);
-        })
-    }
-  }
-module.exports = TestProducer;

package/examples/example3-dynamicIncreasing/run.js

@@ -1,36 +0,0 @@
-let fs = require('fs');
-let pathutil = require('path');
-let makeDir = require('../utils/makedir');
-let RunnerProducer = require('./RunnerProducer');//switch to require from node_modules in your project
-let RunnerConsumer = require('./RunnerConsumer');//switch to require from node_modules in your project
-
-let thisroot = pathutil.parse(__filename).dir;
-let taskId = `workers_of_${Math.random()}`;
-let multi_task_parent_folder = pathutil.resolve(thisroot, `../../example2_dynamicly_increasing`);
-
-let producer = new RunnerProducer(taskId, multi_task_parent_folder)
-let consumer = new RunnerConsumer();
-let master = require('../index').master;
-
-console.log(producer.getTaskName())
-//return;
-
-let NUMBER_OF_WORKS = 5;//LOCK NUMBER OF WORKERS
-
-let taskConfig = {
-    NUMBER_OF_WORKS,
-    dynamicTasksFolderNew: pathutil.resolve(multi_task_parent_folder, `./dynamicTasks-NEW`),
-    dynamicTasksFolderSucc: pathutil.resolve(multi_task_parent_folder, `./dynamicTasks-SUCC`),
-    dynamicTasksFolderFail: pathutil.resolve(multi_task_parent_folder, `./dynamicTasks-FAIL`),
-}
-makeDir.sync(taskConfig.dynamicTasksFolderNew)
-makeDir.sync(taskConfig.dynamicTasksFolderSucc)
-makeDir.sync(taskConfig.dynamicTasksFolderFail)
-
-
-producer.setTaskTrunkInfo(taskConfig);
-consumer.setTaskTrunkInfo(taskConfig)
-master.run(producer, consumer, {
-    quotaOfEachTask:1, //<--Lock as one
-    numberOfWorkers:NUMBER_OF_WORKS
-});

package/examples/example3-dynamicIncreasing/util.js

@@ -1,46 +0,0 @@
-const md5 = require('md5');
-const fs = require('fs');
-let pathutil = require('path');
-
-let getFileName = (moduleName, requestVersion)=>{
-    let fullname = `${moduleName}@${requestVersion}`;
-    let fullnameMd5 = md5(fullname);
-    return fullnameMd5;
-};
-let createNewTask = (moduleName, requestVersion, dynamicTasksFolderNew)=>{
-    let fullnameMd5 = getFileName(moduleName, requestVersion);
-    fs.writeFileSync(pathutil.resolve(dynamicTasksFolderNew, `./${fullnameMd5}.json`), JSON.stringify({
-        info:{
-            moduleName,
-            requestVersion,
-            fullnameMd5,
-        }
-    }));
-};
-function shuffleAnArray(array) {
-    if(array.length === 0 || array.length === 1) return array;
-    let currentIndex = array.length,  randomIndex;
-  
-    // While there remain elements to shuffle.
-    while (currentIndex > 0) {
-  
-      // Pick a remaining element.
-      randomIndex = Math.floor(Math.random() * currentIndex);
-      currentIndex--;
-
-      // And swap it with the current element.
-      let tmp = array[currentIndex];
-      array[currentIndex] = array[randomIndex];
-      array[randomIndex] = tmp;
-  
-    //   [array[currentIndex], array[randomIndex]] = [
-    //     array[randomIndex], array[currentIndex]];
-    }
-  
-    return array;
-  }
-module.exports = {
-    createNewTask,
-    getFileName,
-    shuffleAnArray
-};