multi-tasks 2.0.3 → 2.0.5

package/README.md

@@ -30,7 +30,34 @@ let processTask = (task, helper)=>{
     return 'a result which is not a promise'; 
 };
 
-//Step2.1, or return a promise
+//Step3, run!
+multiTasks({
+    initialTasks: alltasks, 
+    processTask, 
+    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
+    taskId: 'my-task',
+    numberOfWorkers: 3, //Assign how many workers are working in parallel
+    //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
+    shouldTerminate:(info)=>{
+        //return true if you need to terminate the whole process
+    },
+    onFinish: (report)=>{
+        console.log('finish callback', report);
+    }
+});
+
+```
+
+Have a try:
+
+```shell
+node examples/example0/run
+node examples/example1/run
+```
+
+More about the task processing function
+```
+//you can return a promise for async processes
 let processTask = (task, helper)=>{
     let {taskCount} = task;
 
@@ -41,7 +68,7 @@ let processTask = (task, helper)=>{
     })
 };
 
-//Step2.2, dynamically create a new task if find something new while processing
+//dynamically create a new task while processing
 let processTask = (task, helper)=>{
     let {taskCount} = task;
 
@@ -60,7 +87,7 @@ let processTask = (task, helper)=>{
     })
 };
 
-//Step2.3, this example shows how to handle exceptions/errors
+//this example shows how to generate/handle exceptions
 let processTask = (task, helper)=>{
     let {taskCount} = task;
 
@@ -88,34 +115,12 @@ let processTask = (task, helper)=>{
         }, 10)
     })
 }
-
-//Step3, run!
-multiTasks.start({
-    tasks: alltasks, 
-    processTask, 
-    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
-    taskId: 'my-task',
-    numberOfWorkers: 3, //Assign how many workers are working in parallel
-    //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
-    shouldTerminate:(info)=>{
-        //return true if you need to terminate the whole process
-    },
-    onFinish: (report)=>{
-        console.log('finish callback', report);
-    }
-});
-
-```
-
-Have a try:
-
-```shell
-node examples/example0/run
-node examples/example1/run
 ```
 
 Changelog:
 
+- 2.0.5 Update readme examples
+- 2.0.4 Support resume from a failed task
 - 2.0.3 Fix: mkdir bug on windows
 - 2.0.2 new feature: support shouldTerminate
 - 2.0.1 Avoid possible I/O conflicts.

package/examples/example0/run.js

@@ -40,8 +40,8 @@ let processTask = (task, helper)=>{
 }
 
 //Step3, run!
-multiTasks.start({
-    tasks: alltasks, 
+multiTasks({
+    initialTasks: alltasks, 
     processTask, 
     taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
     taskId: 'my-task',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "2.0.3",
+  "version": "2.0.5",
   "description": "",
   "main": "index.js",
   "directories": {

package/test/test_resume/step1_start_and_crash.js

@@ -2,7 +2,7 @@ let multiTasks = require('../../index').multiTasks;
 
 //Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
 let alltasks = [];
-for(let i=0;i<50;i++){
+for(let i=0;i<1000;i++){
     let task_props = {
         index: i,
         name: `task-${i}`,
@@ -40,8 +40,8 @@ let processTask = (task, helper)=>{
 }
 
 //Step3, run!
-multiTasks.start({
-    tasks: alltasks, 
+multiTasks({
+    initialTasks: alltasks, 
     processTask, 
     taskRootFolder: `./tmp`, //a directory to store progress and results files, you can check the progress here
     taskId: 'test_resume',

package/test/test_resume/step2_resume.js

@@ -7,10 +7,11 @@ let processTask = (task, helper)=>{
 };
 
 //Step3, run!
-multiTasks.resume({
+multiTasks({
     processTask, 
-    taskFolder: `./tmp/test_resume`, //a directory to store progress and results files, you can check the progress here
-    numberOfWorkers: 3, //Assign how many workers are working in parallel
+    taskRootFolder: `./tmp`, //a directory to store progress and results files, you can check the progress here
+    taskId: 'test_resume',
+    numberOfWorkers: 11, //Assign how many workers are working in parallel
     //autoCloseAfterCompletion: true,
     shouldTerminate:(info)=>{
         //if(info.taskCount > 30) return true;

package/test/test_resume/test.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 node step1_start_and_crash
+sleep 3
 node step2_resume

package/workers/Multithread.js

@@ -5,6 +5,7 @@ const numCPUs = require('os').cpus().length;
 
 let MSG_REQUEST_NEW_TASK = 'request_new_task';
 let MSG_NO_TASK_FOUND = 'no_task_found';
+let MSG_SET_GLOBAL_PARAMS = 'MSG_SET_GLOBAL_PARAMS';
 
 const asMaster = require('./asMaster');
 const TaskMgr = require('./TaskMgr');
@@ -104,7 +105,10 @@ const start = (config)=>{
                                 }
                             });
                         });
-                    }  
+                    }else 
+                    if(message.MSG_SET_GLOBAL_PARAMS){
+                        console.log('xxxxxxxxxxxxxxxxxxxxxxxxxx', message)
+                    }
                 });
             }
         });
@@ -149,7 +153,10 @@ function subWorker(){
                 subTask.index = taskCount;//for v1 backward
                 subTask.taskCount = taskCount;
                 console.log(`[Task Start]: "${subTask.subid}"`, `pid=${process.pid}`, getDateTimeTxt());
-                runTask(config, subTask, message).then((report)=>{
+                let sendMsg = (msg)=>{
+                    worker.send(msg);
+                };
+                runTask(config, subTask, sendMsg).then((report)=>{
                     let endTimestamp = new Date()*1;
                     let cost = startTimestamp - endTimestamp;
     
@@ -166,12 +173,18 @@ function subWorker(){
     worker.send(MSG_REQUEST_NEW_TASK);//request new task
 
 };
-function runTask(config, subTask, message){
+function runTask(config, subTask, sendMsg){
     return new Promise((resolve)=>{ 
         let resultPromise = Promise.resolve({});
         let resultData;
         let errors = [];
         helper.load(config);
+        helper.setGlobalParams = (params)=>{
+            sendMsg({
+                MSG_SET_GLOBAL_PARAMS,
+                params
+            });
+        };
         try{
             resultPromise = USER_CONFIG.processTask(subTask, helper);//rewrite this method in subclass
         }catch(e){

package/workers/TaskMgr.js

@@ -180,7 +180,7 @@ const TaskMgr = {
         });
     },
     mvTask: (subid, from, to, callback)=> {
-        console.log('Move task:', subid, from, to, callback)
+        console.log('Move task:', subid, `${from}>${to}`);
         let {progressRootFolder} = main_config;
         let oldPath = pathutil.resolve(progressRootFolder, `./${from}/${subid}.task.json`);
         let newPath = pathutil.resolve(progressRootFolder, `./${to}/${subid}.task.json`);

package/workers/asMaster.js

@@ -23,14 +23,8 @@ const getDateTimeTxt = ()=>{
 };
 
 const resume = (config)=>{
-    let {taskFolder} = config;
-    if(!taskFolder){
-        console.log('[ERROR]', 'method resume need param "taskFolder"')
-        throw ('[ERROR]', 'method resume need param "taskFolder"');
-    }
-    let pathinfo = pathUtil.parse(taskFolder);
-    let taskId = pathinfo.name;
-    let taskRootFolder = pathinfo.dir;
+    let {taskRootFolder, taskId} = config;
+    let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
 
     //load from config file
     let task_config = JSON.parse(fs.readFileSync(pathUtil.resolve(taskFolder, 'task_config.json')));
@@ -75,9 +69,8 @@ const resume = (config)=>{
 
 const init = (config)=>{
     if(typeof config === 'undefined') config = {}; 
-    if(!config.tasks){ throw 'Please provide tasks data!';  };
+    if(!config.initialTasks){ throw 'Please provide initialTasks data!';  };
     if(!config.processTask){ throw 'Please provide processTask function!'; };
-    if(!config.tasks){ throw 'Must assign initial tasks!'; };
 
     if(typeof config.numberOfWorkers === 'undefined') config.numberOfWorkers = numCPUs - 1;
 
@@ -100,7 +93,7 @@ const init = (config)=>{
     TaskMgr.load(config);
     config = TaskMgr.initFolders(config);
 
-    config.tasks.forEach((task)=>{
+    config.initialTasks.forEach((task)=>{
         TaskMgr.createNewTask(task);
     });
 

package/workers/helper.js

@@ -8,15 +8,18 @@ const load = (config)=>{
     myconfig = config;
 };
 const createNewTasks = (tasks)=>{
-
     if (!Array.isArray(tasks)) tasks = [tasks];
     TaskMgr.load(myconfig);
     tasks.forEach((task)=>{
-        console.log('in a  helper, ck', task)
+        console.log('Creating new task', task)
         TaskMgr.createNewTask(task);
     });
 };
+const setGlobalParams = (params)=>{
+    //will be overridden in Multithread
+};
 module.exports = {
     load,
-    createNewTasks
+    createNewTasks,
+    setGlobalParams
 };

package/workers/index.js

@@ -1,3 +1,4 @@
+const fs = require('fs');
 const pathUtil = require('path');
 const Multithread = require('./Multithread');
 const backwardv1 = require('../utils/backward-v1');
@@ -21,6 +22,8 @@ const resume = (config)=>{
     config = backwardv1.v1tov2(config);
     config.__resume = true;
 
+    config.initialTasks = [];
+
     config.taskRootFolder = _toAbsPath(config.taskRootFolder);
     config.taskFolder = _toAbsPath(config.taskFolder);
 
@@ -36,7 +39,15 @@ const restart = (config)=>{
 };
 
 const multiTasks = (config)=>{
-    start(config);
+    let {taskRootFolder, taskId} = config;
+    let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
+
+    if(fs.existsSync(taskFolder)){
+        resume(config);
+    }else{
+        start(config);
+    }
+
 };
 
 multiTasks.start = start;