multi-tasks 2.0.8 → 2.1.0

package/README.md

@@ -14,8 +14,7 @@ How to use:
 let multiTasks = require('multi-tasks').multiTasks;
 
 
-//Step1, if you have tasks that need to be executed simultaneously, 
-//       provide them as an array, multi-tasks will automatically split them and execute.
+//Step1, create your tasks that need to be executed simultaneously as an array.
 let alltasks = [];
 for(let i=0;i<50;i++){
     alltasks.push({
@@ -26,9 +25,12 @@ for(let i=0;i<50;i++){
 
 //Step2, provide a function to process a certain sub-task and return the result data
 let processTask = (task, helper)=>{
-    let {taskCount} = task;
+    let {taskCount} = task;//get your task data
+
+    //Run your processing logic here...
 
-    return 'a result which is not a promise'; 
+    //then return the result as a plain javascript object or an async promise
+    return 'result data'; 
 };
 
 //Step3, run!
@@ -78,7 +80,8 @@ let processTask = (task, helper)=>{
 
     
     if(taskCount % 2 === 0){
-        helper.createNewTasks({//dynamically create a new task if find something new while processing
+        //create a new task if needed
+        helper.createNewTasks({
             msg:'a new task'
         });
         return;
@@ -104,22 +107,12 @@ let processTask = (task, helper)=>{
 }
 
 
-//Example4, dynamically create a new task while processing
-let processTask = (task, helper)=>{
-    let {taskCount} = task;
-
-    if(taskCount % 2 === 0){
-        helper.createNewTasks({//create a new task if needed
-            msg:'a new task'
-        });
-    }
-
-    return {data:'succ'};
-}
 ```
 
 Changelog:
 
+- 2.1.0 Add a new output directory "log" where you can view the process time of each subtask
+- 2.0.9 Update readme examples
 - 2.0.8 Update readme examples
 - 2.0.7 Update changelog
 - 2.0.6 Update readme, remove failed examples

package/examples/example0/run-readme.js

@@ -13,7 +13,7 @@ for(let i=0;i<50;i++){
 let processTask = (task, helper)=>{
     let {taskCount} = task;
 
-    return 'a result which is not a promise'; 
+    return 'result data'; 
 };
 
 //Step2.1, or return a promise
@@ -76,10 +76,10 @@ let processTask = (task, helper)=>{
 
 //Step3, run!
 multiTasks.start({
-    tasks: alltasks, 
+    initialTasks: alltasks, 
     processTask, 
     taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
-    taskId: 'my-task',
+    taskId: 'my-readme-task',
     numberOfWorkers: 3, //Assign how many workers are working in parallel
     //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
     onFinish: (report)=>{

package/examples/example0/run.js

@@ -44,7 +44,7 @@ multiTasks({
     initialTasks: alltasks, 
     processTask, 
     taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
-    taskId: 'my-task',
+    taskId: 'my-task' +  Math.random(),
     numberOfWorkers: 3, //Assign how many workers are working in parallel
     //autoCloseAfterCompletion: true,
     onFinish: (report)=>{

package/examples/example0/runm.mjs

@@ -0,0 +1,53 @@
+import {multiTasks} from '../../index.js';
+
+//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
+let alltasks = [];
+for(let i=0;i<50;i++){
+    let task_props = {
+        index: i,
+        name: `task-${i}`,
+        description: `This prop is for a subtask`
+    };
+    alltasks.push(task_props);
+}
+
+//Step2, Provide a process function to handle a certain sub-task and return the result data
+let processTask = (task, helper)=>{
+    //console.log('this is the subtask data you created above:', task);
+    let {taskCount} = task;
+
+    console.log(taskCount)
+
+
+    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
+    if(taskCount===3) throw 'exception';
+    if(taskCount===4) return Promise.reject({err:'a test error'});
+    if(taskCount===5) aaa =  bbb;
+
+    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is also OK
+
+    //by default, you should return a promise, all returned data can be found in the results/succ folder
+    return new Promise((resolve, reject)=>{
+        setTimeout(()=>{
+            if(taskCount % 2 === 0)helper.createNewTasks({//
+                msg:'a new task'
+            });
+            taskCount === 2 ? reject('a rejected error') : resolve({
+                message: 'this is a result from a demo, random data=' + Math.random()
+            });
+        }, 10 * Math.random())
+    })
+}
+
+//Step3, run!
+multiTasks({
+    initialTasks: alltasks, 
+    processTask, 
+    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
+    taskId: 'my-task',
+    numberOfWorkers: 3, //Assign how many workers are working in parallel
+    //autoCloseAfterCompletion: true,
+    onFinish: (report)=>{
+        console.log('finish callback', report);
+    }
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "2.0.8",
+  "version": "2.1.0",
   "description": "",
   "main": "index.js",
   "directories": {

package/test/test_resume/step1_start_and_crash.js

@@ -35,7 +35,7 @@ let processTask = (task, helper)=>{
             taskCount === 2 ? reject('a rejected error') : resolve({
                 message: 'this is a result from a demo, random data=' + Math.random()
             });
-        }, 10 * Math.random())
+        }, 500 * Math.random())
     })
 }
 

package/test/test_resume/test.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+rm -rf ./tmp
 node step1_start_and_crash
 sleep 3
 node step2_resume

package/workers/Multithread.js

@@ -156,9 +156,12 @@ function subWorker(){
                 let sendMsg = (msg)=>{
                     worker.send(msg);
                 };
-                runTask(config, subTask, sendMsg).then((report)=>{
-                    let endTimestamp = new Date()*1;
-                    let cost = startTimestamp - endTimestamp;
+                let subTaskLog = {
+                    taskCount,
+                    startTimestamp
+                };
+                runTask(config, subTask, subTaskLog, sendMsg).then((tasklog)=>{
+                    let {cost, startTimestamp, endTimestamp} = tasklog;
     
                     console.log(`[Task Finished] "${config.masterTaskId}"."${subTask.subid}".`, getDateTimeTxt(), `cost=${cost}ms`)
                 }).then(()=>{
@@ -173,7 +176,7 @@ function subWorker(){
     worker.send(MSG_REQUEST_NEW_TASK);//request new task
 
 };
-function runTask(config, subTask, sendMsg){
+function runTask(config, subTask, subTaskLog, sendMsg){
     return new Promise((resolve)=>{ 
         let resultPromise = Promise.resolve({});
         let resultData;
@@ -185,6 +188,7 @@ function runTask(config, subTask, sendMsg){
                 params
             });
         };
+        TaskMgr.saveTaskLog(subTask.subid, subTaskLog);
         try{
             resultPromise = USER_CONFIG.processTask(subTask, helper);//rewrite this method in subclass
         }catch(e){
@@ -198,11 +202,24 @@ function runTask(config, subTask, sendMsg){
             if(typeof ex !== 'string') ex = ex.toString();
             errors.push({ex});
         }).finally(()=>{
+            let {startTimestamp} = subTaskLog;
+            let endTimestamp = new Date()*1;
+            let cost = endTimestamp - startTimestamp;
+            subTaskLog = {
+                ...subTaskLog,
+                endTimestamp,
+                cost
+            };
             let hasError = errors.length > 0;
-            if(hasError)console.log('hasError=====', hasError, errors)
+            if(hasError)console.log('hasError=====', hasError, errors);            
+            TaskMgr.saveTaskLog(subTask.subid, subTaskLog);
             TaskMgr.saveTaskResult(subTask.subid, hasError, hasError ? errors : resultData);
             TaskMgr.mvTask(subTask.subid, 'running',(hasError ? 'finished_with_errors':'finished'), (newPath)=>{
-                resolve();
+                resolve({
+                    startTimestamp,
+                    endTimestamp,
+                    cost
+                });
             });   
         });
     });

package/workers/TaskMgr.js

@@ -43,19 +43,21 @@ const TaskMgr = {
         };
         //let taskfolder = pathutil.parse(__filename).dir;
         //let taskFolder = pathutil.resolve(taskfolder, `./tasks/${masterTaskId}`);
-        progressRootFolder = pathutil.resolve(taskFolder, './progress');
-        resultsRootFolder = pathutil.resolve(taskFolder, './results');
+        const progressRootFolder = pathutil.resolve(taskFolder, './progress');
+        const resultsRootFolder = pathutil.resolve(taskFolder, './results');
+        const logsRootFolder = pathutil.resolve(taskFolder, './logs');
 
-        taskFolderNew = pathutil.resolve(progressRootFolder, './new');
-        taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
-        taskFolderFinished = pathutil.resolve(progressRootFolder, './finished');
-        taskFolderFailed= pathutil.resolve(progressRootFolder, './finished_with_errors');
+        const taskFolderNew = pathutil.resolve(progressRootFolder, './new');
+        const taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
+        const taskFolderFinished = pathutil.resolve(progressRootFolder, './finished');
+        const taskFolderFailed= pathutil.resolve(progressRootFolder, './finished_with_errors');
 
-        taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
-        taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
+        const taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
+        const taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
         //let mainRptFolder = pathutil.resolve(taskfolder, './report');
     
         //makeDir.sync(mainRptFolder);
+        makeDir.sync(logsRootFolder);
         makeDir.sync(taskFolderNew);
         makeDir.sync(taskFolderRunning);
         makeDir.sync(taskFolderFinished);
@@ -68,6 +70,7 @@ const TaskMgr = {
             taskFolder,
             progressRootFolder,
             resultsRootFolder,
+            logsRootFolder,
 
             taskFolderNew,
             taskFolderRunning,
@@ -208,6 +211,19 @@ const TaskMgr = {
             fcontent = JSON.stringify(data);
         };
         fs.writeFileSync(fpath, fcontent);
+    },
+    saveTaskLog:(subid, log)=>{
+        let {logsRootFolder} = main_config;
+        let folder = logsRootFolder;
+        let fpath = pathutil.resolve(folder, `${subid}.json`);
+        let fcontent;
+        log.subid = subid;
+        if(typeof log === 'undefined') {
+            fcontent = '';
+        }else{
+            fcontent = JSON.stringify(log);
+        };
+        fs.writeFileSync(fpath, fcontent);
     }
 };
 module.exports = TaskMgr;