multi-tasks 2.1.4 → 2.2.0

package/README.md

@@ -1,19 +1,19 @@
 # multi-tasks
-multi-tasks
 
-Install:
+Multi-tasks is used to handle long-term and large-scale parallel computing tasks. It manages progress and tasks based on the file system, which means that even if the host crashes, tasks can be resumed based on file records.
+
+### Install:
 
 ```javascript
 npm install multi-tasks
 ```
 
-How to use:
+### How to use:
 
 ```javascript
 //see examples/example0
 let multiTasks = require('multi-tasks').multiTasks;
 
-
 //Step1, create your tasks that need to be executed simultaneously as an array.
 let alltasks = [];
 for(let i=0;i<50;i++){
@@ -51,14 +51,7 @@ multiTasks({
 
 ```
 
-Have a try:
-
-```shell
-node examples/example0/run
-node examples/example1/run
-```
-
-More about the task processing function
+### More about the task processing function
 
 ```javascript
 //Example1, return a promise for async processes,
@@ -77,7 +70,6 @@ let processTask = (task, helper)=>{
 //Example2, dynamically create a new task while processing
 let processTask = (task, helper)=>{
     let {taskCount} = task;
-
     
     if(taskCount % 2 === 0){
         //create a new task if needed
@@ -109,8 +101,24 @@ let processTask = (task, helper)=>{
 
 ```
 
-Changelog:
+### Resuming
+
+Sometimes the task execution is interrupted due to some reasons (such as power outage), you can resume the execution like this
+
+```javascript
+
+multiTasks({
+    initialTasks: `/myworks/my_scan_tasks/`, //Point 'initialTasks' to the interrupted task directory, multi-tasks will read the tasks in the 'new' folder and initialize them to 'initialTasks' and then continue execution
+    ...
+    ...//Other configurations remain unchanged
+    ...
+});
+
+```
+
+### Changelog:
 
+- 2.2.0 Support Resuming from an interrupted task
 - 2.1.2 Update readme
 - 2.1.1 Fix: recreate a new one when a worker collapsed unexpectedly.
 - 2.1.0 Add a new output directory "log" where you can view the process time of each subtask
@@ -142,10 +150,10 @@ Changelog:
 - 1.0.7 Remove moment
 - 1.0.6 Performance optimization
 
-Github:
+### Github:
 
 [https://github.com/zhanglei923/multi-tasks](https://github.com/zhanglei923/multi-tasks)
 
-Support / Bug Report:
+### Support / Bug Report:
 
 [zhangleisupport923@163.com](mailto:zhangleisupport923@163.com)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "2.1.4",
+  "version": "2.2.0",
   "description": "",
   "main": "index.js",
   "directories": {

package/workers/Multithread.js

@@ -161,12 +161,16 @@ function subWorker(){
             //console.log('ssss', subTask)
             if(subTask){
                 subTask.index = taskCount;//for v1 backward
-                subTask.taskCount = taskCount;
+                subTask.taskCount = taskCount;                
+                subTask.workerId = worker.id;
+                subTask.workerPid = process.pid;
                 console.log(`[Task Start]: "${subTask.subid}"`, `pid=${process.pid}`, getDateTimeTxt());
                 let sendMsg = (msg)=>{
                     worker.send(msg);
                 };
                 let subTaskLog = {
+                    workerId: worker.id,
+                    workerPid: process.pid,
                     taskCount,
                     startTimestamp
                 };

package/workers/index.js

@@ -5,7 +5,7 @@ const backwardv1 = require('../utils/backward-v1');
 
 const _toAbsPath = (path)=>{
     if(!path) return path;
-    if(pathUtil.isAbsolute(path))return;
+    if(pathUtil.isAbsolute(path))return path;
     let abspath = pathUtil.resolve(process.cwd(), path);
     return abspath;
 };
@@ -39,10 +39,29 @@ const restart = (config)=>{
 };
 
 const multiTasks = (config)=>{
-    let {taskRootFolder, taskId} = config;
-    let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
+    let {initialTasks, taskRootFolder, taskId} = config;
+    let isResume = false;
 
-    if(fs.existsSync(taskFolder)){
+    if(typeof initialTasks === 'string'){//might be an existing tasks folder
+        if(fs.existsSync(initialTasks)){
+            isResume = true;
+            config.taskRootFolder = pathUtil.dirname(initialTasks);
+        }else if(taskRootFolder){
+            let folder = pathUtil.resolve(taskRootFolder, initialTasks);
+            if(fs.existsSync(folder)){
+                isResume = true;
+                config.taskRootFolder = pathUtil.dirname(folder);;
+            }
+        }
+        if(!isResume) throw 'FATAL: initialTasks should be an array or a task root folder!';
+    }else{
+        let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
+        if(fs.existsSync(taskFolder)){
+            isResume = true;
+        }
+    }
+
+    if(isResume){
         resume(config);
     }else{
         start(config);
@@ -54,4 +73,4 @@ multiTasks.start = start;
 multiTasks.restart = restart;
 multiTasks.resume = resume;
 
-module.exports = multiTasks;
+module.exports = multiTasks;