multi-tasks 2.2.1 → 3.0.2

package/README.md

@@ -2,6 +2,8 @@
 
 Multi-tasks is a toolkit to manage long-term and large-scale parallel computing tasks. It manages progress and tasks based on the file system, which means that even if the host crashes, tasks can be resumed based on file records.
 
+**Zero dependencies:** multi-tasks has no runtime dependencies at all — it is built entirely on Node.js built-in modules (`cluster`, `fs`, `path`, `os`), so installing it adds nothing extra to your `node_modules`.
+
 ### Install:
 
 ```javascript
@@ -40,6 +42,7 @@ multiTasks({
     taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
     taskId: 'my-task',
     numberOfWorkers: 3, //how many workers are working in parallel
+    //taskTimeout: 30000, //optional, in milliseconds, an overdue task is treated as failed with a timeout error
     //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
     shouldTerminate:(info)=>{
         //return true if you need to terminate the whole process
@@ -116,8 +119,14 @@ multiTasks({
 
 ```
 
+### Testing:
+
+The usage patterns documented above are covered by automated tests (unit tests in `test/`, end-to-end tests in `teste2e/`). The e2e suite runs the API in real child processes with a mixed workload — successful tasks (both promise and non-promise results), planned failures (rejected promises and thrown exceptions), dynamically created tasks via `helper.createNewTasks`, and workers that crash randomly — then resumes repeatedly and verifies that every task lands exactly one result in `results/succ` or `results/errors`.
+
 ### Changelog:
 
+- 3.0.2 Support taskTimeout: a task that does not finish in time is treated as failed with a timeout error
+- 3.0.1 Versions 3.0.0 and above are maintained by AI
 - 2.2.0 Support Resuming from an interrupted task
 - 2.1.2 Update readme
 - 2.1.1 Fix: recreate a new one when a worker collapsed unexpectedly.
@@ -150,10 +159,6 @@ multiTasks({
 - 1.0.7 Remove moment
 - 1.0.6 Performance optimization
 
-### Github:
-
-[https://github.com/zhanglei923/multi-tasks](https://github.com/zhanglei923/multi-tasks)
-
-### Support / Bug Report:
+### Gitee:
 
-[zhangleisupport923@163.com](mailto:zhangleisupport923@163.com)
+[https://gitee.com/zhanglei923/multi-tasks](https://gitee.com/zhanglei923/multi-tasks)

package/index.js

@@ -1,4 +1,4 @@
 const multiTasks = require('./workers');
 module.exports = {
     multiTasks
-}
+};

package/package.json

@@ -1,13 +1,26 @@
 {
   "name": "multi-tasks",
-  "version": "2.2.1",
-  "description": "",
+  "version": "3.0.2",
+  "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
   "main": "index.js",
+  "files": [
+    "index.js",
+    "workers",
+    "utils/*.js"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://gitee.com/zhanglei923/multi-tasks.git"
+  },
   "directories": {
-    "example": "example"
+    "example": "examples"
+  },
+  "engines": {
+    "node": ">=14"
   },
   "scripts": {
-    "test": "jest"
+    "test": "jest",
+    "prepublishOnly": "npm test"
   },
   "keywords": [
     "multitask",

package/utils/asyncQueue.js

@@ -1,14 +1,16 @@
 class AsyncQueue {  
     constructor() {  
       this.queue = [];  
-      this.isProcessing = false;  
+      this.isProcessing = false;
+      this.processing = null;
     }  
     enqueue(task) {      
       this.queue.push(task);  
       
       if (!this.isProcessing) {  
-        this.processNext();  
-      } 
+        this.processing = this.processNext();
+      }
+      return this.processing;
     }  
     async processNext() {  
       this.isProcessing = true;  
@@ -16,20 +18,19 @@ class AsyncQueue {
         this.isProcessing = false;  
         return;  
       }  
-      const task = this.queue.shift();  
-      let result = task();
-      let isPromise = result instanceof Promise;
-      //console.log('r=q',isPromise)
-      if(!isPromise) {
-        console.error('[ERR]Queue functions must return a promise object');  
-        throw 'Queue functions must return a promise object';
-      }    
-      try {  
-        await result; 
-      } catch (error) {  
-        console.error('Error processing task:', error);  
-      }  
-      this.processNext();
+      const task = this.queue.shift();
+      try {
+        const result = task();
+        let isPromise = result instanceof Promise || (result && typeof result.then === 'function');
+        if(!isPromise) {
+          console.error('[ERR]Queue functions must return a promise object');
+        } else {
+          await result;
+        }
+      } catch (error) {
+        console.error('Error processing task:', error);
+      }
+      await this.processNext();
     }  
   }  
 

package/utils/backward-v1.js

@@ -1,8 +1,9 @@
 function v1tov2(config) {      
-    if(config.multi_task_parent_folder) config.taskRootFolder = config.multi_task_parent_folder;//backward compatible
+    if(config.multi_task_parent_folder && !config.taskRootFolder) config.taskRootFolder = config.multi_task_parent_folder;//backward compatible
     delete config.multi_task_parent_folder;
     
     if(config.numberOfWorks && !config.numberOfWorkers) config.numberOfWorkers = config.numberOfWorks; //Backward compatible with the wrong variable name "numberOfWorks"
+    delete config.numberOfWorks;
 
 
     return config;

package/utils/makedir.js

@@ -4,11 +4,6 @@ const path = require('path');
 function sync(directoryPath) {
     directoryPath = directoryPath.replace(/\\/g, '/');
 
-    let isAbsPath = false;
-    if(directoryPath.startsWith('/')){
-        isAbsPath = true;
-    }
-
     fs.mkdirSync(directoryPath, { recursive: true });
 }
 function getDirectFiles(dir, callback) {  
@@ -23,8 +18,7 @@ function getDirectFiles(dir, callback) {
             return callback([]);
         }
         files.forEach(file => {  
-            const filePath = path.join(dir, file);  
-            //console.log(filePath)
+            const filePath = path.join(dir, file);
             try{
                 let stats = fs.statSync(filePath);
                 if (stats.isFile()) {  

package/workers/Multithread.js

@@ -1,11 +1,8 @@
-var fs = require('fs');
-var pathutil = require('path');
 const cluster = require('cluster');
 const numCPUs = require('os').cpus().length;
 
 let MSG_REQUEST_NEW_TASK = 'request_new_task';
 let MSG_NO_TASK_FOUND = 'no_task_found';
-let MSG_SET_GLOBAL_PARAMS = 'MSG_SET_GLOBAL_PARAMS';
 
 const asMaster = require('./asMaster');
 const TaskMgr = require('./TaskMgr');
@@ -23,7 +20,6 @@ let getDateTimeTxt = ()=>{
     let HH=fmtdigit(d.getHours());
     let mm=fmtdigit(d.getMinutes());
     let ss=fmtdigit(d.getSeconds());
-    //return moment().format('YYYY-MM-DD HH:mm:ss');
     return `${YYYY}-${MM}-${DD} ${HH}:${mm}:${ss}`;
 };
 
@@ -31,6 +27,8 @@ let USER_CONFIG;
 
 let taskCount = 0;
 
+let workerTaskTimers = {};//worker.id -> {timer, subTask, startTimestamp}, master 端任务超时倒计时
+
 let killAllWorkers = ()=>{  
     for (const id in cluster.workers) {  
         if (cluster.workers.hasOwnProperty(id)) {  
@@ -41,8 +39,7 @@ let killAllWorkers = ()=>{
     }  
 };
 let killWorker = (worker)=>{
-    worker.disconnect(); 
-    //worker.process.kill(); 
+    worker.disconnect();
     worker.kill();
 };
 
@@ -61,10 +58,45 @@ const start = (config)=>{
         const AsyncQueue = require(`../utils/asyncQueue`);
         const queue = new AsyncQueue();  
 
+        let clearTaskTimer = (workerId)=>{
+            let rec = workerTaskTimers[workerId];
+            if(rec){
+                clearTimeout(rec.timer);
+                delete workerTaskTimers[workerId];
+            }
+        };
+        let finalizeTimeoutTask = (subTask, cost)=>{
+            let subid = subTask.subid;
+            try{
+                let resultType = TaskMgr.getSavedResultType(subid);
+                if(resultType === 'succ'){//超时边界: worker 已成功写盘, 视为已完成
+                    TaskMgr.mvTask(subid, 'running', 'finished', ()=>{});
+                }else if(resultType === 'error'){//超时边界: worker 已自行记错误
+                    TaskMgr.mvTask(subid, 'running', 'finished_with_errors', ()=>{});
+                }else{
+                    TaskMgr.saveTaskResult(subid, true, [{type: '超时', timeout: config.taskTimeout, cost}]);
+                    TaskMgr.mvTask(subid, 'running', 'finished_with_errors', ()=>{});
+                }
+            }catch(e){
+                console.warn('[Master]: failed to finalize timeout task', subid, e);
+            }
+        };
+        let onTaskTimeout = (worker)=>{
+            let rec = workerTaskTimers[worker.id];
+            if(!rec) return;
+            delete workerTaskTimers[worker.id];
+            let {subTask, startTimestamp} = rec;
+            let cost = new Date()*1 - startTimestamp;
+            console.warn(`[Master]: task "${subTask.subid}" timeout after ${cost}ms, killing worker pid=${worker.process.pid}`);
+            worker.kill();//不 disconnect: exit 事件会按崩溃补员
+            finalizeTimeoutTask(subTask, cost);
+        };
+
         let initWorker = (worker)=>{
             WorkerMgr.addWorker(worker);
             worker.on('message', function(message) {
                 if(message === MSG_REQUEST_NEW_TASK){
+                    clearTaskTimer(worker.id);//上一个任务已结束(无论成败), 撤销超时倒计时
                     queue.enqueue(async ()=>{
                         return TaskMgr.updateNewTasks();
                     });//use queue to make sure no I/O conflict
@@ -73,7 +105,6 @@ const start = (config)=>{
                             if(!subTask){
                                 //no tasks left
                                 console.log('[Master]: no tasks left!');
-                                //console.log('workers', cluster.workers);
                                 if(config.autoCloseAfterCompletion){
                                     killWorker(worker);
                                 }else{//waiting for new tasks
@@ -88,21 +119,26 @@ const start = (config)=>{
                                     return process.exit(0);
                                 }
                                 //send new tasks to the worker
+                                let startTimestamp = new Date()*1;
                                 worker.send({
                                     taskCount,
-                                    startTimestamp: new Date()*1,
+                                    startTimestamp,
                                     config,
                                     subTask
                                 },()=>{}, {
                                     keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
                                 });
+                                if(typeof config.taskTimeout === 'number' && config.taskTimeout > 0){
+                                    workerTaskTimers[worker.id] = {
+                                        timer: setTimeout(()=>{ onTaskTimeout(worker); }, config.taskTimeout),
+                                        subTask,
+                                        startTimestamp
+                                    };
+                                }
                                 taskCount++;
                             }
                         });
                     });
-                }else 
-                if(message.MSG_SET_GLOBAL_PARAMS){
-                    console.log('xxxxxxxxxxxxxxxxxxxxxxxxxx', message)
                 }
             });
         };
@@ -117,6 +153,7 @@ const start = (config)=>{
             }
         });
         cluster.on('exit', (worker, code, signal) => {             
+            clearTaskTimer(worker.id);//worker 退出(含崩溃)时清掉其超时计时器
             if (code !== 0 && !worker.exitedAfterDisconnect) {
                 console.log('Worker died unexpectedly starting a new one...');
                 let worker = cluster.fork();
@@ -144,10 +181,6 @@ const start = (config)=>{
 };
 function subWorker(){
     let worker = cluster.worker;
-    //console.log(`>[w]=${process.pid} started`);
-    process.on('message', function(message) {
-        //
-    });
     worker.on('message', function(message) {
         if(message === MSG_NO_TASK_FOUND){//wait for new task
             console.log(`  (Worker #${worker.id}): waiting`);
@@ -158,63 +191,56 @@ function subWorker(){
         if(message.subTask){//new task
             let {config, subTask, startTimestamp, taskCount} = message;
             TaskMgr.load(config);
-            //console.log('ssss', subTask)
-            if(subTask){
-                subTask.index = taskCount;//for v1 backward
-                subTask.taskCount = taskCount;                
-                subTask.workerId = worker.id;
-                subTask.workerPid = process.pid;
-                console.log(`[Task Start]: "${subTask.subid}"`, `pid=${process.pid}`, getDateTimeTxt());
-                let sendMsg = (msg)=>{
-                    worker.send(msg);
-                };
-                let subTaskLog = {
-                    workerId: worker.id,
-                    workerPid: process.pid,
-                    taskCount,
-                    startTimestamp
-                };
-                runTask(config, subTask, subTaskLog, sendMsg).then((tasklog)=>{
-                    let {cost, startTimestamp, endTimestamp} = tasklog;
-    
-                    console.log(`[Task Finished] "${config.masterTaskId}"."${subTask.subid}".`, getDateTimeTxt(), `cost=${cost}ms`)
-                }).then(()=>{
-                    worker.send(MSG_REQUEST_NEW_TASK);//request new task
-                });
-            }else{
-                console.log('Can not find: message.subTask');
-                throw 'can not find subTask!'
-            }
+            subTask.index = taskCount;//for v1 backward
+            subTask.taskCount = taskCount;
+            subTask.workerId = worker.id;
+            subTask.workerPid = process.pid;
+            console.log(`[Task Start]: "${subTask.subid}"`, `pid=${process.pid}`, getDateTimeTxt());
+            let subTaskLog = {
+                workerId: worker.id,
+                workerPid: process.pid,
+                taskCount,
+                startTimestamp
+            };
+            runTask(config, subTask, subTaskLog).then((tasklog)=>{
+                let {cost, startTimestamp, endTimestamp} = tasklog;
+
+                console.log(`[Task Finished] "${config.masterTaskId}"."${subTask.subid}".`, getDateTimeTxt(), `cost=${cost}ms`)
+            }).then(()=>{
+                worker.send(MSG_REQUEST_NEW_TASK);//request new task
+            });
+        }else{
+            console.log('Can not find: message.subTask');
+            throw new Error('can not find subTask!')
         }
     });
     worker.send(MSG_REQUEST_NEW_TASK);//request new task
 
 };
-function runTask(config, subTask, subTaskLog, sendMsg){
+function runTask(config, subTask, subTaskLog){
     return new Promise((resolve)=>{ 
         let resultPromise = Promise.resolve({});
         let resultData;
         let errors = [];
-        helper.load(config);
-        helper.setGlobalParams = (params)=>{
-            sendMsg({
-                MSG_SET_GLOBAL_PARAMS,
-                params
-            });
+        let serializeError = (e)=>{
+            if(e instanceof Error){
+                return {message: e.message, stack: e.stack};
+            }
+            return e;
         };
+        helper.load(config);
         TaskMgr.saveTaskLog(subTask.subid, subTaskLog);
         try{
             resultPromise = USER_CONFIG.processTask(subTask, helper);//rewrite this method in subclass
         }catch(e){
-            errors.push({subTask, exception: 'Exception:' + e.toString()});
+            errors.push({subTask, exception: serializeError(e)});
         }
         let ispromise = resultPromise instanceof Promise;
         if(!ispromise){resultPromise = Promise.resolve(resultPromise);}//convert data to promise
         resultPromise.then((data)=>{
             resultData = data;
         }).catch((ex)=>{
-            if(typeof ex !== 'string') ex = ex.toString();
-            errors.push({ex});
+            errors.push({ex: serializeError(ex)});
         }).finally(()=>{
             let {startTimestamp} = subTaskLog;
             let endTimestamp = new Date()*1;
@@ -225,7 +251,7 @@ function runTask(config, subTask, subTaskLog, sendMsg){
                 cost
             };
             let hasError = errors.length > 0;
-            if(hasError)console.log('hasError=====', hasError, errors);            
+            if(hasError)console.warn('[Task Error]', subTask.subid, errors);
             TaskMgr.saveTaskLog(subTask.subid, subTaskLog);
             TaskMgr.saveTaskResult(subTask.subid, hasError, hasError ? errors : resultData);
             TaskMgr.mvTask(subTask.subid, 'running',(hasError ? 'finished_with_errors':'finished'), (newPath)=>{

package/workers/TaskMgr.js

@@ -3,20 +3,6 @@ const pathutil = require('path');
 const makeDir = require('../utils/makedir');
 const randomsUtil = require('../utils/randoms');
 
-let taskCounter = 0;
-
-let taskFolder;
-let progressRootFolder;
-let resultsRootFolder;
-
-let taskFolderNew;
-let taskFolderRunning;
-let taskFolderFinished;;
-let taskFolderFailed;
-
-let taskFolderResult;
-let taskFolderResultFailed;
-
 let allSubTasks = {
     new:[],
     running:[],
@@ -33,7 +19,7 @@ const TaskMgr = {
     },
     initFolders:()=>{
         let {taskRootFolder, masterTaskId} = main_config;
-        taskFolder = pathutil.resolve(taskRootFolder, masterTaskId);
+        const taskFolder = pathutil.resolve(taskRootFolder, masterTaskId);
         if(fs.existsSync(taskFolder) && !(main_config.__resume || main_config.__restart)){
             let files = fs.readdirSync(taskFolder);
             if(files.length > 0){
@@ -41,8 +27,6 @@ const TaskMgr = {
                 return process.exit(0);
             }
         };
-        //let taskfolder = pathutil.parse(__filename).dir;
-        //let taskFolder = pathutil.resolve(taskfolder, `./tasks/${masterTaskId}`);
         const progressRootFolder = pathutil.resolve(taskFolder, './progress');
         const resultsRootFolder = pathutil.resolve(taskFolder, './results');
         const logsRootFolder = pathutil.resolve(taskFolder, './logs');
@@ -54,9 +38,6 @@ const TaskMgr = {
 
         const taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
         const taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
-        //let mainRptFolder = pathutil.resolve(taskfolder, './report');
-    
-        //makeDir.sync(mainRptFolder);
         makeDir.sync(logsRootFolder);
         makeDir.sync(taskFolderNew);
         makeDir.sync(taskFolderRunning);
@@ -111,8 +92,12 @@ const TaskMgr = {
                                 console.log('[WARN]', `file not found`, fpath)
                             }
                             if(fcontent){
-                                let task = JSON.parse(fcontent);
-                                allSubTasks[type].push(task);
+                                try{
+                                    let task = JSON.parse(fcontent);
+                                    allSubTasks[type].push(task);
+                                }catch(e){
+                                    console.warn('[WARN]', 'failed to parse task file, skipped', fpath);
+                                }
                             }
                         });
                         resolve(allSubTasks[type]);
@@ -145,8 +130,13 @@ const TaskMgr = {
         return new Promise((resolve)=>{
             let newtask = allSubTasks.new.shift();
             if(newtask){
-                TaskMgr.mvTask(newtask.subid, 'new', 'running', (newPath)=>{ 
-                    resolve(newtask);
+                TaskMgr.mvTask(newtask.subid, 'new', 'running', (newPath, succ)=>{ 
+                    if(succ){
+                        resolve(newtask);
+                    }else{
+                        allSubTasks.new.unshift(newtask);
+                        resolve(null);
+                    }
                 });
             }else{
                 resolve(null);
@@ -156,6 +146,9 @@ const TaskMgr = {
     getTask:(type, id)=>{
         let result;
         let arr = allSubTasks[type];
+        if(!arr){
+            return result;
+        }
         for(let i=0;i<arr.length;i++){
             let task = arr[i];
             if(task.subid === id){
@@ -212,15 +205,23 @@ const TaskMgr = {
         };
         fs.writeFileSync(fpath, fcontent);
     },
+    getSavedResultType:(subid)=>{
+        let {taskFolderResult, taskFolderResultFailed} = main_config;
+        let succPath = pathutil.resolve(taskFolderResult, `${subid}.json`);
+        let errPath = pathutil.resolve(taskFolderResultFailed, `${subid}.json`);
+        if(fs.existsSync(succPath)) return 'succ';
+        if(fs.existsSync(errPath)) return 'error';
+        return null;
+    },
     saveTaskLog:(subid, log)=>{
         let {logsRootFolder} = main_config;
         let folder = logsRootFolder;
         let fpath = pathutil.resolve(folder, `${subid}.json`);
         let fcontent;
-        log.subid = subid;
         if(typeof log === 'undefined') {
             fcontent = '';
         }else{
+            log.subid = subid;
             fcontent = JSON.stringify(log);
         };
         fs.writeFileSync(fpath, fcontent);

package/workers/WorkerMgr.js

@@ -1,10 +1,3 @@
-const fs = require('fs');
-const pathutil = require('path');
-const makeDir = require('../utils/makedir');
-
-const BUSY = 1;
-const IDLE = 0;
-
 let workers = {};
 let status = {};
 
@@ -12,7 +5,7 @@ const mgr = {
     addWorker:(worker)=>{
         let id = `${worker.id}`;
         if(workers[id] || status[id]){
-            throw 'FATAL: duplicated worker!';
+            throw new Error('FATAL: duplicated worker!');
         }
         status[id] = {isBusy: true};
         console.log('add worker', worker.id);
@@ -22,10 +15,11 @@ const mgr = {
         return workers[`${id}`];
     },
     setBusy:(id, isBusy)=>{
-
+        status[`${id}`] = {isBusy: isBusy};
     },
     isBusy:(id)=>{
-
+        let s = status[`${id}`];
+        return s ? s.isBusy : false;
     }
 };
 module.exports = mgr;

package/workers/asMaster.js

@@ -2,27 +2,11 @@ var fs = require('fs');
 var pathUtil = require('path');
 const numCPUs = require('os').cpus().length;
 
-let MSG_REQUEST_TASKID = 'request_taskid';
-
 const TaskMgr = require('./TaskMgr');
-const Multithread = require('./Multithread');
-
-
-const fmtDigit = (n)=>{
-    return n > 9 ? "" + n: "0" + n;
-}
-const getDateTimeTxt = ()=>{
-    let d=new Date();
-    let YYYY=d.getFullYear()
-    let MM=fmtDigit(d.getMonth()+1);
-    let DD=fmtDigit(d.getDate());
-    let HH=fmtDigit(d.getHours());
-    let mm=fmtDigit(d.getMinutes());
-    let ss=fmtDigit(d.getSeconds());
-    return `${YYYY}-${MM}-${DD}_${HH}-${mm}-${ss}`;
-};
+const randomsUtil = require('../utils/randoms');
 
 const resume = (config)=>{
+    config.__resume = true;
     let {taskRootFolder, taskId} = config;
     let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
 
@@ -33,33 +17,35 @@ const resume = (config)=>{
         ...task_config
     }
 
-    // config.taskId = taskId;
-    // config.taskRootFolder = taskRootFolder;
-    // config.masterTaskId = taskId;
-
     TaskMgr.load(config);
     config = TaskMgr.initFolders(config);
 
-    //console.log(config)
-
-    return new Promise((resolve)=>{
+    return new Promise((resolve, reject)=>{
 
         TaskMgr.getTaskFiles('running', (taskfiles)=>{
-            let promises = [];
-            taskfiles.forEach((fname)=>{
-                let subid = fname.replace(/\.task\.json$/, '');
-                promises.push(TaskMgr.mvTaskPromise(subid, 'running', 'new'));
-            });
-            Promise.all(promises).then(()=>{
-                TaskMgr.loadAllTasks().then((types)=>{
-                    console.log('******resume multi-thread')
-                    setTimeout(()=>{
-                        resolve(config);
-                    }, 1000);
+            TaskMgr.getTaskFiles('finished_with_errors', (errfiles)=>{
+                let promises = [];
+                taskfiles.forEach((fname)=>{
+                    let subid = fname.replace(/\.task\.json$/, '');
+                    promises.push(TaskMgr.mvTaskPromise(subid, 'running', 'new'));
+                });
+                if(config.__restart){
+                    errfiles.forEach((fname)=>{
+                        let subid = fname.replace(/\.task\.json$/, '');
+                        promises.push(TaskMgr.mvTaskPromise(subid, 'finished_with_errors', 'new'));
+                    });
+                };
+                Promise.all(promises).then(()=>{
+                    TaskMgr.loadAllTasks().then((types)=>{
+                        console.log('******resume multi-thread')
+                        setTimeout(()=>{
+                            resolve(config);
+                        }, 1000);
+                    });
+
+                }).catch((e)=>{
+                    reject(e);
                 });
-
-            }).catch((e)=>{
-                console.log('failed', e)
             });
         });
     });
@@ -68,19 +54,19 @@ const resume = (config)=>{
 };
 
 const init = (config)=>{
-    if(typeof config === 'undefined') config = {}; 
-    if(!config.initialTasks){ throw 'Please provide initialTasks data!';  };
-    if(!config.processTask){ throw 'Please provide processTask function!'; };
+    if(!config) config = {}; 
+    if(!config.initialTasks){ throw new Error('Please provide initialTasks data!');  };
+    if(!config.processTask){ throw new Error('Please provide processTask function!'); };
 
     if(typeof config.numberOfWorkers === 'undefined') config.numberOfWorkers = numCPUs - 1;
 
     let {taskRootFolder, taskName} = config;
 
-    if(taskName && !taskName.match(/[a-zA-Z0-9_-]/)){ throw 'Error: taskName can only contain letters and numbers'; }
+    if(taskName && !taskName.match(/^[a-zA-Z0-9_-]+$/)){ throw new Error('Error: taskName can only contain letters and numbers'); }
     if(!taskName){ taskName = 'multitasks'; }
 
     let {round, pow, random} = Math;
-    let dateTimeText = getDateTimeTxt();//moment().format('YYYY-MM-DD_HH-mm-ss');
+    let dateTimeText = randomsUtil.getDateTimeTxt();
     let randomNum = round((random() * pow(10,8)),8).toString(36);
     let masterTaskId = config.taskId ? config.taskId : `${taskName}${dateTimeText}--${randomNum}`;
 
@@ -97,11 +83,6 @@ const init = (config)=>{
         TaskMgr.createNewTask(task);
     });
 
-    // let tasks = TaskMgr.getTasks('new')
-    // let task =  tasks.shift();
-    // console.log(tasks)
-    // TaskMgr.mvTask(task.subid, 'new', 'finished',()=>{})
-
     //to reload is for to reStart a collapsed task
     return new Promise((resolve)=>{
         TaskMgr.loadNewTasks().then(()=>{

package/workers/helper.js

@@ -8,6 +8,7 @@ const load = (config)=>{
     myconfig = config;
 };
 const createNewTasks = (tasks)=>{
+    if (!myconfig) throw new Error('Please call helper.load(config) first!');
     if (!Array.isArray(tasks)) tasks = [tasks];
     TaskMgr.load(myconfig);
     tasks.forEach((task)=>{
@@ -15,11 +16,7 @@ const createNewTasks = (tasks)=>{
         TaskMgr.createNewTask(task);
     });
 };
-const setGlobalParams = (params)=>{
-    //will be overridden in Multithread
-};
 module.exports = {
     load,
-    createNewTasks,
-    setGlobalParams
-};
+    createNewTasks
+};

package/workers/index.js

@@ -33,12 +33,19 @@ const restart = (config)=>{
     config = backwardv1.v1tov2(config);
     let {taskFolder} = config;
     if(!taskFolder){
-        console.log('[ERROR]', 'method resume need param "taskFolder"')
+        console.log('[ERROR]', 'method restart need param "taskFolder"');
         return;
     }
+    config.taskRootFolder = pathUtil.dirname(taskFolder);
+    config.taskId = pathUtil.basename(taskFolder);
+    config.__resume = true;
+    config.__restart = true;
+    config.initialTasks = [];
+    Multithread.start(config);
 };
 
 const multiTasks = (config)=>{
+    config = backwardv1.v1tov2(config);
     let {initialTasks, taskRootFolder, taskId} = config;
     let isResume = false;
 
@@ -50,10 +57,10 @@ const multiTasks = (config)=>{
             let folder = pathUtil.resolve(taskRootFolder, initialTasks);
             if(fs.existsSync(folder)){
                 isResume = true;
-                config.taskRootFolder = pathUtil.dirname(folder);;
+                config.taskRootFolder = pathUtil.dirname(folder);
             }
         }
-        if(!isResume) throw 'FATAL: initialTasks should be an array or a task root folder!';
+        if(!isResume) throw new Error('FATAL: initialTasks should be an array or a task root folder!');
     }else{
         let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
         if(fs.existsSync(taskFolder)){

package/cmd.js

@@ -1,42 +0,0 @@
-import chalk from 'chalk';
-import readline from 'readline';
-
-
-// 初始化数值  
-let values = [0, 0, 0];  
-  
-// 更新并显示数值的函数  
-function updateAndDisplayValues() {  
-  // 增加一些随机值  
-  values = values.map(value => value + Math.floor(Math.random() * 10));  
-  
-  // 清除当前行并将光标移回行首  
-  process.stdout.clearLine();  
-  process.stdout.cursorTo(0);  
-  
-  // 显示更新后的数值，不换行  
-  process.stdout.write(chalk.green(`Value 1: ${values[0]}\t`));  
-  process.stdout.write(chalk.yellow(`Value 2: ${values[1]}\t`));  
-  process.stdout.write(chalk.blue(`Value 3: ${values[2]}\r`)); // \r 将光标移回行首，准备下一次输出  
-}  
-  
-// 创建readline接口实例以监听键盘输入  
-const rl = readline.createInterface({  
-  input: process.stdin,  
-  output: process.stdout  
-});  
-  
-// 每秒更新和显示数值  
-setInterval(updateAndDisplayValues, 1000);  
-  
-// 监听'q'键退出  
-rl.on('line', (input) => {  
-  if (input.trim().toLowerCase() === 'q') {  
-    clearInterval(intervalId); // 清除定时器  
-    rl.close(); // 关闭readline接口  
-    process.exit(0); // 退出程序  
-  }  
-});  
-  
-// 存储定时器的ID，以便稍后清除它  
-let intervalId = setInterval(updateAndDisplayValues, 1000);

package/examples/example0/run-readme.js

@@ -1,88 +0,0 @@
-let multiTasks = require('../../index').multiTasks;
-
-//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
-let alltasks = [];
-for(let i=0;i<50;i++){
-    alltasks.push({
-        name: `task-${i}`,
-        data: `This prop is for a subtask`
-    });
-};
-
-//Step2, Provide a function to process a certain sub-task and return the result data
-let processTask = (task, helper)=>{
-    let {taskCount} = task;
-
-    return 'result data'; 
-};
-
-//Step2.1, or return a promise
-let processTask = (task, helper)=>{
-    let {taskCount} = task;
-
-    return new Promise((resolve, reject)=>{
-        resolve({
-            data:`task${taskCount} complete`
-        })
-    })
-};
-
-//Step2.2, dynamically create a new task if find something new while processing
-let processTask = (task, helper)=>{
-    let {taskCount} = task;
-
-    
-    if(taskCount % 2 === 0){
-        helper.createNewTasks({//You can dynamically create a new task if find something new while processing
-            msg:'a new task'
-        });
-        return;
-    }
-
-    return new Promise((resolve, reject)=>{
-        resolve({
-            data:`task${taskCount} complete`
-        })
-    })
-};
-//Step2.3, this example shows how to handle exceptions/errors
-let processTask = (task, helper)=>{
-    let {taskCount} = task;
-
-    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
-    if(taskCount===3) throw 'exception';
-    if(taskCount===4) return Promise.reject({err:'a test error'});
-    if(taskCount===5) aaa =  bbb;
-
-    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is OK
-
-    //by default, you should return a promise, all returned data can be found in the results/succ folder
-    //but to return a non-promise result is also OK, see above
-    return new Promise((resolve, reject)=>{
-        setTimeout(()=>{
-            if(taskCount % 2 === 0){
-                helper.createNewTasks({//You can dynamically create a new task if find something new while processing
-                    msg:'a new task'
-                });
-                resolve()
-            }else{
-                resolve({
-                    data:`task${taskCount} complete`
-                })
-            }
-        }, 10)
-    })
-}
-
-//Step3, run!
-multiTasks.start({
-    initialTasks: alltasks, 
-    processTask, 
-    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
-    taskId: 'my-readme-task',
-    numberOfWorkers: 3, //Assign how many workers are working in parallel
-    //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
-    onFinish: (report)=>{
-        console.log('finish callback', report);
-    }
-});

package/examples/example0/run.js

@@ -1,59 +0,0 @@
-let multiTasks = require('../../index').multiTasks;
-
-//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
-let alltasks = [];
-for(let i=0;i<50;i++){
-    let task_props = {
-        index: i,
-        name: `task-${i}`,
-        description: `This prop is for a subtask`
-    };
-    alltasks.push(task_props);
-}
-
-//Step2, Provide a process function to handle a certain sub-task and return the result data
-let processTask = (task, helper)=>{
-    //console.log('this is the subtask data you created above:', task);
-    let {taskCount} = task;
-
-    console.log(taskCount)
-
-
-    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
-    if(taskCount===3) throw 'exception';
-    if(taskCount===4) return Promise.reject({err:'a test error'});
-    if(taskCount===5) aaa =  bbb;
-
-    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is also OK
-    
-    if(taskCount===7) {return new Promise((resolve, reject)=>{//async exception
-        setTimeout(()=>{
-            a=beforeAll;
-        }, 10 * Math.random())
-    });}
-
-    //by default, you should return a promise, all returned data can be found in the results/succ folder
-    return new Promise((resolve, reject)=>{
-        setTimeout(()=>{
-            if(taskCount % 2 === 0)helper.createNewTasks({//
-                msg:'a new task'
-            });
-            taskCount === 2 ? reject('a rejected error') : resolve({
-                message: 'this is a result from a demo, random data=' + Math.random()
-            });
-        }, 10 * Math.random())
-    })
-}
-
-//Step3, run!
-multiTasks({
-    initialTasks: alltasks, 
-    processTask, 
-    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
-    taskId: 'my-task' +  Math.random(),
-    numberOfWorkers: 3, //Assign how many workers are working in parallel
-    //autoCloseAfterCompletion: true,
-    onFinish: (report)=>{
-        console.log('finish callback', report);
-    }
-});

package/examples/example0/runm.mjs

@@ -1,53 +0,0 @@
-import {multiTasks} from '../../index.js';
-
-//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
-let alltasks = [];
-for(let i=0;i<50;i++){
-    let task_props = {
-        index: i,
-        name: `task-${i}`,
-        description: `This prop is for a subtask`
-    };
-    alltasks.push(task_props);
-}
-
-//Step2, Provide a process function to handle a certain sub-task and return the result data
-let processTask = (task, helper)=>{
-    //console.log('this is the subtask data you created above:', task);
-    let {taskCount} = task;
-
-    console.log(taskCount)
-
-
-    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
-    if(taskCount===3) throw 'exception';
-    if(taskCount===4) return Promise.reject({err:'a test error'});
-    if(taskCount===5) aaa =  bbb;
-
-    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is also OK
-
-    //by default, you should return a promise, all returned data can be found in the results/succ folder
-    return new Promise((resolve, reject)=>{
-        setTimeout(()=>{
-            if(taskCount % 2 === 0)helper.createNewTasks({//
-                msg:'a new task'
-            });
-            taskCount === 2 ? reject('a rejected error') : resolve({
-                message: 'this is a result from a demo, random data=' + Math.random()
-            });
-        }, 10 * Math.random())
-    })
-}
-
-//Step3, run!
-multiTasks({
-    initialTasks: alltasks, 
-    processTask, 
-    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
-    taskId: 'my-task',
-    numberOfWorkers: 3, //Assign how many workers are working in parallel
-    //autoCloseAfterCompletion: true,
-    onFinish: (report)=>{
-        console.log('finish callback', report);
-    }
-});

package/monitors/cli.js

@@ -1,28 +0,0 @@
-const cliProgress = require('cli-progress');
-
-// create new container
-const multibar = new cliProgress.MultiBar({
-    clearOnComplete: false,
-    hideCursor: true,
-    format: ' {bar} | {filename} | {value}/{total}',
-}, cliProgress.Presets.shades_grey);
-
-// add bars
-const b1 = multibar.create(200, 0);
-const b2 = multibar.create(1000, 0);
-
-// control bars
-b1.increment();
-b2.update(20, {filename: "test1.txt"});
-b1.update(20, {filename: "helloworld.txt"});
-
-b2.setTotal(2000);
-let v = 200;
-setInterval(()=>{
-    v=v+20
-    b2.update(v, {filename: "2222.txt"});
-
-    // stop all bars
-    
-    //multibar.stop();
-}, 300)

package/test/splitter.test.js

@@ -1,17 +0,0 @@
-let splitter = require('../workers/taskSplitter');
-
-let createArray = (len)=>{  
-  let alltasks = []
-  for(let i=0;i<len;i++){
-      alltasks.push(i)
-  }
-  return alltasks;
-}
-
-test('test taskSplitter', async () => {
-    
-  expect(splitter.split(createArray(3), 5).length).toBe(1);
-  expect(splitter.split(createArray(16), 5).length).toBe(4);
-  expect(splitter.split(createArray(101), 1).length).toBe(101);
-  expect(splitter.split(createArray(101), 33).length).toBe(4);
-});

package/test/test_resume/step1_start_and_crash.js

@@ -1,56 +0,0 @@
-let multiTasks = require('../../index').multiTasks;
-
-//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
-let alltasks = [];
-for(let i=0;i<1000;i++){
-    let task_props = {
-        index: i,
-        name: `task-${i}`,
-        description: `This prop is for a subtask`
-    };
-    alltasks.push(task_props);
-}
-
-//Step2, Provide a process function to handle a certain sub-task and return the result data
-let processTask = (task, helper)=>{
-    //console.log('this is the subtask data you created above:', task);
-    let {taskCount} = task;
-
-    console.log(taskCount)
-
-
-    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
-    if(taskCount===3) throw 'exception';
-    if(taskCount===4) return Promise.reject({err:'a test error'});
-    if(taskCount===5) aaa =  bbb;
-
-    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is also OK
-
-    //by default, you should return a promise, all returned data can be found in the results/succ folder
-    return new Promise((resolve, reject)=>{
-        setTimeout(()=>{
-            if(taskCount % 2 === 0)helper.createNewTasks({//
-                msg:'a new task'
-            });
-            taskCount === 2 ? reject('a rejected error') : resolve({
-                message: 'this is a result from a demo, random data=' + Math.random()
-            });
-        }, 500 * Math.random())
-    })
-}
-
-//Step3, run!
-multiTasks({
-    initialTasks: alltasks, 
-    processTask, 
-    taskRootFolder: `./tmp`, //a directory to store progress and results files, you can check the progress here
-    taskId: 'test_resume',
-    numberOfWorkers: 3, //Assign how many workers are working in parallel
-    //autoCloseAfterCompletion: true,
-    shouldTerminate:(info)=>{
-        if(info.taskCount > 30) return true;
-    },
-    onFinish: (report)=>{
-        console.log('finish callback', report);
-    }
-});

package/test/test_resume/step2_resume.js

@@ -1,22 +0,0 @@
-let multiTasks = require('../../index').multiTasks;
-
-let processTask = (task, helper)=>{
-    let {taskCount} = task;
-
-    return 'a result which is not a promise'; 
-};
-
-//Step3, run!
-multiTasks({
-    processTask, 
-    taskRootFolder: `./tmp`, //a directory to store progress and results files, you can check the progress here
-    taskId: 'test_resume',
-    numberOfWorkers: 11, //Assign how many workers are working in parallel
-    //autoCloseAfterCompletion: true,
-    shouldTerminate:(info)=>{
-        //if(info.taskCount > 30) return true;
-    },
-    onFinish: (report)=>{
-        console.log('finish callback', report);
-    }
-});

package/test/test_resume/test.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-rm -rf ./tmp
-node step1_start_and_crash
-sleep 3
-node step2_resume

package/todo.txt

@@ -1,4 +0,0 @@
-Features:
-  - Continue execution from the point of interruption
-  - Re-run
-  - Show status in command line

package/utils/test/test_asyncqueue.js

@@ -1,116 +0,0 @@
-const AsyncQueue = require(`../asyncQueue`);
-const queue = new AsyncQueue();  
-
-let count=0;
-  
-queue.enqueue(async () => {
-    count++;  
-  console.log(`Task ${count} start`);  
-  await new Promise(resolve => setTimeout(resolve, 100)); 
-  console.log(`Task ${count} end`);  
-});
- 
-  
-queue.enqueue(async () => {
-    count++;  
-  console.log(`Task ${count} start`);  
-  await new Promise(resolve => setTimeout(resolve, 20)); 
-  console.log(`Task ${count} end`);  
-});
- 
-  
-queue.enqueue(() => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    console.log(`Task ${count} end`);  
-    return 11;
-  });
-      
-  
-queue.enqueue(() => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    console.log(`Task ${count} end`);  
-    return 11;
-  });   
-  
-  queue.enqueue(() => {
-      count++;  
-      console.log(`Task ${count} start`);  
-      console.log(`Task ${count} end`);  
-      return 11;
-    });   
-  
-    queue.enqueue(() => {
-        count++;  
-        console.log(`Task ${count} start`);  
-        console.log(`Task ${count} end`);  
-        return 11;
-      });   
-  
-      queue.enqueue(() => {
-          count++;  
-          console.log(`Task ${count} start`);  
-          console.log(`Task ${count} end`);  
-          return 11;
-        });   
-  
-        queue.enqueue(() => {
-            count++;  
-            console.log(`Task ${count} start`);  
-            console.log(`Task ${count} end`);  
-            return 11;
-          });
-  
-queue.enqueue(async () => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    await new Promise(resolve => setTimeout(resolve, 20)); 
-    console.log(`Task ${count} end`);  
-  });
-   
-  
-  
-queue.enqueue(async () => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    await new Promise(resolve => setTimeout(resolve, 20)); 
-    console.log(`Task ${count} end`);  
-  });
-   
-  
-  
-queue.enqueue(async () => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    await new Promise(resolve => setTimeout(resolve, 20)); 
-    console.log(`Task ${count} end`);  
-  });
-   
-  
-  
-queue.enqueue(async () => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    await new Promise(resolve => setTimeout(resolve, 20)); 
-    console.log(`Task ${count} end`);  
-  });
-   
-  
-queue.enqueue(() => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    console.log(`Task ${count} end`);  
-    return 11;
-  });   
-  
-  
-queue.enqueue(async () => {
-    count++;  
-    console.log(`Task ${count} start`);  
-    await new Promise(resolve => setTimeout(resolve, 20)); 
-    console.log(`Task ${count} end`);  
-  });
-   
-            
-  

package/utils/test_makedir.js

@@ -1,3 +0,0 @@
-let makedir = require('./makedir');
-makedir.sync('c:/aaa/bb')
-makedir.sync('c:/cc/dd')

package/workers/asWorker.js

@@ -1,14 +0,0 @@
-var fs = require('fs');
-var pathutil = require('path');
-const numCPUs = require('os').cpus().length;
-
-let MSG_REQUEST_TASKID = 'request_taskid';
-
-const TaskMgr = require('./TaskMgr');
-
-const init = (config, worker)=>{
-
-};
-module.exports = {
-    init
-};