multi-tasks 3.2.3 → 3.4.0

package/workers/WorkerMgr.js

@@ -1,27 +1,18 @@
-let workers = {};
-let status = {};
-
-const runLog = require('../utils/runLog');
-
-const mgr = {
-    addWorker:(worker)=>{
-        let id = `${worker.id}`;
-        if(workers[id] || status[id]){
-            throw new Error('FATAL: duplicated worker!');
-        }
-        status[id] = {isBusy: true};
-        runLog.write('add worker', worker.id);
-        workers[id] = worker;
-    },
-    getWorker:(id)=>{
-        return workers[`${id}`];
-    },
-    setBusy:(id, isBusy)=>{
-        status[`${id}`] = {isBusy: isBusy};
-    },
-    isBusy:(id)=>{
-        let s = status[`${id}`];
-        return s ? s.isBusy : false;
-    }
-};
-module.exports = mgr;
+let workers = {};
+
+const runLog = require('../utils/runLog');
+
+const mgr = {
+    addWorker:(worker)=>{
+        let id = `${worker.id}`;
+        if(workers[id]){
+            throw new Error('FATAL: duplicated worker!');
+        }
+        runLog.write('add worker', worker.id);
+        workers[id] = worker;
+    },
+    removeWorker:(id)=>{//worker 退出(含崩溃补员)时清引用: map 只增不减会累积死 worker
+        delete workers[`${id}`];
+    }
+};
+module.exports = mgr;

package/workers/asMaster.js

@@ -1,101 +1,102 @@
-var fs = require('fs');
-var pathUtil = require('path');
-
-const TaskMgr = require('./TaskMgr');
-const randomsUtil = require('../utils/randoms');
-const { resolveNumberOfWorkers } = require('../utils/resolveWorkers');
-
-const resume = (config)=>{
-    config.__resume = true;
-    let {taskRootFolder, taskId} = config;
-    let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
-
-    //load from config file: 旧快照作底(兼作任务目录合法性校验, 不存在抛 ENOENT),
-    //本次调用传入的配置覆盖同名字段——快照只补未传入的字段
-    let task_config = JSON.parse(fs.readFileSync(pathUtil.resolve(taskFolder, 'task_config.json')));
-    Object.keys(task_config).forEach((key)=>{
-        if(key.startsWith('__')) delete task_config[key];//读侧剔除一次性标记, 兼容 <=3.1.3 旧快照
-    });
-    config = {
-        ...task_config,
-        ...config
-    };
-    config.masterTaskId = taskId;
-
-    TaskMgr.load(config);
-    config = TaskMgr.initFolders(config);
-
-    return new Promise((resolve, reject)=>{
-
-        TaskMgr.getTaskFiles('running', (taskfiles)=>{
-            TaskMgr.getTaskFiles('finished_with_errors', (errfiles)=>{
-                let promises = [];
-                taskfiles.forEach((fname)=>{
-                    let subid = fname.replace(/\.task\.json$/, '');
-                    promises.push(TaskMgr.mvTaskPromise(subid, 'running', 'new'));
-                });
-                if(config.__retryFails){
-                    errfiles.forEach((fname)=>{
-                        let subid = fname.replace(/\.task\.json$/, '');
-                        promises.push(TaskMgr.mvTaskPromise(subid, 'finished_with_errors', 'new').then(()=>{
-                            TaskMgr.resetTaskRetry(subid);//retry_fails 重跑失败任务时重置重试计数(resume 不重置)
-                        }));
-                    });
-                };
-                Promise.all(promises).then(()=>{
-                    TaskMgr.loadAllTasks().then((types)=>{
-                        console.log('******resume multi-thread')
-                        setTimeout(()=>{
-                            resolve(config);
-                        }, 1000);
-                    });
-
-                }).catch((e)=>{
-                    reject(e);
-                });
-            });
-        });
-    });
-
-
-};
-
-const init = (config)=>{
-    if(!config) config = {}; 
-    if(!config.initialTasks){ throw new Error('Please provide initialTasks data!');  };
-    if(!config.processTask){ throw new Error('Please provide processTask function!'); };
-
-    config.numberOfWorkers = resolveNumberOfWorkers(config.numberOfWorkers);//支持百分比字符串, 非法值在此抛错(fail fast, 不建任务目录)
-
-    let {taskRootFolder, taskName} = config;
-
-    if(taskName && !taskName.match(/^[a-zA-Z0-9_-]+$/)){ throw new Error('Error: taskName can only contain letters and numbers'); }
-    if(!taskName){ taskName = 'multitasks'; }
-
-    let {round, pow, random} = Math;
-    let dateTimeText = randomsUtil.getDateTimeTxt();
-    let randomNum = round((random() * pow(10,8)),8).toString(36);
-    let masterTaskId = config.taskId ? config.taskId : `${taskName}${dateTimeText}--${randomNum}`;
-
-    config.taskRootFolder = taskRootFolder;
-    config.masterTaskId = masterTaskId;
-
-
-    TaskMgr.load(config);
-    config = TaskMgr.initFolders(config);
-
-    config.initialTasks.forEach((task)=>{
-        TaskMgr.createNewTask(task);
-    });
-
-    //to reload is for to reStart a collapsed task
-    return new Promise((resolve)=>{
-        TaskMgr.loadNewTasks().then(()=>{
-            resolve(config);
-        });
-    });
-};
-module.exports = {
-    init,
-    resume
+var fs = require('fs');
+var pathUtil = require('path');
+
+const TaskMgr = require('./TaskMgr');
+const randomsUtil = require('../utils/randoms');
+const { resolveNumberOfWorkers } = require('../utils/resolveWorkers');
+
+const resume = (config)=>{
+    config.__resume = true;
+    let {taskRootFolder, taskId} = config;
+    let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
+
+    //load from config file: 旧快照作底(兼作任务目录合法性校验, 不存在抛 ENOENT),
+    //本次调用传入的配置覆盖同名字段——快照只补未传入的字段
+    let task_config = JSON.parse(fs.readFileSync(pathUtil.resolve(taskFolder, 'task_config.json')));
+    Object.keys(task_config).forEach((key)=>{
+        if(key.startsWith('__')) delete task_config[key];//读侧剔除一次性标记, 兼容 <=3.1.3 旧快照
+    });
+    config = {
+        ...task_config,
+        ...config
+    };
+    config.masterTaskId = taskId;
+
+    TaskMgr.load(config);
+    config = TaskMgr.initFolders(config);
+
+    return new Promise((resolve, reject)=>{
+
+        TaskMgr.getTaskFiles('running', (taskfiles)=>{
+            TaskMgr.getTaskFiles('finished_with_errors', (errfiles)=>{
+                let promises = [];
+                taskfiles.forEach((fname)=>{
+                    let subid = fname.replace(/\.task\.json$/, '');
+                    promises.push(TaskMgr.mvTaskPromise(subid, 'running', 'new'));
+                });
+                if(config.__retryFails){
+                    errfiles.forEach((fname)=>{
+                        let subid = fname.replace(/\.task\.json$/, '');
+                        promises.push(TaskMgr.mvTaskPromise(subid, 'finished_with_errors', 'new').then(()=>{
+                            TaskMgr.resetTaskRetry(subid);//retry_fails 重跑失败任务时重置重试计数(resume 不重置)
+                            TaskMgr.deleteUserMessage(subid);//重跑即新一轮: 清掉该任务在 .sys/user_messages 的遗留消息
+                        }));
+                    });
+                };
+                Promise.all(promises).then(()=>{
+                    TaskMgr.loadAllTasks().then((types)=>{
+                        console.log('******resume multi-thread')
+                        setTimeout(()=>{
+                            resolve(config);
+                        }, 1000);
+                    });
+
+                }).catch((e)=>{
+                    reject(e);
+                });
+            });
+        });
+    });
+
+
+};
+
+const init = (config)=>{
+    if(!config) config = {}; 
+    if(!config.initialTasks){ throw new Error('Please provide initialTasks data!');  };
+    if(!config.processTask){ throw new Error('Please provide processTask function!'); };
+
+    config.numberOfWorkers = resolveNumberOfWorkers(config.numberOfWorkers);//支持百分比字符串, 非法值在此抛错(fail fast, 不建任务目录)
+
+    let {taskRootFolder, taskName} = config;
+
+    if(taskName && !taskName.match(/^[a-zA-Z0-9_-]+$/)){ throw new Error('Error: taskName can only contain letters and numbers'); }
+    if(!taskName){ taskName = 'multitasks'; }
+
+    let {round, pow, random} = Math;
+    let dateTimeText = randomsUtil.getDateTimeTxt();
+    let randomNum = round((random() * pow(10,8)),8).toString(36);
+    let masterTaskId = config.taskId ? config.taskId : `${taskName}${dateTimeText}--${randomNum}`;
+
+    config.taskRootFolder = taskRootFolder;
+    config.masterTaskId = masterTaskId;
+
+
+    TaskMgr.load(config);
+    config = TaskMgr.initFolders(config);
+
+    config.initialTasks.forEach((task)=>{
+        TaskMgr.createNewTask(task);
+    });
+
+    //to reload is for to reStart a collapsed task
+    return new Promise((resolve)=>{
+        TaskMgr.loadNewTasks().then(()=>{
+            resolve(config);
+        });
+    });
+};
+module.exports = {
+    init,
+    resume
 };