multi-tasks 3.4.0 → 3.5.0

package/workers/asMaster.js

@@ -1,102 +1,110 @@
-var fs = require('fs');
-var pathUtil = require('path');
-
-const TaskMgr = require('./TaskMgr');
-const randomsUtil = require('../utils/randoms');
-const { resolveNumberOfWorkers } = require('../utils/resolveWorkers');
-
-const resume = (config)=>{
-    config.__resume = true;
-    let {taskRootFolder, taskId} = config;
-    let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
-
-    //load from config file: 旧快照作底(兼作任务目录合法性校验, 不存在抛 ENOENT),
-    //本次调用传入的配置覆盖同名字段——快照只补未传入的字段
-    let task_config = JSON.parse(fs.readFileSync(pathUtil.resolve(taskFolder, 'task_config.json')));
-    Object.keys(task_config).forEach((key)=>{
-        if(key.startsWith('__')) delete task_config[key];//读侧剔除一次性标记, 兼容 <=3.1.3 旧快照
-    });
-    config = {
-        ...task_config,
-        ...config
-    };
-    config.masterTaskId = taskId;
-
-    TaskMgr.load(config);
-    config = TaskMgr.initFolders(config);
-
-    return new Promise((resolve, reject)=>{
-
-        TaskMgr.getTaskFiles('running', (taskfiles)=>{
-            TaskMgr.getTaskFiles('finished_with_errors', (errfiles)=>{
-                let promises = [];
-                taskfiles.forEach((fname)=>{
-                    let subid = fname.replace(/\.task\.json$/, '');
-                    promises.push(TaskMgr.mvTaskPromise(subid, 'running', 'new'));
-                });
-                if(config.__retryFails){
-                    errfiles.forEach((fname)=>{
-                        let subid = fname.replace(/\.task\.json$/, '');
-                        promises.push(TaskMgr.mvTaskPromise(subid, 'finished_with_errors', 'new').then(()=>{
-                            TaskMgr.resetTaskRetry(subid);//retry_fails 重跑失败任务时重置重试计数(resume 不重置)
-                            TaskMgr.deleteUserMessage(subid);//重跑即新一轮: 清掉该任务在 .sys/user_messages 的遗留消息
-                        }));
-                    });
-                };
-                Promise.all(promises).then(()=>{
-                    TaskMgr.loadAllTasks().then((types)=>{
-                        console.log('******resume multi-thread')
-                        setTimeout(()=>{
-                            resolve(config);
-                        }, 1000);
-                    });
-
-                }).catch((e)=>{
-                    reject(e);
-                });
-            });
-        });
-    });
-
-
-};
-
-const init = (config)=>{
-    if(!config) config = {}; 
-    if(!config.initialTasks){ throw new Error('Please provide initialTasks data!');  };
-    if(!config.processTask){ throw new Error('Please provide processTask function!'); };
-
-    config.numberOfWorkers = resolveNumberOfWorkers(config.numberOfWorkers);//支持百分比字符串, 非法值在此抛错(fail fast, 不建任务目录)
-
-    let {taskRootFolder, taskName} = config;
-
-    if(taskName && !taskName.match(/^[a-zA-Z0-9_-]+$/)){ throw new Error('Error: taskName can only contain letters and numbers'); }
-    if(!taskName){ taskName = 'multitasks'; }
-
-    let {round, pow, random} = Math;
-    let dateTimeText = randomsUtil.getDateTimeTxt();
-    let randomNum = round((random() * pow(10,8)),8).toString(36);
-    let masterTaskId = config.taskId ? config.taskId : `${taskName}${dateTimeText}--${randomNum}`;
-
-    config.taskRootFolder = taskRootFolder;
-    config.masterTaskId = masterTaskId;
-
-
-    TaskMgr.load(config);
-    config = TaskMgr.initFolders(config);
-
-    config.initialTasks.forEach((task)=>{
-        TaskMgr.createNewTask(task);
-    });
-
-    //to reload is for to reStart a collapsed task
-    return new Promise((resolve)=>{
-        TaskMgr.loadNewTasks().then(()=>{
-            resolve(config);
-        });
-    });
-};
-module.exports = {
-    init,
-    resume
+var fs = require('fs');
+var pathUtil = require('path');
+
+const TaskMgr = require('./TaskMgr');
+const { createMemoryStore } = require('./MemoryStore');
+const randomsUtil = require('../utils/randoms');
+const { resolveNumberOfWorkers } = require('../utils/resolveWorkers');
+
+const resume = (config)=>{
+    config.__resume = true;
+    let {taskRootFolder, taskId} = config;
+    let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
+
+    //load from config file: 旧存档作底(兼作任务目录合法性校验, 不存在抛 ENOENT),
+    //本次调用传入的配置覆盖同名字段——存档只补未传入的字段
+    let task_config = JSON.parse(fs.readFileSync(pathUtil.resolve(taskFolder, 'task_config.json')));
+    Object.keys(task_config).forEach((key)=>{
+        if(key.startsWith('__')) delete task_config[key];//读侧剔除一次性标记, 兼容 <=3.1.3 旧存档
+    });
+    config = {
+        ...task_config,
+        ...config
+    };
+    config.masterTaskId = taskId;
+
+    TaskMgr.load(config);
+    config = TaskMgr.initFolders(config);
+
+    return new Promise((resolve, reject)=>{
+
+        TaskMgr.getTaskFiles('running', (taskfiles)=>{
+            TaskMgr.getTaskFiles('finished_with_errors', (errfiles)=>{
+                let promises = [];
+                taskfiles.forEach((fname)=>{
+                    let subid = fname.replace(/\.task\.json$/, '');
+                    promises.push(TaskMgr.mvTaskPromise(subid, 'running', 'new'));
+                });
+                if(config.__retryFails){
+                    errfiles.forEach((fname)=>{
+                        let subid = fname.replace(/\.task\.json$/, '');
+                        promises.push(TaskMgr.mvTaskPromise(subid, 'finished_with_errors', 'new').then(()=>{
+                            TaskMgr.resetTaskRetry(subid);//retry_fails 重跑失败任务时重置重试计数(resume 不重置)
+                            TaskMgr.deleteUserMessage(subid);//重跑即新一轮: 清掉该任务在 .sys/user_messages 的遗留消息
+                        }));
+                    });
+                };
+                Promise.all(promises).then(()=>{
+                    TaskMgr.loadAllTasks().then((types)=>{
+                        console.log('******resume multi-thread')
+                        setTimeout(()=>{
+                            resolve({config, store: null});//memory 档 resume 在 index.js 已被拦截 throw, 到这里的都是 file 档
+                        }, 1000);
+                    });
+
+                }).catch((e)=>{
+                    reject(e);
+                });
+            });
+        });
+    });
+
+
+};
+
+const init = (config)=>{
+    if(!config) config = {}; 
+    if(!config.initialTasks){ throw new Error('Please provide initialTasks data!');  };
+    if(!config.processTask){ throw new Error('Please provide processTask function!'); };
+
+    config.numberOfWorkers = resolveNumberOfWorkers(config.numberOfWorkers);//支持百分比字符串, 非法值在此抛错(fail fast, 不建任务目录)
+
+    let {taskRootFolder, taskName} = config;
+
+    if(taskName && !taskName.match(/^[a-zA-Z0-9_-]+$/)){ throw new Error('Error: taskName can only contain letters and numbers'); }
+    if(!taskName){ taskName = 'multitasks'; }
+
+    let {round, pow, random} = Math;
+    let dateTimeText = randomsUtil.getDateTimeTxt();
+    let randomNum = round((random() * pow(10,8)),8).toString(36);
+    let masterTaskId = config.taskId ? config.taskId : `${taskName}${dateTimeText}--${randomNum}`;
+
+    config.taskRootFolder = taskRootFolder;
+    config.masterTaskId = masterTaskId;
+
+
+    TaskMgr.load(config);
+    config = TaskMgr.initFolders(config);
+
+    if(config.task_storage === 'memory'){
+        //memory 档: 不建任务文件(目录与 task_config.json 照建照写), 播种进 master 内存存储
+        const store = createMemoryStore(config);
+        store.addMany(config.initialTasks);
+        return Promise.resolve({config, store});
+    };
+
+    config.initialTasks.forEach((task)=>{
+        TaskMgr.createNewTask(task);
+    });
+
+    //to reload is for to reStart a collapsed task
+    return new Promise((resolve)=>{
+        TaskMgr.loadNewTasks().then(()=>{
+            resolve({config, store: null});
+        });
+    });
+};
+module.exports = {
+    init,
+    resume
 };

package/workers/helper.js

@@ -1,67 +1,77 @@
-let fs = require('fs');
-let pathutil = require('path');
-
-let TaskMgr = require('./TaskMgr');
-let runLog = require('../utils/runLog');
-
-let myconfig;
-let listeners = {};//event -> handler, worker 进程内模块级广播监听 registry(每个 event 一个 handler)
-
-const load = (config)=>{
-    myconfig = config;
-};
-const createNewTasks = (tasks)=>{
-    if (!myconfig) throw new Error('Please call helper.load(config) first!');
-    if (!Array.isArray(tasks)) tasks = [tasks];
-    TaskMgr.load(myconfig);
-    tasks.forEach((task)=>{
-        runLog.write('Creating new task', task)
-        TaskMgr.createNewTask(task);
-    });
-};
-
-//广播事件给所有 worker(经 master 中转); option.includingMe 默认 true(也发给自己)
-const emit = (event, payload, option)=>{
-    if (typeof event !== 'string' || !event) throw new Error('helper.emit: event must be a non-empty string');
-    if (typeof process.send !== 'function') throw new Error('helper.emit: broadcast is only available in worker processes');
-    let includingMe = !option || option.includingMe !== false;
-    process.send({__broadcast: true, event, payload, includingMe});
-};
-//发送系统事件给 master(master 自己消费, 不转发); 内建事件 TERMINATE_ALL_WORKERS,
-//其余交给 config.setSysListener
-const emitSys = (event, payload)=>{
-    if (typeof event !== 'string' || !event) throw new Error('helper.emitSys: event must be a non-empty string');
-    if (typeof process.send !== 'function') throw new Error('helper.emitSys: system events are only available in worker processes');
-    process.send({__sysEvent: true, event, payload});
-};
-//监听广播事件, 每个 event 每个 worker 进程只保留一个 handler(重复注册覆盖),
-//handler 签名 (payload, meta), meta 含 fromWorkerId/fromWorkerPid
-const setListener = (event, handler)=>{
-    if (typeof event !== 'string' || !event) throw new Error('helper.setListener: event must be a non-empty string');
-    if (typeof handler !== 'function') throw new Error('helper.setListener: handler must be a function');
-    listeners[event] = handler;
-};
-//任务日志: 一律写 <taskFolder>/.sys/run.log; progressBar 未开/非 TTY 回退时 echo 上屏(效果等同 console.log)
-const log = (...args)=>{
-    runLog.write(...args);
-};
-//内部方法: 供 subWorker 收到 master 中转的广播时调用, 按 event 分发
-const __dispatchBroadcast = (message)=>{
-    let {event, payload, meta} = message;
-    let handler = listeners[event];
-    if (!handler) return;
-    try{
-        handler(payload, meta);
-    }catch(e){
-        runLog.error(`[Broadcast] handler for event "${event}" threw:`, e);
-    }
-};
-module.exports = {
-    load,
-    createNewTasks,
-    emit,
-    emitSys,
-    setListener,
-    log,
-    __dispatchBroadcast
-};
+let fs = require('fs');
+let pathutil = require('path');
+
+let TaskMgr = require('./TaskMgr');
+let IpcTaskStore = require('./IpcTaskStore');
+let runLog = require('../utils/runLog');
+
+let myconfig;
+let listeners = {};//event -> handler, worker 进程内模块级广播监听 registry(每个 event 一个 handler)
+
+const load = (config)=>{
+    myconfig = config;
+};
+const createNewTasks = (tasks)=>{
+    if (!myconfig) throw new Error('Please call helper.load(config) first!');
+    if (!Array.isArray(tasks)) tasks = [tasks];
+    if(myconfig.task_storage === 'memory'){
+        //memory 档 worker 不碰盘: 每个动态任务一条 createTask 消息即时发 master(逐条同构, 禁止缓冲攒批)
+        IpcTaskStore.load(myconfig);
+        tasks.forEach((task)=>{
+            runLog.write('Creating new task', task)
+            IpcTaskStore.createNewTask(task);
+        });
+        return;
+    };
+    TaskMgr.load(myconfig);
+    tasks.forEach((task)=>{
+        runLog.write('Creating new task', task)
+        TaskMgr.createNewTask(task);
+    });
+};
+
+//广播事件给所有 worker(经 master 中转); option.includingMe 默认 true(也发给自己)
+const emit = (event, payload, option)=>{
+    if (typeof event !== 'string' || !event) throw new Error('helper.emit: event must be a non-empty string');
+    if (typeof process.send !== 'function') throw new Error('helper.emit: broadcast is only available in worker processes');
+    let includingMe = !option || option.includingMe !== false;
+    process.send({__broadcast: true, event, payload, includingMe});
+};
+//发送系统事件给 master(master 自己消费, 不转发); 内建事件 TERMINATE_ALL_WORKERS,
+//其余交给 config.setSysListener
+const emitSys = (event, payload)=>{
+    if (typeof event !== 'string' || !event) throw new Error('helper.emitSys: event must be a non-empty string');
+    if (typeof process.send !== 'function') throw new Error('helper.emitSys: system events are only available in worker processes');
+    process.send({__sysEvent: true, event, payload});
+};
+//监听广播事件, 每个 event 每个 worker 进程只保留一个 handler(重复注册覆盖),
+//handler 签名 (payload, meta), meta 含 fromWorkerId/fromWorkerPid
+const setListener = (event, handler)=>{
+    if (typeof event !== 'string' || !event) throw new Error('helper.setListener: event must be a non-empty string');
+    if (typeof handler !== 'function') throw new Error('helper.setListener: handler must be a function');
+    listeners[event] = handler;
+};
+//任务日志: 一律写 <taskFolder>/.sys/run.log; progressBar 未开/非 TTY 回退时 echo 上屏(效果等同 console.log)
+const log = (...args)=>{
+    runLog.write(...args);
+};
+//内部方法: 供 subWorker 收到 master 中转的广播时调用, 按 event 分发
+const __dispatchBroadcast = (message)=>{
+    let {event, payload, meta} = message;
+    let handler = listeners[event];
+    if (!handler) return;
+    try{
+        handler(payload, meta);
+    }catch(e){
+        runLog.error(`[Broadcast] handler for event "${event}" threw:`, e);
+    }
+};
+module.exports = {
+    load,
+    createNewTasks,
+    emit,
+    emitSys,
+    setListener,
+    log,
+    __dispatchBroadcast
+};

package/workers/index.js

@@ -5,6 +5,27 @@ const Multithread = require('./Multithread');
 const backwardv1 = require('../utils/backward-v1');
 const { FUNCTION_CONFIG_KEYS, reviveConfig } = require('../utils/serializeFns');
 
+const TASK_STORAGE_MODES = ['file', 'memory'];
+//task_storage 两档: file(默认, 任务状态实时写磁盘, 可 resume) / memory(纯内存, 进程退出即丢)
+const _resolveTaskStorage = (config)=>{
+    let mode = config.task_storage;
+    if(typeof mode === 'undefined' || mode === null){
+        mode = 'file';
+    };
+    if(TASK_STORAGE_MODES.indexOf(mode) < 0){
+        throw new Error(`invalid task_storage "${mode}": must be one of ${TASK_STORAGE_MODES.join('/')}`);
+    };
+    config.task_storage = mode;
+};
+//纯 memory 档目录中无可恢复状态, resume 系 API 调用即 throw;
+//只在 master 进程拦截: worker 进程重跑入口脚本时目录已被 master 创建, 会走 resume 链路拿 __sysLogFile,
+//worker 侧合并配置后不进 init/resume, 不受此限(否则 memory 档 worker 一起步就 throw)
+const _assertMemoryModeResumable = (config)=>{
+    if(config.task_storage === 'memory' && (cluster.isPrimary || cluster.isMaster)){
+        throw new Error('task_storage "memory" does not persist task state: resume/retry_fails/restart are not available (nothing recoverable in the task folder)');
+    };
+};
+
 const _toAbsPath = (path)=>{
     if(!path) return path;
     if(pathUtil.isAbsolute(path))return path;
@@ -97,6 +118,7 @@ const start = (config)=>{
         config.taskRootFolder = _toAbsPath(config.taskRootFolder);
         config.taskFolder = _toAbsPath(config.taskFolder);
         _computeSysLogFile(config);
+        _resolveTaskStorage(config);
 
         Multithread.start(config);
     });
@@ -119,6 +141,8 @@ const resume = (config)=>{
     return _proceedAfterEval(_evalInitialTasks(config), ()=>{
         _computeSysLogFile(config);
         config = _mergeSnapshotConfig(config);
+        _resolveTaskStorage(config);
+        _assertMemoryModeResumable(config);
         Multithread.start(config);
     });
 };
@@ -136,6 +160,8 @@ const retry_fails = (config)=>{
     return _proceedAfterEval(_evalInitialTasks(config), ()=>{
         _computeSysLogFile(config);
         config = _mergeSnapshotConfig(config);
+        _resolveTaskStorage(config);
+        _assertMemoryModeResumable(config);
         Multithread.start(config);
     });
 };
@@ -152,6 +178,8 @@ const restart = (config)=>{
     return _proceedAfterEval(_evalInitialTasks(config), ()=>{
         _computeSysLogFile(config);
         config = _mergeSnapshotConfig(config);
+        _resolveTaskStorage(config);
+        _assertMemoryModeResumable(config);
         Multithread.start(config);
     });
 };