multi-tasks 3.4.0 → 3.5.0

package/workers/Multithread.js

@@ -7,6 +7,7 @@ let MSG_NO_TASK_FOUND = 'no_task_found';
 
 const asMaster = require('./asMaster');
 const TaskMgr = require('./TaskMgr');
+const IpcTaskStore = require('./IpcTaskStore');
 const WorkerMgr = require('./WorkerMgr');
 const helper = require('./helper');
 const { resolveNumberOfWorkers } = require('../utils/resolveWorkers');
@@ -58,6 +59,53 @@ const start = (config)=>{
         const masterStartTimestamp = new Date()*1;//本轮 master 启动时间(resume 不追溯首轮)
         let workerCrashCount = 0;//累计崩溃补员次数, 进 onFinish report
 
+        let memoryStore = null;//memory 档: 任务状态/结果全在 master 内存(file 档恒为 null, 走 TaskMgr 磁盘链路)
+        let getStats = ()=> memoryStore ? memoryStore.stats() : TaskMgr.getFinalStats();
+        let getRemaining = ()=> memoryStore ? memoryStore.getRemaining() : TaskMgr.getTasks('new').length;
+
+        //memory 档 worker 存储操作的应用点: 只在 __storageOp 分支的 queue.enqueue 里调(与派发同一串行通道,
+        //同 worker 的操作消息排在其后续领新请求之前, pop 时 Map 必已含该 worker 此前创建的全部任务)
+        let applyStorageOp = (worker, message)=>{
+            let op = message.op;
+            let args = (message.args || []).map((value)=>{
+                return (value && value[IpcTaskStore.__UNDEFINED_SENTINEL]) ? undefined : value;
+            });
+            let ackValue;
+            let ok = true;
+            let error = null;
+            try{
+                switch(op){
+                    case 'createTask':
+                        memoryStore.add(args[0]);
+                        break;
+                    case 'saveLog': memoryStore.saveLog(args[0], args[1]); break;
+                    case 'saveResult': memoryStore.saveResult(args[0], args[1], args[2]); break;
+                    case 'deleteResult': memoryStore.deleteResult(args[0]); break;
+                    case 'mvTask': {
+                        let succ = memoryStore.move(args[0], args[1], args[2]);
+                        ackValue = {newPath: `${args[2]}/${args[0]}.task.json`, succ};
+                        break;
+                    };
+                    case 'incrementRetry': ackValue = memoryStore.incrementRetry(args[0]); break;
+                    case 'saveUserMessage': memoryStore.addMessage(args[0]); break;
+                    case 'deleteUserMessage': memoryStore.clearMessages(args[0]); break;
+                    default:
+                        throw new Error(`unknown storage op "${op}"`);
+                };
+            }catch(e){
+                ok = false;
+                error = {message: e && e.message ? e.message : String(e)};
+                runLog.warn('[Master]: failed to apply storage op', op, e);
+            };
+            if(message.needAck){
+                try{
+                    worker.send({__storageOpResult: true, ok, value: ackValue, error});
+                }catch(sendErr){
+                    //来源 worker 已死: 回执发不出, 吞掉(在飞任务由崩溃处理兜底)
+                };
+            };
+        };
+
         let clearTaskTimer = (workerId)=>{
             let rec = workerTaskTimers[workerId];
             if(rec){
@@ -71,16 +119,58 @@ const start = (config)=>{
                 clearTaskTimer(id);
             };
             killAllWorkers();
-            //计数器行与正常完成路径一致: 始终进 run.log(progressBar 未接管终端时 echo 上屏);
-            //runLog 写流与 stdout 都是异步写, 先写完再延迟 exit, 防计数器行被 process.exit 截断
-            let finalStats = TaskMgr.getFinalStats();
-            runLog.write(`[Tasks] succeeded=${finalStats.succeeded}, failed=${finalStats.failed}, succ-but-save-failed=${finalStats.succButSaveFailed || 0}`);
-            progressCtl.finish();
-            setTimeout(()=>{
-                process.exit(0);
-            }, 10);
+            let writeCounterAndExit = ()=>{
+                //计数器行与正常完成路径一致: 始终进 run.log(progressBar 未接管终端时 echo 上屏);
+                //runLog 写流与 stdout 都是异步写, 先写完再延迟 exit, 防计数器行被 process.exit 截断
+                let finalStats = getStats();
+                runLog.write(`[Tasks] succeeded=${finalStats.succeeded}, failed=${finalStats.failed}, succ-but-save-failed=${finalStats.succButSaveFailed || 0}`);
+                progressCtl.finish();
+                setTimeout(()=>{
+                    process.exit(0);
+                }, 10);
+            };
+            if(memoryStore){
+                //memory 档: 收尾排进派发同一串行队列(append-and-return, 绝不阻塞等队列——本函数在
+                //shouldTerminate 路径就是在队列任务内被调用的, 阻塞 await 队列即死锁): 已到达未应用的
+                //worker 存储操作先于收尾计数应用; 此后排队的已死 worker 操作按 terminate 语义丢弃
+                queue.enqueue(async ()=>{
+                    writeCounterAndExit();
+                });
+                return;
+            };
+            writeCounterAndExit();
+        };
+        let finalizeTimeoutTaskInMemory = (subTask, cost)=>{
+            //memory 档: 与 file 档同语义同顺序(先结果后迁移); 只在 queue.enqueue 里调(与 worker 存储操作
+            //同一串行通道): 该 worker 已到达未应用的 saveResult 等操作必先于本判定应用, 已交卷任务不会被误杀
+            let subid = subTask.subid;
+            try{
+                let resultType = memoryStore.getSavedResultType(subid);
+                if(resultType === 'succ'){//超时边界: worker 已成功交卷, 视为已完成
+                    memoryStore.move(subid, 'running', 'finished');
+                }else if(resultType === 'error'){//超时边界: worker 已自行记错误
+                    memoryStore.move(subid, 'running', 'finished_with_errors');
+                }else{
+                    if(typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0){
+                        let retryCount = memoryStore.incrementRetry(subid);
+                        if(retryCount <= config.maxTaskRetries){
+                            runLog.warn(`[Master]: task "${subid}" timeout, retry (${retryCount}/${config.maxTaskRetries})`);
+                            memoryStore.move(subid, 'running', 'new');
+                            return;
+                        };
+                    };
+                    memoryStore.saveResult(subid, true, [{type: 'timeout', timeout: config.taskTimeout, cost}]);
+                    memoryStore.move(subid, 'running', 'finished_with_errors');
+                }
+            }catch(e){
+                runLog.warn('[Master]: failed to finalize timeout task', subid, e);
+            }
         };
         let finalizeTimeoutTask = (subTask, cost)=>{
+            if(memoryStore){
+                queue.enqueue(async ()=>{ finalizeTimeoutTaskInMemory(subTask, cost); });
+                return;
+            };
             let subid = subTask.subid;
             try{
                 let resultType = TaskMgr.getSavedResultType(subid);
@@ -114,8 +204,30 @@ const start = (config)=>{
             worker.kill();//不 disconnect: exit 事件会按崩溃补员
             finalizeTimeoutTask(subTask, cost);
         };
+        let handleCrashedTaskInMemory = (subTask)=>{
+            //memory 档: 与 file 档同语义(重试移回 new / 超限记 worker_crash 归档), 操作走内存存储;
+            //只在 queue.enqueue 里调: 该 worker 崩溃前已到达的存储操作必先应用, 再判定重试/归档
+            let subid = subTask.subid;
+            try{
+                let retryCount = memoryStore.incrementRetry(subid);
+                if(retryCount <= config.maxTaskRetries){
+                    runLog.warn(`[Master]: worker crashed, retry task "${subid}" (${retryCount}/${config.maxTaskRetries})`);
+                    memoryStore.move(subid, 'running', 'new');
+                }else{
+                    runLog.warn(`[Master]: task "${subid}" crashed ${retryCount} times, give up`);
+                    memoryStore.saveResult(subid, true, [{type: 'worker_crash', retryCount: config.maxTaskRetries}]);
+                    memoryStore.move(subid, 'running', 'finished_with_errors');
+                };
+            }catch(e){
+                runLog.warn('[Master]: failed to handle crashed task', subid, e);
+            }
+        };
         let handleCrashedTask = (subTask)=>{
             if(!(typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0)) return;
+            if(memoryStore){
+                queue.enqueue(async ()=>{ handleCrashedTaskInMemory(subTask); });
+                return;
+            };
             let subid = subTask.subid;
             try{
                 let retryCount = TaskMgr.incrementTaskRetry(subid);
@@ -177,25 +289,36 @@ const start = (config)=>{
                     }
                     return;
                 }
+                if(memoryStore && message && message.__storageOp){//memory 档 worker 存储操作: 与派发同一串行通道逐条应用, 禁止捎带合批
+                    queue.enqueue(async ()=>{ applyStorageOp(worker, message); });
+                    return;
+                }
                 if(message === MSG_REQUEST_NEW_TASK){
                     clearTaskTimer(worker.id);//上一个任务已结束(无论成败), 撤销超时倒计时
                     progressCtl.taskCompleted(worker.id);//worker 行回到 idle; 完成数按盘上结果目录统计, 不在此计数
                     queue.enqueue(async () => {
-                        //use queue to make sure no I/O conflict; update/pop/判空须在同一个串行任务里原子完成
-                        await TaskMgr.updateNewTasks();
-                        let subTask = await TaskMgr.popNewTask();
-                        if(!subTask){
-                            //判空前强制绕开缓存重读一次磁盘再 pop: 防其他 worker 动态建任务(createNewTasks)
-                            //落在 update 与判空之间被漏判, 导致 autoClose 提前杀光 worker / 丢新任务
-                            await TaskMgr.loadNewTasks();
-                            subTask = await TaskMgr.popNewTask();
-                        }
-                        if(!subTask && config.autoCloseAfterCompletion){
-                            //杀 worker 前延时 1s 再兜底确认一次: 等其他 worker 可能正在落盘的动态新任务
-                            //(best-effort 再压小竞态窗口, 只限 autoClose 路径, 普通派发零开销)
-                            await new Promise((resolve)=>{ setTimeout(resolve, 1000); });
-                            await TaskMgr.loadNewTasks();
+                        let subTask = null;
+                        if(memoryStore){
+                            //memory 档: 任务全在内存队列, pop 即出队; 无需磁盘重读兜底(动态任务经 __storageOp
+                            //同一串行通道进来, pop 时必然已可见)
+                            subTask = memoryStore.pop();
+                        }else{
+                            //use queue to make sure no I/O conflict; update/pop/判空须在同一个串行任务里原子完成
+                            await TaskMgr.updateNewTasks();
                             subTask = await TaskMgr.popNewTask();
+                            if(!subTask){
+                                //判空前强制绕开缓存重读一次磁盘再 pop: 防其他 worker 动态建任务(createNewTasks)
+                                //落在 update 与判空之间被漏判, 导致 autoClose 提前杀光 worker / 丢新任务
+                                await TaskMgr.loadNewTasks();
+                                subTask = await TaskMgr.popNewTask();
+                            }
+                            if(!subTask && config.autoCloseAfterCompletion){
+                                //杀 worker 前延时 1s 再兜底确认一次: 等其他 worker 可能正在写盘的动态新任务
+                                //(best-effort 再压小竞态窗口, 只限 autoClose 路径, 普通派发零开销)
+                                await new Promise((resolve)=>{ setTimeout(resolve, 1000); });
+                                await TaskMgr.loadNewTasks();
+                                subTask = await TaskMgr.popNewTask();
+                            }
                         }
                         if(!subTask){
                             //no tasks left
@@ -224,7 +347,7 @@ const start = (config)=>{
                         //send new tasks to the worker
                         let startTimestamp = new Date()*1;
                         //IPC 用裁剪副本: worker 端只取目录路径, 不需要 initialTasks 全量数组;
-                        //带着它每次派发都要 O(N) 序列化(总量 O(N²))。config 本体保留: resume/restart 重写快照要用
+                        //带着它每次派发都要 O(N) 序列化(总量 O(N²))。config 本体保留: resume/restart 重写存档要用
                         let ipcConfig = { ...config };
                         delete ipcConfig.initialTasks;
                         worker.send({
@@ -250,8 +373,9 @@ const start = (config)=>{
         };
 
         let initPromise = config.__resume ? asMaster.resume(config) : asMaster.init(config);
-        initPromise.then((updated_config)=>{
-            config = updated_config;
+        initPromise.then((inited)=>{
+            config = inited.config;
+            memoryStore = inited.store || null;
             //一次性启动块(走 console, 在劫持前上屏): 标题框横幅 + 平铺关键字段, 模式由 __ 标记推导;
             //init 成功后所有信息齐备(new tasks 数/masterTaskId/__sysLogFile), 一次打完
             let startupInfo = {
@@ -261,7 +385,7 @@ const start = (config)=>{
                 numOfWorkers,
                 numCPUs,
                 pid: process.pid,
-                newTasks: TaskMgr.getTasks('new').length,
+                newTasks: getRemaining(),
                 sysLogFile: config.__sysLogFile,
                 startedAt: randomsUtil.getDateTimeTxt(),
             };
@@ -269,8 +393,8 @@ const start = (config)=>{
             progressCtl.setup({//TTY 判定 + 渲染器创建都在控制器内; 统计口径注入, 控制器不依赖 TaskMgr
                 config,
                 stream: process.stdout,
-                getStats: () => TaskMgr.getFinalStats(),
-                getRemaining: () => TaskMgr.getTasks('new').length,
+                getStats,
+                getRemaining,
             });
             //console 劫持写 run.log 是永久机制, 与 progressBar 无关: 进度区接管终端(开且 TTY)时只写文件,
             //否则 echo 同时透传 console 上屏; 判定收敛在 initAndHook 内, 无 __sysLogFile 时维持 console
@@ -310,12 +434,13 @@ const start = (config)=>{
                 allkilled = allkilled && worker.killed;
             }
             if(allkilled){
+                let finishAll = ()=>{
                 if(USER_CONFIG.onFinish){
                     let endTimestamp = new Date()*1;
-                    let stats = TaskMgr.getFinalStats();
+                    let stats = getStats();
                     let report = {
                         taskId: config.masterTaskId,
-                        taskFolder: stats.taskFolder,
+                        taskFolder: config.taskFolder,
                         startTimestamp: masterStartTimestamp,
                         endTimestamp,
                         cost: endTimestamp - masterStartTimestamp,
@@ -326,23 +451,43 @@ const start = (config)=>{
                         succButSaveFailed: stats.succButSaveFailed || 0,//任务成功但结果保存失败(序列化兜底)的任务数, 明细见 .sys/user_messages
                         workers: {count: numOfWorkers, crashed: workerCrashCount},
                     };
-                    let onFinishHelper = {
-                        foreachSuccResult: (cb) => TaskMgr.forEachSuccResult(cb),
-                        foreachErrorResult: (cb) => TaskMgr.forEachErrorResult(cb),
-                        foreachResult: (cb) => TaskMgr.forEachResult(cb),
-                    };
+                    let onFinishHelper;
+                    if(config.task_storage === 'memory'){
+                        //memory 档: 结果只在内存(盘上 results/ 恒空), 遍历接口委托 memoryStore(与 TaskMgr 同签名同口径)
+                        onFinishHelper = {
+                            foreachSuccResult: (cb) => memoryStore.forEachSuccResult(cb),
+                            foreachErrorResult: (cb) => memoryStore.forEachErrorResult(cb),
+                            foreachResult: (cb) => memoryStore.forEachResult(cb),
+                        };
+                    }else{
+                        onFinishHelper = {
+                            foreachSuccResult: (cb) => TaskMgr.forEachSuccResult(cb),
+                            foreachErrorResult: (cb) => TaskMgr.forEachErrorResult(cb),
+                            foreachResult: (cb) => TaskMgr.forEachResult(cb),
+                        };
+                    }
                     USER_CONFIG.onFinish(report, onFinishHelper);
                     delete USER_CONFIG.onFinish;
                 }
                 //计数器行先进 run.log(progressBar 未接管终端时 echo 上屏, 接管时由 progressCtl.finish() 打在屏幕);
                 //stdout/run.log 都是异步写, 写完留 10ms 再 exit, 防 process.exit 截断未 flush 的输出
-                let finalStats = TaskMgr.getFinalStats();
+                let finalStats = getStats();
                 runLog.write(`[Tasks] succeeded=${finalStats.succeeded}, failed=${finalStats.failed}, succ-but-save-failed=${finalStats.succButSaveFailed || 0}`);
                 runLog.write('[Master]: all workers are done, process exit.');
                 progressCtl.finish();//单行进度条换行定格 + 计数器行, 别让 shell 提示符盖住它
                 setTimeout(()=>{
                     process.exit(0);
                 }, 10);
+                };
+                if(memoryStore){
+                    //memory 档: 同 terminateAllAndExit 口径, 收尾排进派发串行队列, 队列里已到达的存储操作
+                    //先应用再统计; file 档维持原同步收尾一字不动
+                    queue.enqueue(async ()=>{
+                        finishAll();
+                    });
+                }else{
+                    finishAll();
+                }
             }
         }); 
     }else{
@@ -371,6 +516,10 @@ function subWorker(config){
             helper.__dispatchBroadcast(message);
             return;
         }
+        if(message && message.__storageOpResult){//memory 档存储操作回执: 兑付 IpcTaskStore 的 needAck 等待队列
+            IpcTaskStore.__dispatchStorageOpResult(message);
+            return;
+        }
         if(message === MSG_NO_TASK_FOUND){//wait for new task
             runLog.write(`  (Worker #${worker.id}): waiting`);
             setTimeout(()=>{
@@ -379,7 +528,10 @@ function subWorker(config){
         }else
         if(message.subTask){//new task
             let {config, subTask, startTimestamp, taskCount} = message;
-            TaskMgr.load(config);
+            //memory 档 worker 不碰任务目录: 存储后端换 IpcTaskStore(逐条 {__storageOp} 发 master),
+            //操作序列与 file 档逐条同构; file 档后端即 TaskMgr(磁盘, 现状不变)
+            let store = config.task_storage === 'memory' ? IpcTaskStore : TaskMgr;
+            store.load(config);
             //用户任务自带的 index/taskCount 保留原值(重试/续跑后任务身份不变),
             //未提供时才填全局派发计数器(v1 兼容)
             if(typeof subTask.index === 'undefined') subTask.index = taskCount;
@@ -393,7 +545,7 @@ function subWorker(config){
                 taskCount,
                 startTimestamp
             };
-            runTask(config, subTask, subTaskLog).then((tasklog)=>{
+            runTask(config, subTask, subTaskLog, store).then((tasklog)=>{
                 if(tasklog.retried) return;//重试任务已回 new 等重跑, 未完成, 不打 [Task Finished]
                 let {cost, startTimestamp, endTimestamp} = tasklog;
 
@@ -425,13 +577,13 @@ function serializeError(e){
     };
     return serialized;
 };
-function runTask(config, subTask, subTaskLog){
+function runTask(config, subTask, subTaskLog, store){
     return new Promise((resolve)=>{
         let resultPromise = Promise.resolve({});
         let resultData;
         let errors = [];
         helper.load(config);
-        TaskMgr.saveTaskLog(subTask.subid, subTaskLog);
+        store.saveTaskLog(subTask.subid, subTaskLog);
         try{
             resultPromise = USER_CONFIG.processTask(subTask, helper);//rewrite this method in subclass
         }catch(e){
@@ -456,21 +608,23 @@ function runTask(config, subTask, subTaskLog){
             if(hasError)runLog.warn('[Task Error]', subTask.subid, errors);
             let retried = false;
             if(hasError && typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0){
-                let retryCount = TaskMgr.incrementTaskRetry(subTask.subid);
+                //重试决策在 worker(file 档如此, memory 档同构勿收归 master); await 两档通用:
+                //file 档 incrementTaskRetry 是同步值, memory 档是 needAck 回执 Promise——worker 侧唯一 async 化点
+                let retryCount = await store.incrementTaskRetry(subTask.subid);
                 if(retryCount <= config.maxTaskRetries){
                     runLog.warn(`[Task Retry] "${subTask.subid}" failed, retry (${retryCount}/${config.maxTaskRetries})`);
                     retried = true;
-                    //等 rename 落盘再 resolve/请求新任务: fire-and-forget 的异步 rename 会随 worker 被
-                    //autoClose 杀掉而消亡, 任务文件滞留 running 被静默丢弃(高负载偶发丢任务的根因, 勿回退)
+                    //等 rename 生效再 resolve/请求新任务: file 档等 rename 写盘完成(fire-and-forget 的异步 rename 会随
+                    //worker 被 autoClose 杀掉而消亡), memory 档等价等 master 回执——约定两档继续成立, 勿回退
                     try{
-                        await TaskMgr.mvTaskPromise(subTask.subid, 'running', 'new');
+                        await store.mvTaskPromise(subTask.subid, 'running', 'new');
                     }catch(newPath){
                         runLog.warn(`[Task Retry] "${subTask.subid}" failed to move back to new, left in running (recoverable via resume)`);
                     };
-                    TaskMgr.deleteUserMessage(subTask.subid);//重跑即新一轮: 清掉该任务前序尝试的遗留消息
+                    store.deleteUserMessage(subTask.subid);//重跑即新一轮: 清掉该任务前序尝试的遗留消息
                 };
             };
-            TaskMgr.saveTaskLog(subTask.subid, subTaskLog);
+            store.saveTaskLog(subTask.subid, subTaskLog);
             if(retried){
                 resolve({
                     startTimestamp,
@@ -481,10 +635,10 @@ function runTask(config, subTask, subTaskLog){
                 return;
             };
             if(!hasError){
-                TaskMgr.deleteSavedResult(subTask.subid);//清掉前序尝试可能残留的(超时竞争)错误结果
+                store.deleteSavedResult(subTask.subid);//清掉前序尝试可能残留的(超时竞争)错误结果
             };
-            TaskMgr.saveTaskResult(subTask.subid, hasError, hasError ? errors : resultData);
-            TaskMgr.mvTask(subTask.subid, 'running',(hasError ? 'finished_with_errors':'finished'), (newPath)=>{
+            store.saveTaskResult(subTask.subid, hasError, hasError ? errors : resultData);
+            store.mvTask(subTask.subid, 'running',(hasError ? 'finished_with_errors':'finished'), (newPath)=>{
                 resolve({
                     startTimestamp,
                     endTimestamp,