multi-tasks 3.5.1 → 4.0.0-rc.1

package/workers/Multithread.js

@@ -59,11 +59,11 @@ const start = (config)=>{
         const masterStartTimestamp = new Date()*1;//本轮 master 启动时间(resume 不追溯首轮)
         let workerCrashCount = 0;//累计崩溃补员次数, 进 onFinish report
 
-        let memoryStore = null;//memory 档: 任务状态/结果全在 master 内存(file 档恒为 null, 走 TaskMgr 磁盘链路)
-        let getStats = ()=> memoryStore ? memoryStore.stats() : TaskMgr.getFinalStats();
-        let getRemaining = ()=> memoryStore ? memoryStore.getRemaining() : TaskMgr.getTasks('new').length;
+        let taskStore = null;//memory/aof 档: 任务状态/结果在 master 内存(file 档恒为 null, 走 TaskMgr 磁盘链路)
+        let getStats = ()=> taskStore ? taskStore.stats() : TaskMgr.getFinalStats();
+        let getRemaining = ()=> taskStore ? taskStore.getRemaining() : TaskMgr.getTasks('new').length;
 
-        //memory 档 worker 存储操作的应用点: 只在 __storageOp 分支的 queue.enqueue 里调(与派发同一串行通道,
+        //memory/aof 档 worker 存储操作的应用点: 只在 __storageOp 分支的 queue.enqueue 里调(与派发同一串行通道,
         //同 worker 的操作消息排在其后续领新请求之前, pop 时 Map 必已含该 worker 此前创建的全部任务)
         let applyStorageOp = (worker, message)=>{
             let op = message.op;
@@ -76,19 +76,19 @@ const start = (config)=>{
             try{
                 switch(op){
                     case 'createTask':
-                        memoryStore.add(args[0]);
+                        taskStore.add(args[0]);
                         break;
-                    case 'saveLog': memoryStore.saveLog(args[0], args[1]); break;
-                    case 'saveResult': memoryStore.saveResult(args[0], args[1], args[2]); break;
-                    case 'deleteResult': memoryStore.deleteResult(args[0]); break;
+                    case 'saveLog': taskStore.saveLog(args[0], args[1]); break;
+                    case 'saveResult': taskStore.saveResult(args[0], args[1], args[2]); break;
+                    case 'deleteResult': taskStore.deleteResult(args[0]); break;
                     case 'mvTask': {
-                        let succ = memoryStore.move(args[0], args[1], args[2]);
+                        let succ = taskStore.move(args[0], args[1], args[2]);
                         ackValue = {newPath: `${args[2]}/${args[0]}.task.json`, succ};
                         break;
                     };
-                    case 'incrementRetry': ackValue = memoryStore.incrementRetry(args[0]); break;
-                    case 'saveUserMessage': memoryStore.addMessage(args[0]); break;
-                    case 'deleteUserMessage': memoryStore.clearMessages(args[0]); break;
+                    case 'incrementRetry': ackValue = taskStore.incrementRetry(args[0]); break;
+                    case 'saveUserMessage': taskStore.addMessage(args[0]); break;
+                    case 'deleteUserMessage': taskStore.clearMessages(args[0]); break;
                     default:
                         throw new Error(`unknown storage op "${op}"`);
                 };
@@ -129,46 +129,54 @@ const start = (config)=>{
                     process.exit(0);
                 }, 10);
             };
-            if(memoryStore){
-                //memory 档: 收尾排进派发同一串行队列(append-and-return, 绝不阻塞等队列——本函数在
+            if(taskStore){
+                //memory/aof 档: 收尾排进派发同一串行队列(append-and-return, 绝不阻塞等队列——本函数在
                 //shouldTerminate 路径就是在队列任务内被调用的, 阻塞 await 队列即死锁): 已到达未应用的
-                //worker 存储操作先于收尾计数应用; 此后排队的已死 worker 操作按 terminate 语义丢弃
+                //worker 存储操作先于收尾计数应用; 此后排队的已死 worker 操作按 terminate 语义丢弃。
+                //aof 档先 await 最终 checkpoint 再写计数器行: await 的只是本 enqueue 自身 promise
+                //(此前项已随队列推进完毕), 不构成「等队列 drain」。
                 queue.enqueue(async ()=>{
+                    if(typeof taskStore.checkpoint === 'function'){
+                        await taskStore.checkpoint();
+                    };
+                    if(typeof taskStore.flushResults === 'function'){
+                        await taskStore.flushResults();//aof 档: 内存结果全量物化到 results/ 完再写计数器行
+                    };
                     writeCounterAndExit();
                 });
                 return;
             };
             writeCounterAndExit();
         };
-        let finalizeTimeoutTaskInMemory = (subTask, cost)=>{
-            //memory 档: 与 file 档同语义同顺序(先结果后迁移); 只在 queue.enqueue 里调(与 worker 存储操作
+        let finalizeTimeoutTaskInStore = (subTask, cost)=>{
+            //memory/aof 档: 与 file 档同语义同顺序(先结果后迁移); 只在 queue.enqueue 里调(与 worker 存储操作
             //同一串行通道): 该 worker 已到达未应用的 saveResult 等操作必先于本判定应用, 已交卷任务不会被误杀
             let subid = subTask.subid;
             try{
-                let resultType = memoryStore.getSavedResultType(subid);
+                let resultType = taskStore.getSavedResultType(subid);
                 if(resultType === 'succ'){//超时边界: worker 已成功交卷, 视为已完成
-                    memoryStore.move(subid, 'running', 'finished');
+                    taskStore.move(subid, 'running', 'finished');
                 }else if(resultType === 'error'){//超时边界: worker 已自行记错误
-                    memoryStore.move(subid, 'running', 'finished_with_errors');
+                    taskStore.move(subid, 'running', 'finished_with_errors');
                 }else{
                     if(typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0){
-                        let retryCount = memoryStore.incrementRetry(subid);
+                        let retryCount = taskStore.incrementRetry(subid);
                         if(retryCount <= config.maxTaskRetries){
                             runLog.warn(`[Master]: task "${subid}" timeout, retry (${retryCount}/${config.maxTaskRetries})`);
-                            memoryStore.move(subid, 'running', 'new');
+                            taskStore.move(subid, 'running', 'new');
                             return;
                         };
                     };
-                    memoryStore.saveResult(subid, true, [{type: 'timeout', timeout: config.taskTimeout, cost}]);
-                    memoryStore.move(subid, 'running', 'finished_with_errors');
+                    taskStore.saveResult(subid, true, [{type: 'timeout', timeout: config.taskTimeout, cost}]);
+                    taskStore.move(subid, 'running', 'finished_with_errors');
                 }
             }catch(e){
                 runLog.warn('[Master]: failed to finalize timeout task', subid, e);
             }
         };
         let finalizeTimeoutTask = (subTask, cost)=>{
-            if(memoryStore){
-                queue.enqueue(async ()=>{ finalizeTimeoutTaskInMemory(subTask, cost); });
+            if(taskStore){
+                queue.enqueue(async ()=>{ finalizeTimeoutTaskInStore(subTask, cost); });
                 return;
             };
             let subid = subTask.subid;
@@ -204,19 +212,19 @@ const start = (config)=>{
             worker.kill();//不 disconnect: exit 事件会按崩溃补员
             finalizeTimeoutTask(subTask, cost);
         };
-        let handleCrashedTaskInMemory = (subTask)=>{
-            //memory 档: 与 file 档同语义(重试移回 new / 超限记 worker_crash 归档), 操作走内存存储;
+        let handleCrashedTaskInStore = (subTask)=>{
+            //memory/aof 档: 与 file 档同语义(重试移回 new / 超限记 worker_crash 归档), 操作走 master 侧 store;
             //只在 queue.enqueue 里调: 该 worker 崩溃前已到达的存储操作必先应用, 再判定重试/归档
             let subid = subTask.subid;
             try{
-                let retryCount = memoryStore.incrementRetry(subid);
+                let retryCount = taskStore.incrementRetry(subid);
                 if(retryCount <= config.maxTaskRetries){
                     runLog.warn(`[Master]: worker crashed, retry task "${subid}" (${retryCount}/${config.maxTaskRetries})`);
-                    memoryStore.move(subid, 'running', 'new');
+                    taskStore.move(subid, 'running', 'new');
                 }else{
                     runLog.warn(`[Master]: task "${subid}" crashed ${retryCount} times, give up`);
-                    memoryStore.saveResult(subid, true, [{type: 'worker_crash', retryCount: config.maxTaskRetries}]);
-                    memoryStore.move(subid, 'running', 'finished_with_errors');
+                    taskStore.saveResult(subid, true, [{type: 'worker_crash', retryCount: config.maxTaskRetries}]);
+                    taskStore.move(subid, 'running', 'finished_with_errors');
                 };
             }catch(e){
                 runLog.warn('[Master]: failed to handle crashed task', subid, e);
@@ -224,8 +232,8 @@ const start = (config)=>{
         };
         let handleCrashedTask = (subTask)=>{
             if(!(typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0)) return;
-            if(memoryStore){
-                queue.enqueue(async ()=>{ handleCrashedTaskInMemory(subTask); });
+            if(taskStore){
+                queue.enqueue(async ()=>{ handleCrashedTaskInStore(subTask); });
                 return;
             };
             let subid = subTask.subid;
@@ -289,8 +297,15 @@ const start = (config)=>{
                     }
                     return;
                 }
-                if(memoryStore && message && message.__storageOp){//memory 档 worker 存储操作: 与派发同一串行通道逐条应用, 禁止捎带合批
-                    queue.enqueue(async ()=>{ applyStorageOp(worker, message); });
+                if(taskStore && message && message.__storageOp){//memory/aof 档 worker 存储操作: 与派发同一串行通道逐条应用, 禁止捎带合批
+                    queue.enqueue(async ()=>{
+                        applyStorageOp(worker, message);
+                        //aof 档周期 checkpoint: 与 op 应用同一串行任务内触发并 await, snapshot 才对应 aof
+                        //前缀一致态(不 await 则后续 op 先进队列应用, 截断时其 aof 行被抹掉); memory 档无此方法, 跳过
+                        if(typeof taskStore.maybeCheckpoint === 'function'){
+                            await taskStore.maybeCheckpoint();
+                        };
+                    });
                     return;
                 }
                 if(message === MSG_REQUEST_NEW_TASK){
@@ -298,10 +313,10 @@ const start = (config)=>{
                     progressCtl.taskCompleted(worker.id);//worker 行回到 idle; 完成数按盘上结果目录统计, 不在此计数
                     queue.enqueue(async () => {
                         let subTask = null;
-                        if(memoryStore){
-                            //memory 档: 任务全在内存队列, pop 即出队; 无需磁盘重读兜底(动态任务经 __storageOp
-                            //同一串行通道进来, pop 时必然已可见)
-                            subTask = memoryStore.pop();
+                        if(taskStore){
+                            //memory/aof 档: 任务全在 master 侧 store 队列, pop 即出队(aof 档 pop 落 aof 行);
+                            //无需磁盘重读兜底(动态任务经 __storageOp 同一串行通道进来, pop 时必然已可见)
+                            subTask = taskStore.pop();
                         }else{
                             //use queue to make sure no I/O conflict; update/pop/判空须在同一个串行任务里原子完成
                             await TaskMgr.updateNewTasks();
@@ -375,7 +390,7 @@ const start = (config)=>{
         let initPromise = config.__resume ? asMaster.resume(config) : asMaster.init(config);
         initPromise.then((inited)=>{
             config = inited.config;
-            memoryStore = inited.store || null;
+            taskStore = inited.store || null;
             //一次性启动块(走 console, 在劫持前上屏): 标题框横幅 + 平铺关键字段, 模式由 __ 标记推导;
             //init 成功后所有信息齐备(new tasks 数/masterTaskId/__sysLogFile), 一次打完
             let startupInfo = {
@@ -386,6 +401,8 @@ const start = (config)=>{
                 numCPUs,
                 pid: process.pid,
                 newTasks: getRemaining(),
+                taskStorage: config.task_storage,
+                storageDefaultApplied: !!config.__storageDefaultApplied,
                 sysLogFile: config.__sysLogFile,
                 startedAt: randomsUtil.getDateTimeTxt(),
             };
@@ -452,12 +469,12 @@ const start = (config)=>{
                         workers: {count: numOfWorkers, crashed: workerCrashCount},
                     };
                     let onFinishHelper;
-                    if(config.task_storage === 'memory'){
-                        //memory 档: 结果只在内存(盘上 results/ 恒空), 遍历接口委托 memoryStore(与 TaskMgr 同签名同口径)
+                    if(taskStore){
+                        //memory/aof 档: 结果在 master 侧 store(aof 档收尾物化 results/ 文件, 内容一致), 遍历接口委托 taskStore(与 TaskMgr 同签名同口径)
                         onFinishHelper = {
-                            foreachSuccResult: (cb) => memoryStore.forEachSuccResult(cb),
-                            foreachErrorResult: (cb) => memoryStore.forEachErrorResult(cb),
-                            foreachResult: (cb) => memoryStore.forEachResult(cb),
+                            foreachSuccResult: (cb) => taskStore.forEachSuccResult(cb),
+                            foreachErrorResult: (cb) => taskStore.forEachErrorResult(cb),
+                            foreachResult: (cb) => taskStore.forEachResult(cb),
                         };
                     }else{
                         onFinishHelper = {
@@ -466,7 +483,15 @@ const start = (config)=>{
                             foreachResult: (cb) => TaskMgr.forEachResult(cb),
                         };
                     }
-                    USER_CONFIG.onFinish(report, onFinishHelper);
+                    //onFinish 是用户回调: 同步抛错/异步 reject 都只告警, 不得中断收尾——
+                    //memory/aof 档本函数跑在串行 queue 里, 抛错会被 queue 吞掉, 计数器行与 process.exit 永远不到, master 挂死
+                    try{
+                        Promise.resolve(USER_CONFIG.onFinish(report, onFinishHelper)).catch((e)=>{
+                            runLog.warn('[Master]: onFinish callback rejected, continue finishing', e);
+                        });
+                    }catch(e){
+                        runLog.warn('[Master]: onFinish callback threw, continue finishing', e);
+                    };
                     delete USER_CONFIG.onFinish;
                 }
                 //计数器行先进 run.log(progressBar 未接管终端时 echo 上屏, 接管时由 progressCtl.finish() 打在屏幕);
@@ -479,10 +504,16 @@ const start = (config)=>{
                     process.exit(0);
                 }, 10);
                 };
-                if(memoryStore){
-                    //memory 档: 同 terminateAllAndExit 口径, 收尾排进派发串行队列, 队列里已到达的存储操作
-                    //先应用再统计; file 档维持原同步收尾一字不动
+                if(taskStore){
+                    //memory/aof 档: 同 terminateAllAndExit 口径, 收尾排进派发串行队列, 队列里已到达的存储操作
+                    //先应用再统计; aof 档先 await 最终 checkpoint(snapshot+截断落盘)再收尾; file 档维持原同步收尾一字不动
                     queue.enqueue(async ()=>{
+                        if(typeof taskStore.checkpoint === 'function'){
+                            await taskStore.checkpoint();
+                        };
+                        if(typeof taskStore.flushResults === 'function'){
+                            await taskStore.flushResults();//aof 档: 内存结果全量物化到 results/ 完再收尾
+                        };
                         finishAll();
                     });
                 }else{
@@ -528,9 +559,9 @@ function subWorker(config){
         }else
         if(message.subTask){//new task
             let {config, subTask, startTimestamp, taskCount} = message;
-            //memory 档 worker 不碰任务目录: 存储后端换 IpcTaskStore(逐条 {__storageOp} 发 master),
+            //memory/aof 档 worker 不碰任务目录: 存储后端换 IpcTaskStore(逐条 {__storageOp} 发 master),
             //操作序列与 file 档逐条同构; file 档后端即 TaskMgr(磁盘, 现状不变)
-            let store = config.task_storage === 'memory' ? IpcTaskStore : TaskMgr;
+            let store = config.task_storage === 'file' ? TaskMgr : IpcTaskStore;
             store.load(config);
             //用户任务自带的 index/taskCount 保留原值(重试/续跑后任务身份不变),
             //未提供时才填全局派发计数器(v1 兼容)
@@ -610,7 +641,15 @@ function runTask(config, subTask, subTaskLog, store){
             if(hasError && typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0){
                 //重试决策在 worker(file 档如此, memory 档同构勿收归 master); await 两档通用:
                 //file 档 incrementTaskRetry 是同步值, memory 档是 needAck 回执 Promise——worker 侧唯一 async 化点
-                let retryCount = await store.incrementTaskRetry(subTask.subid);
+                //失败保护: 回执 {ok:false} reject / file 档读文件 ENOENT throw 时, 不包则 finally 链尾部无人接
+                //(worker unhandled rejection 崩进程, resolve 永远不到)——按「放弃本次重试, 落错误结果」继续, resolve 必达
+                let retryCount;
+                try{
+                    retryCount = await store.incrementTaskRetry(subTask.subid);
+                }catch(e){
+                    runLog.warn(`[Task Retry] "${subTask.subid}" failed to increment retry count, give up retrying`, e);
+                    retryCount = config.maxTaskRetries + 1;
+                };
                 if(retryCount <= config.maxTaskRetries){
                     runLog.warn(`[Task Retry] "${subTask.subid}" failed, retry (${retryCount}/${config.maxTaskRetries})`);
                     retried = true;

package/workers/TaskMgr.js

@@ -15,6 +15,22 @@ let allSubTasks = {
 
 let main_config;
 
+//initialTasks 播种批量写的并发上限(createNewTasks 专用)
+const SEED_WRITE_CONCURRENCY = 128;
+
+//建任务文件的准备(赋 subid + 序列化): createNewTask(单条同步写)与 createNewTasks(批量并发写)共用
+const _prepareNewTask = (task)=>{
+    const {masterTaskId, taskFolderNew} = main_config;
+    const subid = randomsUtil.get36Id();
+
+    task.masterTaskId = masterTaskId;
+    task.subid = `${masterTaskId}--${subid}`;
+    return {
+        fpath: pathutil.resolve(taskFolderNew, `./${task.subid}.task.json`),
+        content: JSON.stringify(task),
+    };
+};
+
 //.sys/user_messages 消息文件命名规范(严谨, 解析依赖以下两条不变量):
 //  文件名 = <type>.<subid>.json; type 是库内机器常量, 只允许 [a-z0-9_](必须不含点), 写入时有同样校验;
 //  subid 是首个点之后的全部(可含点, 非空)。type 无点 ⇒ 按首个点切分无歧义
@@ -105,13 +121,42 @@ const TaskMgr = {
         return main_config;
     },
     createNewTask:(task)=>{
-        const {masterTaskId, taskFolderNew} = main_config;
-        const subid = randomsUtil.get36Id();
-
-        task.masterTaskId = masterTaskId;
-        task.subid = `${masterTaskId}--${subid}`;
-        let taskfilepath = pathutil.resolve(taskFolderNew, `./${task.subid}.task.json`);
-        fs.writeFileSync(taskfilepath, JSON.stringify(task));
+        let {fpath, content} = _prepareNewTask(task);
+        fs.writeFileSync(fpath, content);
+    },
+    //批量建任务(initialTasks 播种专用): 与 createNewTask 同规则赋 subid, 有界并发异步写,
+    //全部写完才 resolve(任一写失败即 reject, 走 init 异步失败 FATAL exit(1) 链路);
+    //worker 动态建任务仍走 createNewTask 单条同步立即写(master 靠强制重读磁盘发现, 不能异步/缓冲)
+    createNewTasks:(tasks)=>{
+        let items = tasks.map((task)=>_prepareNewTask(task));
+        return new Promise((resolve, reject)=>{
+            let cursor = 0;
+            let active = 0;
+            let failed = false;
+            const launch = ()=>{
+                if(failed) return;
+                if(cursor >= items.length){
+                    if(active === 0) resolve();
+                    return;
+                };
+                while(active < SEED_WRITE_CONCURRENCY && cursor < items.length){
+                    let item = items[cursor++];
+                    active++;
+                    fs.writeFile(item.fpath, item.content, (err)=>{
+                        active--;
+                        if(err){
+                            if(!failed){
+                                failed = true;
+                                reject(err);
+                            };
+                            return;
+                        };
+                        launch();
+                    });
+                };
+            };
+            launch();
+        });
     },
     loadAllTasks:(loadtype)=>{
         const {progressRootFolder} = main_config;

package/workers/asMaster.js

@@ -1,110 +1,139 @@
-var fs = require('fs');
-var pathUtil = require('path');
-
-const TaskMgr = require('./TaskMgr');
-const { createMemoryStore } = require('./MemoryStore');
-const randomsUtil = require('../utils/randoms');
-const { resolveNumberOfWorkers } = require('../utils/resolveWorkers');
-
-const resume = (config)=>{
-    config.__resume = true;
-    let {taskRootFolder, taskId} = config;
-    let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
-
-    //load from config file: 旧存档作底(兼作任务目录合法性校验, 不存在抛 ENOENT),
-    //本次调用传入的配置覆盖同名字段——存档只补未传入的字段
-    let task_config = JSON.parse(fs.readFileSync(pathUtil.resolve(taskFolder, 'task_config.json')));
-    Object.keys(task_config).forEach((key)=>{
-        if(key.startsWith('__')) delete task_config[key];//读侧剔除一次性标记, 兼容 <=3.1.3 旧存档
-    });
-    config = {
-        ...task_config,
-        ...config
-    };
-    config.masterTaskId = taskId;
-
-    TaskMgr.load(config);
-    config = TaskMgr.initFolders(config);
-
-    return new Promise((resolve, reject)=>{
-
-        TaskMgr.getTaskFiles('running', (taskfiles)=>{
-            TaskMgr.getTaskFiles('finished_with_errors', (errfiles)=>{
-                let promises = [];
-                taskfiles.forEach((fname)=>{
-                    let subid = fname.replace(/\.task\.json$/, '');
-                    promises.push(TaskMgr.mvTaskPromise(subid, 'running', 'new'));
-                });
-                if(config.__retryFails){
-                    errfiles.forEach((fname)=>{
-                        let subid = fname.replace(/\.task\.json$/, '');
-                        promises.push(TaskMgr.mvTaskPromise(subid, 'finished_with_errors', 'new').then(()=>{
-                            TaskMgr.resetTaskRetry(subid);//retry_fails 重跑失败任务时重置重试计数(resume 不重置)
-                            TaskMgr.deleteUserMessage(subid);//重跑即新一轮: 清掉该任务在 .sys/user_messages 的遗留消息
-                        }));
-                    });
-                };
-                Promise.all(promises).then(()=>{
-                    TaskMgr.loadAllTasks().then((types)=>{
-                        console.log('******resume multi-thread')
-                        setTimeout(()=>{
-                            resolve({config, store: null});//memory 档 resume 在 index.js 已被拦截 throw, 到这里的都是 file 档
-                        }, 1000);
-                    });
-
-                }).catch((e)=>{
-                    reject(e);
-                });
-            });
-        });
-    });
-
-
-};
-
-const init = (config)=>{
-    if(!config) config = {}; 
-    if(!config.initialTasks){ throw new Error('Please provide initialTasks data!');  };
-    if(!config.processTask){ throw new Error('Please provide processTask function!'); };
-
-    config.numberOfWorkers = resolveNumberOfWorkers(config.numberOfWorkers);//支持百分比字符串, 非法值在此抛错(fail fast, 不建任务目录)
-
-    let {taskRootFolder, taskName} = config;
-
-    if(taskName && !taskName.match(/^[a-zA-Z0-9_-]+$/)){ throw new Error('Error: taskName can only contain letters and numbers'); }
-    if(!taskName){ taskName = 'multitasks'; }
-
-    let {round, pow, random} = Math;
-    let dateTimeText = randomsUtil.getDateTimeTxt();
-    let randomNum = round((random() * pow(10,8)),8).toString(36);
-    let masterTaskId = config.taskId ? config.taskId : `${taskName}${dateTimeText}--${randomNum}`;
-
-    config.taskRootFolder = taskRootFolder;
-    config.masterTaskId = masterTaskId;
-
-
-    TaskMgr.load(config);
-    config = TaskMgr.initFolders(config);
-
-    if(config.task_storage === 'memory'){
-        //memory 档: 不建任务文件(目录与 task_config.json 照建照写), 播种进 master 内存存储
-        const store = createMemoryStore(config);
-        store.addMany(config.initialTasks);
-        return Promise.resolve({config, store});
-    };
-
-    config.initialTasks.forEach((task)=>{
-        TaskMgr.createNewTask(task);
-    });
-
-    //to reload is for to reStart a collapsed task
-    return new Promise((resolve)=>{
-        TaskMgr.loadNewTasks().then(()=>{
-            resolve({config, store: null});
-        });
-    });
-};
-module.exports = {
-    init,
-    resume
+var fs = require('fs');
+var pathUtil = require('path');
+
+const TaskMgr = require('./TaskMgr');
+const { createMemoryStore } = require('./MemoryStore');
+const { createAofStore, loadFromDisk } = require('./AofStore');
+const { attachResultAppender } = require('./resultFileAppender');
+const randomsUtil = require('../utils/randoms');
+const { resolveNumberOfWorkers } = require('../utils/resolveWorkers');
+
+const resume = (config)=>{
+    config.__resume = true;
+    let {taskRootFolder, taskId} = config;
+    let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
+
+    //load from config file: 旧存档作底(兼作任务目录合法性校验, 不存在抛 ENOENT),
+    //本次调用传入的配置覆盖同名字段——存档只补未传入的字段
+    let task_config = JSON.parse(fs.readFileSync(pathUtil.resolve(taskFolder, 'task_config.json')));
+    Object.keys(task_config).forEach((key)=>{
+        if(key.startsWith('__')) delete task_config[key];//读侧剔除一次性标记, 兼容 <=3.1.3 旧存档
+    });
+    config = {
+        ...task_config,
+        ...config
+    };
+    config.masterTaskId = taskId;
+
+    TaskMgr.load(config);
+    config = TaskMgr.initFolders(config);
+
+    if(config.task_storage === 'aof'){
+        //aof 档 resume: snapshot + aof 增量重放重建内存态, 语义与 file 档逐项同构:
+        //running 全量回 new(resume 不重置重试计数); retry_fails 再把 finished_with_errors 回 new
+        //并重置计数/清消息。回退动作本身记 aof 行, 保持「重放=真相」。
+        const store = loadFromDisk(config);
+        store.getTasks('running').forEach((task)=>{
+            store.move(task.subid, 'running', 'new');
+        });
+        if(config.__retryFails){
+            store.getTasks('finished_with_errors').forEach((task)=>{
+                store.move(task.subid, 'finished_with_errors', 'new');
+                store.resetRetry(task.subid);
+                store.clearMessages(task.subid);
+            });
+        };
+        console.log('******resume multi-thread');
+        return Promise.resolve({config, store});
+    };
+
+    return new Promise((resolve, reject)=>{
+
+        TaskMgr.getTaskFiles('running', (taskfiles)=>{
+            TaskMgr.getTaskFiles('finished_with_errors', (errfiles)=>{
+                let promises = [];
+                taskfiles.forEach((fname)=>{
+                    let subid = fname.replace(/\.task\.json$/, '');
+                    promises.push(TaskMgr.mvTaskPromise(subid, 'running', 'new'));
+                });
+                if(config.__retryFails){
+                    errfiles.forEach((fname)=>{
+                        let subid = fname.replace(/\.task\.json$/, '');
+                        promises.push(TaskMgr.mvTaskPromise(subid, 'finished_with_errors', 'new').then(()=>{
+                            TaskMgr.resetTaskRetry(subid);//retry_fails 重跑失败任务时重置重试计数(resume 不重置)
+                            TaskMgr.deleteUserMessage(subid);//重跑即新一轮: 清掉该任务在 .sys/user_messages 的遗留消息
+                        }));
+                    });
+                };
+                Promise.all(promises).then(()=>{
+                    TaskMgr.loadAllTasks().then((types)=>{
+                        console.log('******resume multi-thread')
+                        setTimeout(()=>{
+                            resolve({config, store: null});//memory 档被拦截, aof 档上方已分支返回, 到这里的都是 file 档
+                        }, 1000);
+                    });
+
+                }).catch((e)=>{
+                    reject(e);
+                });
+            });
+        });
+    });
+
+
+};
+
+const init = (config)=>{
+    if(!config) config = {}; 
+    if(!config.initialTasks){ throw new Error('Please provide initialTasks data!');  };
+    if(!config.processTask){ throw new Error('Please provide processTask function!'); };
+
+    config.numberOfWorkers = resolveNumberOfWorkers(config.numberOfWorkers);//支持百分比字符串, 非法值在此抛错(fail fast, 不建任务目录)
+
+    let {taskRootFolder, taskName} = config;
+
+    if(taskName && !taskName.match(/^[a-zA-Z0-9_-]+$/)){ throw new Error('Error: taskName can only contain letters and numbers'); }
+    if(!taskName){ taskName = 'multitasks'; }
+
+    let {round, pow, random} = Math;
+    let dateTimeText = randomsUtil.getDateTimeTxt();
+    let randomNum = round((random() * pow(10,8)),8).toString(36);
+    let masterTaskId = config.taskId ? config.taskId : `${taskName}${dateTimeText}--${randomNum}`;
+
+    config.taskRootFolder = taskRootFolder;
+    config.masterTaskId = masterTaskId;
+
+
+    TaskMgr.load(config);
+    config = TaskMgr.initFolders(config);
+
+    if(config.task_storage === 'memory'){
+        //memory 档: 不建任务文件(目录与 task_config.json 照建照写), 播种进 master 内存存储
+        const store = createMemoryStore(config);
+        store.addMany(config.initialTasks);
+        if(config.memory_append_result_files === true){
+            //内部开关 memory_append_result_files(不对外文档化): memory 档也追加结果聚合文件(只追加不重写)
+            attachResultAppender(store, config.taskFolder);
+        };
+        return Promise.resolve({config, store});
+    };
+
+    if(config.task_storage === 'aof'){
+        //aof 档: 与 memory 档同形——不建任务文件, 播种进 AofStore(内存+aof 追加)
+        const store = createAofStore(config);
+        store.addMany(config.initialTasks);
+        return Promise.resolve({config, store});
+    };
+
+    //批量并发写播种(initialTasks 量大时同步逐条 writeFileSync 会长时间阻塞 master 启动)
+    return TaskMgr.createNewTasks(config.initialTasks).then(()=>{
+        //to reload is for to reStart a collapsed task
+        return TaskMgr.loadNewTasks().then(()=>{
+            return {config, store: null};
+        });
+    });
+};
+module.exports = {
+    init,
+    resume
 };