multi-tasks 3.2.2 → 3.3.0

package/workers/TaskMgr.js

@@ -1,294 +1,381 @@
-const fs = require('fs');
-const pathutil = require('path');
-const makeDir = require('../utils/makedir');
-const randomsUtil = require('../utils/randoms');
-const { serializeConfig } = require('../utils/serializeFns');
-const { forEachResult } = require('../utils/foreachResults');
-const runLog = require('../utils/runLog');
-
-let allSubTasks = {
-    new:[],
-    running:[],
-    finished:[],
-    finished_with_errors:[]
-};
-
-let main_config;
-
-const TaskMgr = {
-    load:(config)=>{
-        main_config = config;
-        return TaskMgr;
-    },
-    initFolders:()=>{
-        let {taskRootFolder, masterTaskId} = main_config;
-        const taskFolder = pathutil.resolve(taskRootFolder, masterTaskId);
-        if(fs.existsSync(taskFolder) && !(main_config.__resume || main_config.__retryFails || main_config.__restart)){
-            let files = fs.readdirSync(taskFolder);
-            if(files.length > 0){
-                console.log('[FATAL] taskRootFolder already exist and not empty, delete it and try again: ', taskFolder);
-                return process.exit(0);
-            }
-        };
-        if(main_config.__restart && fs.existsSync(taskFolder)){
-            TaskMgr.resetTaskFolders(taskFolder);//restart: 整目录重置, 随后由下方 makeDir 重建
-        };
-        const progressRootFolder = pathutil.resolve(taskFolder, './progress');
-        const resultsRootFolder = pathutil.resolve(taskFolder, './results');
-        const logsRootFolder = pathutil.resolve(taskFolder, './logs');
-        const sysRootFolder = pathutil.resolve(taskFolder, './.sys');//库自身运行日志(run.log)等系统文件, 与任务数据/结果隔离
-
-        const taskFolderNew = pathutil.resolve(progressRootFolder, './new');
-        const taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
-        const taskFolderFinished = pathutil.resolve(progressRootFolder, './finished');
-        const taskFolderFailed= pathutil.resolve(progressRootFolder, './finished_with_errors');
-
-        const taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
-        const taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
-        makeDir.sync(logsRootFolder);
-        makeDir.sync(sysRootFolder);
-        makeDir.sync(taskFolderNew);
-        makeDir.sync(taskFolderRunning);
-        makeDir.sync(taskFolderFinished);
-        makeDir.sync(taskFolderFailed);
-        makeDir.sync(taskFolderResult);
-        makeDir.sync(taskFolderResultFailed);
-
-        main_config = {
-            ...main_config,
-            taskFolder,
-            progressRootFolder,
-            resultsRootFolder,
-            logsRootFolder,
-            sysRootFolder,
-            //run.log 路径唯一兜底计算点: 入口(workers/index.js)没算过时在这里补(__ 前缀, 不进快照)
-            __sysLogFile: main_config.__sysLogFile || pathutil.resolve(sysRootFolder, 'run.log'),
-
-            taskFolderNew,
-            taskFolderRunning,
-            taskFolderFinished,
-            taskFolderFailed,
-            taskFolderResult,
-            taskFolderResultFailed,
-        };
-
-        //for resuming an interrupted task
-        let main_config2 = JSON.parse(JSON.stringify(serializeConfig(main_config)));//函数型配置键先序列化为源码字符串, 供 resume 从快照复活
-        delete main_config2.tasks;//will load from folder 'new', so no need the early tasks data
-        Object.keys(main_config2).forEach((key)=>{
-            if(key.startsWith('__')) delete main_config2[key];//__-prefixed keys are one-shot runtime markers, never persisted
-        });
-        fs.writeFileSync(pathutil.resolve(taskFolder, 'task_config.json'), JSON.stringify(main_config2));
-        return main_config;
-    },
-    createNewTask:(task)=>{
-        const {masterTaskId, taskFolderNew} = main_config;
-        const subid = randomsUtil.get36Id();
-
-        task.masterTaskId = masterTaskId;
-        task.subid = `${masterTaskId}--${subid}`;
-        let taskfilepath = pathutil.resolve(taskFolderNew, `./${task.subid}.task.json`);
-        fs.writeFileSync(taskfilepath, JSON.stringify(task));
-    },
-    loadAllTasks:(loadtype)=>{
-        const {progressRootFolder} = main_config;
-        let load = (type)=>{
-            allSubTasks[type] = [];
-            return new Promise((resolve)=>{
-                    let folder = pathutil.resolve(progressRootFolder, type);
-                    makeDir.getDirectFiles(folder, (taskFiles)=>{
-                        taskFiles.forEach((filename)=>{
-                            let fpath = pathutil.resolve(folder, filename);
-                            let fcontent;
-                            try{
-                                fcontent = fs.readFileSync(fpath, 'utf-8');
-                            }catch(e){
-                                runLog.warn('[WARN]', `file not found`, fpath)
-                            }
-                            if(fcontent){
-                                try{
-                                    let task = JSON.parse(fcontent);
-                                    allSubTasks[type].push(task);
-                                }catch(e){
-                                    runLog.warn('[WARN]', 'failed to parse task file, skipped', fpath);
-                                }
-                            }
-                        });
-                        resolve(allSubTasks[type]);
-                    });                
-            });
-        };
-        let promises = [];
-        if(typeof loadtype === 'undefined'){
-            for(let type in allSubTasks){
-                promises.push(load(type));
-            }
-        }else{
-            promises.push(load(loadtype));
-        }
-        return Promise.all(promises);
-    },
-    loadNewTasks:()=>{
-        return TaskMgr.loadAllTasks('new');
-    },
-    updateNewTasks:()=>{
-        let p;
-        if(allSubTasks.new.length === 0){
-            p = TaskMgr.loadNewTasks();//load again
-        }else{
-            p = Promise.resolve();
-        }
-        return p;
-    },
-    popNewTask:()=>{
-        return new Promise((resolve)=>{
-            let newtask = allSubTasks.new.shift();
-            if(newtask){
-                TaskMgr.mvTask(newtask.subid, 'new', 'running', (newPath, succ)=>{ 
-                    if(succ){
-                        resolve(newtask);
-                    }else{
-                        allSubTasks.new.unshift(newtask);
-                        resolve(null);
-                    }
-                });
-            }else{
-                resolve(null);
-            }
-        });
-    },
-    getTask:(type, id)=>{
-        let result;
-        let arr = allSubTasks[type];
-        if(!arr){
-            return result;
-        }
-        for(let i=0;i<arr.length;i++){
-            let task = arr[i];
-            if(task.subid === id){
-                result = task;
-                break;
-            } 
-        }
-        return result;
-    },
-    getTasks:(type)=>{
-        return allSubTasks[type];
-    },
-    getTaskFiles:(type, callback)=>{
-        let rootdir = main_config.progressRootFolder;
-        let targetFolder = pathutil.resolve(rootdir, `./${type}`);
-        makeDir.getDirectFiles(targetFolder, (files)=>{
-            callback(files);
-        });
-    },
-    getFinalStats:()=>{//master 收尾用: 以盘上终态为准统计成功/失败(resume 跨轮累计也是对的)
-        const {taskFolder, taskFolderFinished, taskFolderFailed} = main_config;
-        const listSubids = (folder)=>{
-            return fs.readdirSync(folder)
-                .filter((fname)=>fname.endsWith('.task.json'))
-                .map((fname)=>fname.replace(/\.task\.json$/, ''));
-        };
-        const failedTasks = listSubids(taskFolderFailed);
-        return {
-            taskFolder,
-            succeeded: listSubids(taskFolderFinished).length,
-            failed: failedTasks.length,
-            failedTasks,
-        };
-    },
-    mvTaskPromise: (subid, from, to)=> {
-        return new Promise((resolve, reject)=>{
-            TaskMgr.mvTask(subid, from, to, (newPath, succ)=>{
-                succ ? resolve(newPath) : reject(newPath);
-            });
-        });
-    },
-    mvTask: (subid, from, to, callback)=> {
-        runLog.write('Move task:', subid, `${from}>${to}`);
-        let {progressRootFolder} = main_config;
-        let oldPath = pathutil.resolve(progressRootFolder, `./${from}/${subid}.task.json`);
-        let newPath = pathutil.resolve(progressRootFolder, `./${to}/${subid}.task.json`);
-        try{
-            fs.rename(oldPath, newPath, (err) => {
-                if (err) {
-                    callback(newPath, false);
-                }else{
-                    callback(newPath, true);
-                }
-            });
-        }catch(e){
-            runLog.warn(e);
-            callback(newPath, false);
-        }
-    },
-    saveTaskResult:(subid, hasError, data)=>{        
-        let {taskFolderResult, taskFolderResultFailed} = main_config;
-        let folder = hasError ? taskFolderResultFailed : taskFolderResult;
-        let fpath = pathutil.resolve(folder, `${subid}.json`);
-        let fcontent;
-        if(typeof data === 'undefined') {
-            fcontent = 'undefined';
-        }else{
-            fcontent = JSON.stringify(data);
-        };
-        fs.writeFileSync(fpath, fcontent);
-    },
-    getSavedResultType:(subid)=>{
-        let {taskFolderResult, taskFolderResultFailed} = main_config;
-        let succPath = pathutil.resolve(taskFolderResult, `${subid}.json`);
-        let errPath = pathutil.resolve(taskFolderResultFailed, `${subid}.json`);
-        if(fs.existsSync(succPath)) return 'succ';
-        if(fs.existsSync(errPath)) return 'error';
-        return null;
-    },
-    saveTaskLog:(subid, log)=>{
-        let {logsRootFolder} = main_config;
-        let folder = logsRootFolder;
-        let fpath = pathutil.resolve(folder, `${subid}.json`);
-        let fcontent;
-        if(typeof log === 'undefined') {
-            fcontent = '';
-        }else{
-            log.subid = subid;
-            fcontent = JSON.stringify(log);
-        };
-        fs.writeFileSync(fpath, fcontent);
-    },
-    incrementTaskRetry:(subid)=>{
-        let {taskFolderRunning} = main_config;
-        let fpath = pathutil.resolve(taskFolderRunning, `./${subid}.task.json`);
-        let task = JSON.parse(fs.readFileSync(fpath, 'utf-8'));
-        task.__retryCount = (task.__retryCount || 0) + 1;
-        fs.writeFileSync(fpath, JSON.stringify(task));
-        return task.__retryCount;
-    },
-    deleteSavedResult:(subid)=>{
-        let {taskFolderResult, taskFolderResultFailed} = main_config;
-        [taskFolderResult, taskFolderResultFailed].forEach((folder)=>{
-            let fpath = pathutil.resolve(folder, `${subid}.json`);
-            if(fs.existsSync(fpath)){
-                fs.unlinkSync(fpath);
-            }
-        });
-    },
-    resetTaskRetry:(subid)=>{
-        let {taskFolderNew} = main_config;
-        let fpath = pathutil.resolve(taskFolderNew, `./${subid}.task.json`);
-        let task = JSON.parse(fs.readFileSync(fpath, 'utf-8'));
-        delete task.__retryCount;
-        fs.writeFileSync(fpath, JSON.stringify(task));
-    },
-    forEachSuccResult:(cb)=>{
-        let {taskFolderResult, taskFolderFinished} = main_config;
-        forEachResult(taskFolderResult, taskFolderFinished, cb);
-    },
-    forEachErrorResult:(cb)=>{
-        let {taskFolderResultFailed, taskFolderFailed} = main_config;
-        forEachResult(taskFolderResultFailed, taskFolderFailed, cb);
-    },
-    resetTaskFolders:(taskFolder)=>{
-        fs.rmSync(taskFolder, {recursive: true, force: true});
-    },
-};
+const fs = require('fs');
+const pathutil = require('path');
+const makeDir = require('../utils/makedir');
+const randomsUtil = require('../utils/randoms');
+const { serializeConfig } = require('../utils/serializeFns');
+const { forEachResult } = require('../utils/foreachResults');
+const runLog = require('../utils/runLog');
+
+let allSubTasks = {
+    new:[],
+    running:[],
+    finished:[],
+    finished_with_errors:[]
+};
+
+let main_config;
+
+//.sys/user_messages 消息文件命名规范(严谨, 解析依赖以下两条不变量):
+//  文件名 = <type>.<subid>.json; type 是库内机器常量, 只允许 [a-z0-9_](必须不含点), 写入时有同样校验;
+//  subid 是首个点之后的全部(可含点, 非空)。type 无点 ⇒ 按首个点切分无歧义
+const MESSAGE_TYPE_PATTERN = /^[a-z0-9_]+$/;
+//解析消息文件名 → subid; 不满足命名规范的文件(脏文件)返回 null, 调用方按无消息处理
+const parseMessageSubid = (fname)=>{
+    if(!fname.endsWith('.json')){
+        return null;
+    };
+    let body = fname.slice(0, -'.json'.length);
+    let dotIndex = body.indexOf('.');
+    if(dotIndex <= 0 || dotIndex === body.length - 1){//无点/点开头/subid 为空, 均不合规
+        return null;
+    };
+    if(!MESSAGE_TYPE_PATTERN.test(body.slice(0, dotIndex))){
+        return null;
+    };
+    return body.slice(dotIndex + 1);
+};
+
+const TaskMgr = {
+    load:(config)=>{
+        main_config = config;
+        return TaskMgr;
+    },
+    initFolders:()=>{
+        let {taskRootFolder, masterTaskId} = main_config;
+        const taskFolder = pathutil.resolve(taskRootFolder, masterTaskId);
+        if(fs.existsSync(taskFolder) && !(main_config.__resume || main_config.__retryFails || main_config.__restart)){
+            let files = fs.readdirSync(taskFolder);
+            if(files.length > 0){
+                console.log('[FATAL] taskRootFolder already exist and not empty, delete it and try again: ', taskFolder);
+                return process.exit(0);
+            }
+        };
+        if(main_config.__restart && fs.existsSync(taskFolder)){
+            TaskMgr.resetTaskFolders(taskFolder);//restart: 整目录重置, 随后由下方 makeDir 重建
+        };
+        const progressRootFolder = pathutil.resolve(taskFolder, './progress');
+        const resultsRootFolder = pathutil.resolve(taskFolder, './results');
+        const logsRootFolder = pathutil.resolve(taskFolder, './logs');
+        const sysRootFolder = pathutil.resolve(taskFolder, './.sys');//库自身运行日志(run.log)等系统文件, 与任务数据/结果隔离
+        const userMessagesFolder = pathutil.resolve(sysRootFolder, './user_messages');//用户可见问题消息(一任务一问题一文件), monitor 工具可读
+
+        const taskFolderNew = pathutil.resolve(progressRootFolder, './new');
+        const taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
+        const taskFolderFinished = pathutil.resolve(progressRootFolder, './finished');
+        const taskFolderFailed= pathutil.resolve(progressRootFolder, './finished_with_errors');
+
+        const taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
+        const taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
+        makeDir.sync(logsRootFolder);
+        makeDir.sync(sysRootFolder);
+        makeDir.sync(userMessagesFolder);
+        makeDir.sync(taskFolderNew);
+        makeDir.sync(taskFolderRunning);
+        makeDir.sync(taskFolderFinished);
+        makeDir.sync(taskFolderFailed);
+        makeDir.sync(taskFolderResult);
+        makeDir.sync(taskFolderResultFailed);
+
+        main_config = {
+            ...main_config,
+            taskFolder,
+            progressRootFolder,
+            resultsRootFolder,
+            logsRootFolder,
+            sysRootFolder,
+            userMessagesFolder,
+            //run.log 路径唯一兜底计算点: 入口(workers/index.js)没算过时在这里补(__ 前缀, 不进快照)
+            __sysLogFile: main_config.__sysLogFile || pathutil.resolve(sysRootFolder, 'run.log'),
+
+            taskFolderNew,
+            taskFolderRunning,
+            taskFolderFinished,
+            taskFolderFailed,
+            taskFolderResult,
+            taskFolderResultFailed,
+        };
+
+        //for resuming an interrupted task
+        let main_config2 = JSON.parse(JSON.stringify(serializeConfig(main_config)));//函数型配置键先序列化为源码字符串, 供 resume 从快照复活
+        delete main_config2.tasks;//will load from folder 'new', so no need the early tasks data
+        Object.keys(main_config2).forEach((key)=>{
+            if(key.startsWith('__')) delete main_config2[key];//__-prefixed keys are one-shot runtime markers, never persisted
+        });
+        fs.writeFileSync(pathutil.resolve(taskFolder, 'task_config.json'), JSON.stringify(main_config2));
+        return main_config;
+    },
+    createNewTask:(task)=>{
+        const {masterTaskId, taskFolderNew} = main_config;
+        const subid = randomsUtil.get36Id();
+
+        task.masterTaskId = masterTaskId;
+        task.subid = `${masterTaskId}--${subid}`;
+        let taskfilepath = pathutil.resolve(taskFolderNew, `./${task.subid}.task.json`);
+        fs.writeFileSync(taskfilepath, JSON.stringify(task));
+    },
+    loadAllTasks:(loadtype)=>{
+        const {progressRootFolder} = main_config;
+        let load = (type)=>{
+            allSubTasks[type] = [];
+            return new Promise((resolve)=>{
+                    let folder = pathutil.resolve(progressRootFolder, type);
+                    makeDir.getDirectFiles(folder, (taskFiles)=>{
+                        taskFiles.forEach((filename)=>{
+                            let fpath = pathutil.resolve(folder, filename);
+                            let fcontent;
+                            try{
+                                fcontent = fs.readFileSync(fpath, 'utf-8');
+                            }catch(e){
+                                runLog.warn('[WARN]', `file not found`, fpath)
+                            }
+                            if(fcontent){
+                                try{
+                                    let task = JSON.parse(fcontent);
+                                    allSubTasks[type].push(task);
+                                }catch(e){
+                                    runLog.warn('[WARN]', 'failed to parse task file, skipped', fpath);
+                                }
+                            }
+                        });
+                        resolve(allSubTasks[type]);
+                    });                
+            });
+        };
+        let promises = [];
+        if(typeof loadtype === 'undefined'){
+            for(let type in allSubTasks){
+                promises.push(load(type));
+            }
+        }else{
+            promises.push(load(loadtype));
+        }
+        return Promise.all(promises);
+    },
+    loadNewTasks:()=>{
+        return TaskMgr.loadAllTasks('new');
+    },
+    updateNewTasks:()=>{
+        let p;
+        if(allSubTasks.new.length === 0){
+            p = TaskMgr.loadNewTasks();//load again
+        }else{
+            p = Promise.resolve();
+        }
+        return p;
+    },
+    popNewTask:()=>{
+        return new Promise((resolve)=>{
+            let newtask = allSubTasks.new.shift();
+            if(newtask){
+                TaskMgr.mvTask(newtask.subid, 'new', 'running', (newPath, succ)=>{ 
+                    if(succ){
+                        resolve(newtask);
+                    }else{
+                        allSubTasks.new.unshift(newtask);
+                        resolve(null);
+                    }
+                });
+            }else{
+                resolve(null);
+            }
+        });
+    },
+    getTask:(type, id)=>{
+        let result;
+        let arr = allSubTasks[type];
+        if(!arr){
+            return result;
+        }
+        for(let i=0;i<arr.length;i++){
+            let task = arr[i];
+            if(task.subid === id){
+                result = task;
+                break;
+            } 
+        }
+        return result;
+    },
+    getTasks:(type)=>{
+        return allSubTasks[type];
+    },
+    getTaskFiles:(type, callback)=>{
+        let rootdir = main_config.progressRootFolder;
+        let targetFolder = pathutil.resolve(rootdir, `./${type}`);
+        makeDir.getDirectFiles(targetFolder, (files)=>{
+            callback(files);
+        });
+    },
+    getFinalStats:()=>{//master 收尾用: 以盘上终态为准统计成功/失败(resume 跨轮累计也是对的)
+        const {taskFolder, taskFolderFinished, taskFolderFailed, userMessagesFolder} = main_config;
+        const listSubids = (folder)=>{
+            return fs.readdirSync(folder)
+                .filter((fname)=>fname.endsWith('.task.json'))
+                .map((fname)=>fname.replace(/\.task\.json$/, ''));
+        };
+        const listMessageSubids = ()=>{//目录缺失(旧版本任务文件夹)按无消息计; 不合规文件名剔除, 防空 subid 虚增计数
+            try{
+                return fs.readdirSync(userMessagesFolder)
+                    .map((fname)=>parseMessageSubid(fname))
+                    .filter((subid)=>subid);
+            }catch(e){
+                return [];
+            };
+        };
+        const failedTasks = listSubids(taskFolderFailed);
+        return {
+            taskFolder,
+            succeeded: listSubids(taskFolderFinished).length,
+            failed: failedTasks.length,
+            failedTasks,
+            succButSaveFailed: new Set(listMessageSubids()).size,//任务成功但结果保存失败的任务数(一任务多问题只计一次)
+        };
+    },
+    mvTaskPromise: (subid, from, to)=> {
+        return new Promise((resolve, reject)=>{
+            TaskMgr.mvTask(subid, from, to, (newPath, succ)=>{
+                succ ? resolve(newPath) : reject(newPath);
+            });
+        });
+    },
+    mvTask: (subid, from, to, callback)=> {
+        runLog.write('Move task:', subid, `${from}>${to}`);
+        let {progressRootFolder} = main_config;
+        let oldPath = pathutil.resolve(progressRootFolder, `./${from}/${subid}.task.json`);
+        let newPath = pathutil.resolve(progressRootFolder, `./${to}/${subid}.task.json`);
+        try{
+            fs.rename(oldPath, newPath, (err) => {
+                if (err) {
+                    callback(newPath, false);
+                }else{
+                    callback(newPath, true);
+                }
+            });
+        }catch(e){
+            runLog.warn(e);
+            callback(newPath, false);
+        }
+    },
+    saveTaskResult:(subid, hasError, data)=>{
+        //数据不可 JSON 序列化时不抛错(抛错会把任务吞掉)也不翻转成败: 任务跑通了只是结果存不下,
+        //结果文件改写为说明占位(成功侧对象本体, 错误侧包成数组), 任务状态不受影响; 同步写用户消息文件
+        if(typeof data !== 'undefined'){
+            try{
+                let json = JSON.stringify(data);
+                if(typeof json !== 'string'){//函数/Symbol: stringify 返回 undefined 而不抛错, 同样视同序列化失败(否则 writeFileSync 抛 TypeError, 任务滞留 running 静默丢失)
+                    throw new Error('result value is not JSON-serializable (function or symbol)');
+                };
+            }catch(serializeErr){
+                runLog.error('[Task Error] failed to serialize task result', subid, serializeErr);
+                let placeholder = {
+                    type: 'result_serialize_error',
+                    message: serializeErr && serializeErr.message ? serializeErr.message : String(serializeErr),
+                };
+                data = hasError ? [placeholder] : placeholder;
+                TaskMgr.saveUserMessage({
+                    type: 'result_serialize_error',
+                    level: 'warn',//结果写失败: 用户需关注的警告级问题
+                    subid,
+                    side: hasError ? 'error' : 'result',
+                    message: placeholder.message,
+                });
+            };
+        };
+        let {taskFolderResult, taskFolderResultFailed} = main_config;
+        let folder = hasError ? taskFolderResultFailed : taskFolderResult;
+        let fpath = pathutil.resolve(folder, `${subid}.json`);
+        let fcontent;
+        if(typeof data === 'undefined') {
+            fcontent = 'undefined';
+        }else{
+            fcontent = JSON.stringify(data);
+        };
+        fs.writeFileSync(fpath, fcontent);
+    },
+    saveUserMessage:({type, subid, ...rest})=>{
+        //用户可见问题消息: 一任务一问题一文件(<type>.<subid>.json), JSON 格式, monitor 工具可读;
+        //每条必带 level(默认 info, 调用方可按严重度覆盖, 如结果写失败为 warn);
+        //尽力而为: 写失败只记日志, 不干扰任务主流程
+        try{
+            if(!MESSAGE_TYPE_PATTERN.test(type)){//type 含点会让文件名按首点切分出错的(见文件头命名规范), 拒写并显式报错
+                throw new Error(`invalid user message type "${type}": must match ${MESSAGE_TYPE_PATTERN}`);
+            };
+            let {userMessagesFolder} = main_config;
+            let fpath = pathutil.resolve(userMessagesFolder, `${type}.${subid}.json`);
+            let message = {type, subid, level: 'info', ...rest, timestamp: randomsUtil.getDateTimeTxt()};
+            fs.writeFileSync(fpath, JSON.stringify(message, null, 2) + '\n');
+        }catch(e){
+            runLog.error('[Task Error] failed to save user message', type, subid, e);
+        };
+    },
+    deleteUserMessage:(subid)=>{
+        //任务重跑(retry_fails/maxTaskRetries 重试)时清掉它在 .sys/user_messages 的全部类型消息,
+        //防前序尝试的遗留消息在新一轮成功后仍是脏数据; 尽力而为: 失败只记日志不干扰主流程
+        try{
+            let {userMessagesFolder} = main_config;
+            fs.readdirSync(userMessagesFolder)
+                .filter((fname)=>parseMessageSubid(fname) === subid)//按文件名解析精确比对, 后缀匹配会误删带点 subid 的他人消息
+                .forEach((fname)=>{
+                    fs.unlinkSync(pathutil.resolve(userMessagesFolder, fname));
+                });
+        }catch(e){
+            runLog.error('[Task Error] failed to delete user message', subid, e);
+        };
+    },
+    getSavedResultType:(subid)=>{
+        let {taskFolderResult, taskFolderResultFailed} = main_config;
+        let succPath = pathutil.resolve(taskFolderResult, `${subid}.json`);
+        let errPath = pathutil.resolve(taskFolderResultFailed, `${subid}.json`);
+        if(fs.existsSync(succPath)) return 'succ';
+        if(fs.existsSync(errPath)) return 'error';
+        return null;
+    },
+    saveTaskLog:(subid, log)=>{
+        let {logsRootFolder} = main_config;
+        let folder = logsRootFolder;
+        let fpath = pathutil.resolve(folder, `${subid}.json`);
+        let fcontent;
+        if(typeof log === 'undefined') {
+            fcontent = '';
+        }else{
+            log.subid = subid;
+            fcontent = JSON.stringify(log);
+        };
+        fs.writeFileSync(fpath, fcontent);
+    },
+    incrementTaskRetry:(subid)=>{
+        let {taskFolderRunning} = main_config;
+        let fpath = pathutil.resolve(taskFolderRunning, `./${subid}.task.json`);
+        let task = JSON.parse(fs.readFileSync(fpath, 'utf-8'));
+        task.__retryCount = (task.__retryCount || 0) + 1;
+        fs.writeFileSync(fpath, JSON.stringify(task));
+        return task.__retryCount;
+    },
+    deleteSavedResult:(subid)=>{
+        let {taskFolderResult, taskFolderResultFailed} = main_config;
+        [taskFolderResult, taskFolderResultFailed].forEach((folder)=>{
+            let fpath = pathutil.resolve(folder, `${subid}.json`);
+            if(fs.existsSync(fpath)){
+                fs.unlinkSync(fpath);
+            }
+        });
+    },
+    resetTaskRetry:(subid)=>{
+        let {taskFolderNew} = main_config;
+        let fpath = pathutil.resolve(taskFolderNew, `./${subid}.task.json`);
+        let task = JSON.parse(fs.readFileSync(fpath, 'utf-8'));
+        delete task.__retryCount;
+        fs.writeFileSync(fpath, JSON.stringify(task));
+    },
+    forEachSuccResult:(cb)=>{
+        let {taskFolderResult, taskFolderFinished} = main_config;
+        forEachResult(taskFolderResult, taskFolderFinished, cb);
+    },
+    forEachErrorResult:(cb)=>{
+        let {taskFolderResultFailed, taskFolderFailed} = main_config;
+        forEachResult(taskFolderResultFailed, taskFolderFailed, cb);
+    },
+    resetTaskFolders:(taskFolder)=>{
+        fs.rmSync(taskFolder, {recursive: true, force: true});
+    },
+};
 module.exports = TaskMgr;