multi-tasks 3.5.1 → 4.0.0-rc.1

package/workers/index.js

@@ -1,224 +1,282 @@
-const fs = require('fs');
-const pathUtil = require('path');
-const cluster = require('cluster');
-const Multithread = require('./Multithread');
-const backwardv1 = require('../utils/backward-v1');
-const { FUNCTION_CONFIG_KEYS, reviveConfig } = require('../utils/serializeFns');
-
-const TASK_STORAGE_MODES = ['file', 'memory'];
-//task_storage 两档: file(默认, 任务状态实时写磁盘, 可 resume) / memory(纯内存, 进程退出即丢)
-const _resolveTaskStorage = (config)=>{
-    let mode = config.task_storage;
-    if(typeof mode === 'undefined' || mode === null){
-        mode = 'file';
-    };
-    if(TASK_STORAGE_MODES.indexOf(mode) < 0){
-        throw new Error(`invalid task_storage "${mode}": must be one of ${TASK_STORAGE_MODES.join('/')}`);
-    };
-    config.task_storage = mode;
-};
-//纯 memory 档目录中无可恢复状态, resume 系 API 调用即 throw;
-//只在 master 进程拦截: worker 进程重跑入口脚本时目录已被 master 创建, 会走 resume 链路拿 __sysLogFile,
-//worker 侧合并配置后不进 init/resume, 不受此限(否则 memory 档 worker 一起步就 throw)
-const _assertMemoryModeResumable = (config)=>{
-    if(config.task_storage === 'memory' && (cluster.isPrimary || cluster.isMaster)){
-        throw new Error('task_storage "memory" does not persist task state: resume/retry_fails/restart are not available (nothing recoverable in the task folder)');
-    };
-};
-
-const _toAbsPath = (path)=>{
-    if(!path) return path;
-    if(pathUtil.isAbsolute(path))return path;
-    let abspath = pathUtil.resolve(process.cwd(), path);
-    return abspath;
-};
-
-const _normalizeResumeArg = (method, config)=>{
-    if(typeof config === 'string'){
-        throw new Error(`method ${method} need a config object with param "taskFolder" (string path form is no longer supported)`);
-    };
-    return backwardv1.v1tov2(config);
-};
-
-//读取任务目录里的 task_config.json 快照, 复活其中的函数字符串后作底;
-//用户传入的同名字段优先(合并结果会随 resume 被 initFolders 重写回快照, 成为新基线)。
-const _mergeSnapshotConfig = (config)=>{
-    let snapshot = JSON.parse(fs.readFileSync(pathUtil.resolve(config.taskFolder, 'task_config.json')));
-    Object.keys(snapshot).forEach((key)=>{
-        if(key.startsWith('__')) delete snapshot[key];//读侧同样剔除一次性标记: 兼容 <=3.1.3 旧版本写出的带标记快照(防 stale 标记误触发 wipe/错走链路)
-    });
-    let userConfig = { ...config };
-    if(typeof userConfig.initialTasks === 'string'){
-        delete userConfig.initialTasks;//目录指针用法(auto-resume), 不应覆盖/污染快照里的任务数组
-    };
-    //用户已传入同名函数时, 快照里的源码字符串无需复活(反正会被覆盖)——容忍不可复活的源码(如绑定函数)
-    FUNCTION_CONFIG_KEYS.forEach((key)=>{
-        if(typeof userConfig[key] === 'function' && typeof snapshot[key] === 'string'){
-            delete snapshot[key];
-        };
-    });
-    let revived = reviveConfig(snapshot);
-    let merged = { ...revived, ...userConfig };
-    if(typeof merged.processTask !== 'function'){
-        throw new Error('FATAL: no processTask available: task_config.json has none and none was passed in config.');
-    };
-    return merged;
-};
-
-//run.log 路径在入口统一算好挂到 config(__ 前缀: 一次性派生值, 不进快照), 与 progressBar 无关;
-//master/worker 各进程都重跑入口、算出同一路径, Multithread 直接使用, 不做任何路径推导。
-const _computeSysLogFile = (config)=>{
-    let taskFolder = config.taskFolder || ((config.taskId && config.taskRootFolder) ? pathUtil.resolve(config.taskRootFolder, config.taskId) : null);
-    if(taskFolder){
-        config.__sysLogFile = pathUtil.resolve(taskFolder, '.sys', 'run.log');
-    };
-};
-
-//initialTasks 支持函数形式(惰性生成): 只在 master 进程求值一次, worker 进程重跑入口脚本时跳过,
-//防生成逻辑(扫目录/查库等初始化脚本)在每个 worker 里各执行一遍。
-//函数可同步返回数组, 也可返回 Promise(resolve 值同样必须是数组)——同步路径保持同步 throw,
-//Promise 路径返回 then 链交给 _proceedAfterEval 等待。求值在入口统一完成(先于 _mergeSnapshotConfig
-//与 Multithread.start): 求值后的数组才会参与合并并写入 task_config.json(函数本身会被
-//JSON.stringify 丢弃, 不进存档、不参与 revive)。
-const _evalInitialTasks = (config)=>{
-    if(typeof config.initialTasks !== 'function') return null;
-    if(!(cluster.isPrimary || cluster.isMaster)) return null;
-    let tasks = config.initialTasks();
-    let applyTasks = (resolved)=>{
-        if(!Array.isArray(resolved)){
-            throw new Error('FATAL: initialTasks function should return an array!');
-        };
-        config.initialTasks = resolved;
-    };
-    if(tasks && typeof tasks.then === 'function'){//Promise 形式: 异步求值, resolve 后同样校验数组(假设真 Promise, 即 .then 返回 Promise; 非标准 thenable 不支持)
-        return tasks.then(applyTasks);
-    };
-    applyTasks(tasks);
-    return null;
-};
-
-//同步路径(pending 为 null)直接执行 proceed, 保持既有同步 throw 语义;
-//Promise 路径等 resolve 后执行, resolve 非数组/reject/proceed 抛错时打 FATAL 并 exit(1)
-//(与 init 异步失败一致; Promise 路径只发生在 master, 无同步 throw 对象)。
-const _proceedAfterEval = (pending, proceed)=>{
-    if(!pending){
-        proceed();
-        return;
-    };
-    return pending.then(proceed).catch((e)=>{
-        console.error('[FATAL] failed to evaluate initialTasks:', e);
-        process.exit(1);
-    });
-};
-
-const start = (config)=>{
-    config = backwardv1.v1tov2(config);
-    return _proceedAfterEval(_evalInitialTasks(config), ()=>{
-
-        config.taskRootFolder = _toAbsPath(config.taskRootFolder);
-        config.taskFolder = _toAbsPath(config.taskFolder);
-        _computeSysLogFile(config);
-        _resolveTaskStorage(config);
-
-        Multithread.start(config);
-    });
-};
-const resume = (config)=>{
-    config = _normalizeResumeArg('resume', config);
-    //先校验/推导 taskFolder 再求值 initialTasks(与 retry_fails/restart 一致):
-    //缺目录时同步 throw fail fast, 不让可能昂贵的生成逻辑白跑一遍
-    config.taskRootFolder = _toAbsPath(config.taskRootFolder);
-    config.taskFolder = _toAbsPath(config.taskFolder);
-    if(!config.taskFolder){
-        if(!config.taskRootFolder || !config.taskId){
-            throw new Error('method resume need param "taskFolder" (or "taskRootFolder" + "taskId")');
-        };
-        config.taskFolder = pathUtil.resolve(config.taskRootFolder, config.taskId);
-    };
-    config.taskRootFolder = pathUtil.dirname(config.taskFolder);
-    config.taskId = pathUtil.basename(config.taskFolder);
-    config.__resume = true;
-    return _proceedAfterEval(_evalInitialTasks(config), ()=>{
-        _computeSysLogFile(config);
-        config = _mergeSnapshotConfig(config);
-        _resolveTaskStorage(config);
-        _assertMemoryModeResumable(config);
-        Multithread.start(config);
-    });
-};
-const retry_fails = (config)=>{
-    config = _normalizeResumeArg('retry_fails', config);
-    let taskFolder = _toAbsPath(config.taskFolder);
-    if(!taskFolder){
-        throw new Error('method retry_fails need param "taskFolder"');
-    }
-    config.taskFolder = taskFolder;
-    config.taskRootFolder = pathUtil.dirname(taskFolder);
-    config.taskId = pathUtil.basename(taskFolder);
-    config.__resume = true;
-    config.__retryFails = true;
-    return _proceedAfterEval(_evalInitialTasks(config), ()=>{
-        _computeSysLogFile(config);
-        config = _mergeSnapshotConfig(config);
-        _resolveTaskStorage(config);
-        _assertMemoryModeResumable(config);
-        Multithread.start(config);
-    });
-};
-const restart = (config)=>{
-    config = _normalizeResumeArg('restart', config);
-    let taskFolder = _toAbsPath(config.taskFolder);
-    if(!taskFolder){
-        throw new Error('method restart need param "taskFolder"');
-    }
-    config.taskFolder = taskFolder;
-    config.taskRootFolder = pathUtil.dirname(taskFolder);
-    config.taskId = pathUtil.basename(taskFolder);
-    config.__restart = true;
-    return _proceedAfterEval(_evalInitialTasks(config), ()=>{
-        _computeSysLogFile(config);
-        config = _mergeSnapshotConfig(config);
-        _resolveTaskStorage(config);
-        _assertMemoryModeResumable(config);
-        Multithread.start(config);
-    });
-};
-
-const multiTasks = (config)=>{
-    config = backwardv1.v1tov2(config);
-    let {initialTasks, taskRootFolder, taskId} = config;
-    let isResume = false;
-
-    if(typeof initialTasks === 'string'){//might be an existing tasks folder
-        if(fs.existsSync(initialTasks)){
-            isResume = true;
-            config.taskRootFolder = pathUtil.dirname(initialTasks);
-        }else if(taskRootFolder){
-            let folder = pathUtil.resolve(taskRootFolder, initialTasks);
-            if(fs.existsSync(folder)){
-                isResume = true;
-                config.taskRootFolder = pathUtil.dirname(folder);
-            }
-        }
-        if(!isResume) throw new Error('FATAL: initialTasks should be an array, a function or a task root folder!');
-    }else{
-        let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
-        if(fs.existsSync(taskFolder)){
-            isResume = true;
-        }
-    }
-
-    if(isResume){
-        return resume(config);
-    }else{
-        return start(config);
-    }
-
-};
-
-multiTasks.start = start;
-multiTasks.retry_fails = retry_fails;
-multiTasks.resume = resume;
-multiTasks.restart = restart;
-
-module.exports = multiTasks;
+const fs = require('fs');
+const pathUtil = require('path');
+const cluster = require('cluster');
+const Multithread = require('./Multithread');
+const backwardv1 = require('../utils/backward-v1');
+const { FUNCTION_CONFIG_KEYS, reviveConfig } = require('../utils/serializeFns');
+
+const TASK_STORAGE_MODES = ['file', 'memory', 'aof'];
+//task_storage 三档: file(任务状态实时写磁盘, 可 resume) / memory(纯内存, 进程退出即丢) /
+//aof(master 内存 + append-only aof, 可 resume, 接近 memory 档速度)。
+//4.0.0 起缺省值 file -> aof; 新默认值只作用于 fresh start, resume 链路永远跟随目录里的真实模式(见 _resolveResumeTaskStorage)
+const DEFAULT_TASK_STORAGE = 'aof';
+const _resolveTaskStorage = (config)=>{
+    let mode = config.task_storage;
+    if(typeof mode === 'undefined' || mode === null){
+        mode = DEFAULT_TASK_STORAGE;
+        //一次性标记: 启动横幅据此提示「默认值已变, 可显式 task_storage:'file' 回旧行为」; __ 键写存档前剔除
+        config.__storageDefaultApplied = true;
+    };
+    if(TASK_STORAGE_MODES.indexOf(mode) < 0){
+        throw new Error(`invalid task_storage "${mode}": must be one of ${TASK_STORAGE_MODES.join('/')}`);
+    };
+    config.task_storage = mode;
+};
+//resume 链路真实模式判定: 存档 task_storage 键(>=3.5.0 的存档都有)优先; 无键(<3.5.0 旧档)按目录结构嗅探——
+//.sys/aof.log 或 .sys/aof.snapshot.json 存在即 aof, 否则 file。不套 start 的新默认值, 旧 file 档不被默认值变更误伤
+const _sniffTaskStorage = (taskFolder)=>{
+    if(fs.existsSync(pathUtil.resolve(taskFolder, '.sys', 'aof.log'))) return 'aof';
+    if(fs.existsSync(pathUtil.resolve(taskFolder, '.sys', 'aof.snapshot.json'))) return 'aof';
+    return 'file';
+};
+const _readSnapshotTaskStorage = (taskFolder)=>{
+    try{
+        return JSON.parse(fs.readFileSync(pathUtil.resolve(taskFolder, 'task_config.json'))).task_storage;
+    }catch(e){
+        return undefined;//快照缺失/损坏由 _mergeSnapshotConfig 统一报错
+    };
+};
+const _resolveResumeTaskStorage = (config, userStorage)=>{
+    let real = _readSnapshotTaskStorage(config.taskFolder);
+    if(typeof real === 'undefined' || real === null){
+        real = _sniffTaskStorage(config.taskFolder);
+    };
+    if(TASK_STORAGE_MODES.indexOf(real) < 0){
+        throw new Error(`invalid task_storage "${real}" in task_config.json: must be one of ${TASK_STORAGE_MODES.join('/')}`);
+    };
+    if(typeof userStorage !== 'undefined' && userStorage !== null){
+        if(TASK_STORAGE_MODES.indexOf(userStorage) < 0){
+            throw new Error(`invalid task_storage "${userStorage}": must be one of ${TASK_STORAGE_MODES.join('/')}`);
+        };
+        if(userStorage !== real){
+            throw new Error(`task_storage mismatch: task folder was created with task_storage "${real}" but "${userStorage}" was passed; drop task_storage to resume in the folder's own mode`);
+        };
+    };
+    config.task_storage = real;
+};
+//aof 档周期 checkpoint 配置: 仅 aof 档解析, 避免污染 file/memory 存档。
+//双模式(拍板): aof_checkpoint_ops 显式设置 = 旧固定行数模式(新旧对比开关);
+//未设置 = 字节几何模式(aof_checkpoint_bytes, 语义/默认值见 AofStore);
+//两者 <=0 均关闭周期触发(正常退出时的收尾 checkpoint 不受影响)。缺省不透传, 默认值单一来源在 AofStore。
+const _resolveAofCheckpointConfig = (config)=>{
+    if(config.task_storage !== 'aof') return;
+    ['aof_checkpoint_ops', 'aof_checkpoint_bytes'].forEach((key)=>{
+        let value = config[key];
+        if(typeof value === 'undefined' || value === null) return;
+        if(typeof value !== 'number' || isNaN(value)){
+            throw new Error(`invalid ${key} "${value}": must be a number (<=0 disables periodic checkpoint)`);
+        };
+    });
+};
+//纯 memory 档目录中无可恢复状态, resume 系 API 调用即 throw;
+//只在 master 进程拦截: worker 进程重跑入口脚本时目录已被 master 创建, 会走 resume 链路拿 __sysLogFile,
+//worker 侧合并配置后不进 init/resume, 不受此限(否则 memory 档 worker 一起步就 throw)
+const _assertMemoryModeResumable = (config)=>{
+    if(config.task_storage === 'memory' && (cluster.isPrimary || cluster.isMaster)){
+        throw new Error('task_storage "memory" does not persist task state: resume/retry_fails/restart are not available (nothing recoverable in the task folder)');
+    };
+};
+
+const _toAbsPath = (path)=>{
+    if(!path) return path;
+    if(pathUtil.isAbsolute(path))return path;
+    let abspath = pathUtil.resolve(process.cwd(), path);
+    return abspath;
+};
+
+const _normalizeResumeArg = (method, config)=>{
+    if(typeof config === 'string'){
+        throw new Error(`method ${method} need a config object with param "taskFolder" (string path form is no longer supported)`);
+    };
+    return backwardv1.v1tov2(config);
+};
+
+//读取任务目录里的 task_config.json 快照, 复活其中的函数字符串后作底;
+//用户传入的同名字段优先(合并结果会随 resume 被 initFolders 重写回快照, 成为新基线)。
+const _mergeSnapshotConfig = (config)=>{
+    let snapshot = JSON.parse(fs.readFileSync(pathUtil.resolve(config.taskFolder, 'task_config.json')));
+    Object.keys(snapshot).forEach((key)=>{
+        if(key.startsWith('__')) delete snapshot[key];//读侧同样剔除一次性标记: 兼容 <=3.1.3 旧版本写出的带标记快照(防 stale 标记误触发 wipe/错走链路)
+    });
+    let userConfig = { ...config };
+    if(typeof userConfig.initialTasks === 'string'){
+        delete userConfig.initialTasks;//目录指针用法(auto-resume), 不应覆盖/污染快照里的任务数组
+    };
+    //用户已传入同名函数时, 快照里的源码字符串无需复活(反正会被覆盖)——容忍不可复活的源码(如绑定函数)
+    FUNCTION_CONFIG_KEYS.forEach((key)=>{
+        if(typeof userConfig[key] === 'function' && typeof snapshot[key] === 'string'){
+            delete snapshot[key];
+        };
+    });
+    let revived = reviveConfig(snapshot);
+    let merged = { ...revived, ...userConfig };
+    if(typeof merged.processTask !== 'function'){
+        throw new Error('FATAL: no processTask available: task_config.json has none and none was passed in config.');
+    };
+    return merged;
+};
+
+//run.log 路径在入口统一算好挂到 config(__ 前缀: 一次性派生值, 不进快照), 与 progressBar 无关;
+//master/worker 各进程都重跑入口、算出同一路径, Multithread 直接使用, 不做任何路径推导。
+const _computeSysLogFile = (config)=>{
+    let taskFolder = config.taskFolder || ((config.taskId && config.taskRootFolder) ? pathUtil.resolve(config.taskRootFolder, config.taskId) : null);
+    if(taskFolder){
+        config.__sysLogFile = pathUtil.resolve(taskFolder, '.sys', 'run.log');
+    };
+};
+
+//initialTasks 支持函数形式(惰性生成): 只在 master 进程求值一次, worker 进程重跑入口脚本时跳过,
+//防生成逻辑(扫目录/查库等初始化脚本)在每个 worker 里各执行一遍。
+//函数可同步返回数组, 也可返回 Promise(resolve 值同样必须是数组)——同步路径保持同步 throw,
+//Promise 路径返回 then 链交给 _proceedAfterEval 等待。求值在入口统一完成(先于 _mergeSnapshotConfig
+//与 Multithread.start): 求值后的数组才会参与合并并写入 task_config.json(函数本身会被
+//JSON.stringify 丢弃, 不进存档、不参与 revive)。
+const _evalInitialTasks = (config)=>{
+    if(typeof config.initialTasks !== 'function') return null;
+    if(!(cluster.isPrimary || cluster.isMaster)) return null;
+    let tasks = config.initialTasks();
+    let applyTasks = (resolved)=>{
+        if(!Array.isArray(resolved)){
+            throw new Error('FATAL: initialTasks function should return an array!');
+        };
+        config.initialTasks = resolved;
+    };
+    if(tasks && typeof tasks.then === 'function'){//Promise 形式: 异步求值, resolve 后同样校验数组(假设真 Promise, 即 .then 返回 Promise; 非标准 thenable 不支持)
+        return tasks.then(applyTasks);
+    };
+    applyTasks(tasks);
+    return null;
+};
+
+//同步路径(pending 为 null)直接执行 proceed, 保持既有同步 throw 语义;
+//Promise 路径等 resolve 后执行, resolve 非数组/reject/proceed 抛错时打 FATAL 并 exit(1)
+//(与 init 异步失败一致; Promise 路径只发生在 master, 无同步 throw 对象)。
+const _proceedAfterEval = (pending, proceed)=>{
+    if(!pending){
+        proceed();
+        return;
+    };
+    return pending.then(proceed).catch((e)=>{
+        console.error('[FATAL] failed to evaluate initialTasks:', e);
+        process.exit(1);
+    });
+};
+
+const start = (config)=>{
+    config = backwardv1.v1tov2(config);
+    return _proceedAfterEval(_evalInitialTasks(config), ()=>{
+
+        config.taskRootFolder = _toAbsPath(config.taskRootFolder);
+        config.taskFolder = _toAbsPath(config.taskFolder);
+        _computeSysLogFile(config);
+        _resolveTaskStorage(config);
+        _resolveAofCheckpointConfig(config);
+
+        Multithread.start(config);
+    });
+};
+const resume = (config)=>{
+    config = _normalizeResumeArg('resume', config);
+    //先校验/推导 taskFolder 再求值 initialTasks(与 retry_fails/restart 一致):
+    //缺目录时同步 throw fail fast, 不让可能昂贵的生成逻辑白跑一遍
+    config.taskRootFolder = _toAbsPath(config.taskRootFolder);
+    config.taskFolder = _toAbsPath(config.taskFolder);
+    if(!config.taskFolder){
+        if(!config.taskRootFolder || !config.taskId){
+            throw new Error('method resume need param "taskFolder" (or "taskRootFolder" + "taskId")');
+        };
+        config.taskFolder = pathUtil.resolve(config.taskRootFolder, config.taskId);
+    };
+    config.taskRootFolder = pathUtil.dirname(config.taskFolder);
+    config.taskId = pathUtil.basename(config.taskFolder);
+    config.__resume = true;
+    return _proceedAfterEval(_evalInitialTasks(config), ()=>{
+        _computeSysLogFile(config);
+        let userTaskStorage = config.task_storage;
+        config = _mergeSnapshotConfig(config);
+        _resolveResumeTaskStorage(config, userTaskStorage);
+        _resolveAofCheckpointConfig(config);
+        _assertMemoryModeResumable(config);
+        Multithread.start(config);
+    });
+};
+const retry_fails = (config)=>{
+    config = _normalizeResumeArg('retry_fails', config);
+    let taskFolder = _toAbsPath(config.taskFolder);
+    if(!taskFolder){
+        throw new Error('method retry_fails need param "taskFolder"');
+    }
+    config.taskFolder = taskFolder;
+    config.taskRootFolder = pathUtil.dirname(taskFolder);
+    config.taskId = pathUtil.basename(taskFolder);
+    config.__resume = true;
+    config.__retryFails = true;
+    return _proceedAfterEval(_evalInitialTasks(config), ()=>{
+        _computeSysLogFile(config);
+        let userTaskStorage = config.task_storage;
+        config = _mergeSnapshotConfig(config);
+        _resolveResumeTaskStorage(config, userTaskStorage);
+        _resolveAofCheckpointConfig(config);
+        _assertMemoryModeResumable(config);
+        Multithread.start(config);
+    });
+};
+const restart = (config)=>{
+    config = _normalizeResumeArg('restart', config);
+    let taskFolder = _toAbsPath(config.taskFolder);
+    if(!taskFolder){
+        throw new Error('method restart need param "taskFolder"');
+    }
+    config.taskFolder = taskFolder;
+    config.taskRootFolder = pathUtil.dirname(taskFolder);
+    config.taskId = pathUtil.basename(taskFolder);
+    config.__restart = true;
+    return _proceedAfterEval(_evalInitialTasks(config), ()=>{
+        _computeSysLogFile(config);
+        let userTaskStorage = config.task_storage;
+        config = _mergeSnapshotConfig(config);
+        _resolveResumeTaskStorage(config, userTaskStorage);
+        _resolveAofCheckpointConfig(config);
+        _assertMemoryModeResumable(config);
+        Multithread.start(config);
+    });
+};
+
+const multiTasks = (config)=>{
+    config = backwardv1.v1tov2(config);
+    let {initialTasks, taskRootFolder, taskId} = config;
+    let isResume = false;
+
+    if(typeof initialTasks === 'string'){//might be an existing tasks folder
+        if(fs.existsSync(initialTasks)){
+            isResume = true;
+            config.taskRootFolder = pathUtil.dirname(initialTasks);
+        }else if(taskRootFolder){
+            let folder = pathUtil.resolve(taskRootFolder, initialTasks);
+            if(fs.existsSync(folder)){
+                isResume = true;
+                config.taskRootFolder = pathUtil.dirname(folder);
+            }
+        }
+        if(!isResume) throw new Error('FATAL: initialTasks should be an array, a function or a task root folder!');
+    }else{
+        let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
+        if(fs.existsSync(taskFolder)){
+            isResume = true;
+        }
+    }
+
+    if(isResume){
+        return resume(config);
+    }else{
+        return start(config);
+    }
+
+};
+
+multiTasks.start = start;
+multiTasks.retry_fails = retry_fails;
+multiTasks.resume = resume;
+multiTasks.restart = restart;
+
+module.exports = multiTasks;

package/workers/resultFileAppender.js

@@ -0,0 +1,62 @@
+const fs = require('fs');
+const pathUtil = require('path');
+const runLog = require('../utils/runLog');
+
+//内部开关 memory_append_result_files 的 memory 档侧(不对外文档化, 与 aof_checkpoint_* 同属内部调参键):
+//  config.memory_append_result_files === true 时把结果行追加到 results/succ.json|errors.json
+//  (与 aof 档同款 {subid,data} NDJSON 行 + {subid,deleted:true} 墓碑行, 读者按 subid 取最后一行)。
+//  只追加、收尾不做整体重写(memory 档无 resume, 结果终态出口本来就是 onFinish);
+//  挂上的 flushResults 只关追加流——Multithread 三条退出路径 typeof flushResults==='function'
+//  即 await, 保证流缓冲冲下盘再写计数器行退出。
+const attachResultAppender = (store, taskFolder) => {
+  const resultStreams = { succ: null, errors: null };
+  let broken = false;
+  let warned = false;
+
+  const appendLine = (hasError, record) => {
+    if (broken) return;
+    const side = hasError ? 'errors' : 'succ';
+    if (!resultStreams[side]) {
+      resultStreams[side] = fs.createWriteStream(pathUtil.resolve(taskFolder, 'results', `${side}.json`), { flags: 'a' });
+      resultStreams[side].on('error', (e) => {
+        broken = true;
+        if (!warned) {
+          warned = true;
+          runLog.warn('[resultFileAppender]: result stream broken, result files degraded for this run', e);
+        };
+      });
+    };
+    resultStreams[side].write(JSON.stringify(record) + '\n');
+  };
+
+  const rawSaveResult = store.saveResult;
+  store.saveResult = (subid, hasError, data) => {
+    const result = rawSaveResult(subid, hasError, data);
+    appendLine(hasError, typeof data === 'undefined' ? { subid } : { subid, data });
+    return result;
+  };
+  const rawDeleteResult = store.deleteResult;
+  store.deleteResult = (subid) => {
+    const type = store.getSavedResultType(subid);//'succ'/'error'/null: 删前取侧, 墓碑行追加到对应文件
+    const result = rawDeleteResult(subid);
+    if (type) appendLine(type === 'error', { subid, deleted: true });
+    return result;
+  };
+
+  //用 end(cb) 而非等 'close': 流中途出错已销毁时 'close' 早已发完不再重发, 等它会永远悬挂
+  //(退出路径 await 本函数, 悬挂 = 挂死 master 收尾); end(cb) 对已销毁流也回调
+  store.flushResults = async () => {
+    for (const side of ['succ', 'errors']) {
+      const s = resultStreams[side];
+      if (s) {
+        resultStreams[side] = null;
+        await new Promise((resolve) => { s.end(resolve); });
+      };
+    };
+  };
+
+  store.__resultStreams = resultStreams;//测试探针(模拟流中途出错)
+  return store;
+};
+
+module.exports = { attachResultAppender };