multi-tasks 3.4.1 → 3.5.1

package/bench-storage.svg

@@ -0,0 +1,17 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="720" height="252" viewBox="0 0 720 252" font-family="Segoe UI, Helvetica, Arial, sans-serif">
+  <rect x="1" y="1" width="718" height="250" rx="8" fill="#ffffff" stroke="#dadce0"/>
+  <text x="24" y="38" font-size="17" font-weight="600" fill="#202124">task_storage benchmark — 30,000 tasks × 28 workers</text>
+  <text x="24" y="60" font-size="12" fill="#5f6368">wall-clock time of a full run, average of 3 runs · lower is better</text>
+
+  <text x="112" y="114" font-size="14" fill="#202124" text-anchor="end">file</text>
+  <rect x="120" y="94" width="420" height="34" rx="3" fill="#9aa0a6"/>
+  <text x="548" y="116" font-size="14" fill="#202124">195.1s · 154 tasks/s</text>
+
+  <text x="112" y="172" font-size="14" fill="#202124" text-anchor="end">memory</text>
+  <rect x="120" y="152" width="24" height="34" rx="3" fill="#188038"/>
+  <text x="152" y="174" font-size="14" fill="#202124">11.2s · 2,676 tasks/s</text>
+  <text x="360" y="174" font-size="15" font-weight="700" fill="#188038">17.4× faster</text>
+
+  <line x1="120" y1="208" x2="540" y2="208" stroke="#e8eaed"/>
+  <text x="24" y="232" font-size="11" fill="#5f6368">per-round walls — file: 202.7 / 193.6 / 188.9s · memory: 11.3 / 11.1 / 11.3s · Windows 11, SSD · script: test-manual/bench-storage.js</text>
+</svg>

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "multi-tasks",
-  "version": "3.4.1",
+  "version": "3.5.1",
   "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
   "main": "index.js",
   "files": [
     "index.js",
     "workers",
-    "utils/*.js"
+    "utils/*.js",
+    "bench-storage.svg"
   ],
   "repository": {
     "type": "git",

package/utils/startupBanner.js

@@ -1,51 +1,51 @@
-//启动信息横幅渲染器(纯函数, 不碰 console/fs/cluster): 仅标题行 ASCII 框装饰,
-//框宽只由标题行内容决定——长路径只出现在平铺字段行, 不会撑破框、不会折行。
-//renderBanner: 终端上屏用(劫持前 console 直出); renderLogHeader: run.log 文件头部用(含 MODE 行);
-//writeLogHeader: 判 sysLogFile 后写 runLog——判断收敛在方法内部, 调用点不写 if;
-//无路径时(内部 API multiTasks.start 自动生成目录名)no-op, 不透传 console。
-const runLog = require('./runLog');
-
-const VERSION = require('../package.json').version;
-
-const KEY_WIDTH = 19;//键列宽: 需容纳最长键 'monitor install cmd', 改动时同步调
-
-const renderTitleBox = (mode) => {
-  const left = `multi-tasks v${VERSION}`;
-  const right = `MODE: ${mode}`;
-  const innerWidth = left.length + right.length + 4;//左右内容间至少 4 空格
-  const middle = `| ${left}${' '.repeat(innerWidth - left.length - right.length)}${right} |`;
-  const border = `+${'-'.repeat(innerWidth + 2)}+`;
-  return [border, middle, border];
-};
-
-const renderFieldLines = (info) => {
-  const rows = [
-    ['taskId', info.taskId],
-    ['taskFolder', info.taskFolder],
-    ['workers', `${info.numOfWorkers} (numCPUs=${info.numCPUs})`],
-    ['master PID', `${info.pid}`],
-    ['new tasks', `${info.newTasks}`],
-    ['run.log', info.sysLogFile || '(not available)'],
-    ['started at', info.startedAt],
-    ['monitor install cmd', 'npm install multi-tasks-monitor'],
-    ['monitor start cmd', `npx multi-tasks-monitor --task-dir "${info.taskFolder}" --port 3777`],
-  ];
-  return rows.map(([key, value]) => `${key.padEnd(KEY_WIDTH)} : ${value}`);
-};
-
-const renderBanner = (info) => {
-  return [...renderTitleBox(info.mode), ...renderFieldLines(info)].join('\n');
-};
-
-const renderLogHeader = (info) => {
-  return [`${'MODE'.padEnd(KEY_WIDTH)} : ${info.mode}`, ...renderFieldLines(info)].join('\n');
-};
-
-const writeLogHeader = (info) => {
-  if (!info.sysLogFile) {
-    return;
-  };
-  runLog.write(renderLogHeader(info));
-};
-
-module.exports = { renderBanner, renderLogHeader, writeLogHeader };
+//启动信息横幅渲染器(纯函数, 不碰 console/fs/cluster): 仅标题行 ASCII 框装饰,
+//框宽只由标题行内容决定——长路径只出现在平铺字段行, 不会撑破框、不会折行。
+//renderBanner: 终端上屏用(劫持前 console 直出); renderLogHeader: run.log 文件头部用(含 MODE 行);
+//writeLogHeader: 判 sysLogFile 后写 runLog——判断收敛在方法内部, 调用点不写 if;
+//无路径时(内部 API multiTasks.start 自动生成目录名)no-op, 不透传 console。
+const runLog = require('./runLog');
+
+const VERSION = require('../package.json').version;
+
+const KEY_WIDTH = 19;//键列宽: 需容纳最长键 'monitor install cmd', 改动时同步调
+
+const renderTitleBox = (mode) => {
+  const left = `multi-tasks v${VERSION}`;
+  const right = `MODE: ${mode}`;
+  const innerWidth = left.length + right.length + 4;//左右内容间至少 4 空格
+  const middle = `| ${left}${' '.repeat(innerWidth - left.length - right.length)}${right} |`;
+  const border = `+${'-'.repeat(innerWidth + 2)}+`;
+  return [border, middle, border];
+};
+
+const renderFieldLines = (info) => {
+  const rows = [
+    ['taskId', info.taskId],
+    ['taskFolder', info.taskFolder],
+    ['workers', `${info.numOfWorkers} (numCPUs=${info.numCPUs})`],
+    ['master PID', `${info.pid}`],
+    ['new tasks', `${info.newTasks}`],
+    ['run.log', info.sysLogFile || '(not available)'],
+    ['started at', info.startedAt],
+    ['monitor install cmd', 'npm install multi-tasks-monitor'],
+    ['monitor start cmd', `npx multi-tasks-monitor --task-dir "${info.taskFolder}" --port 3777`],
+  ];
+  return rows.map(([key, value]) => `${key.padEnd(KEY_WIDTH)} : ${value}`);
+};
+
+const renderBanner = (info) => {
+  return [...renderTitleBox(info.mode), ...renderFieldLines(info)].join('\n');
+};
+
+const renderLogHeader = (info) => {
+  return [`${'MODE'.padEnd(KEY_WIDTH)} : ${info.mode}`, ...renderFieldLines(info)].join('\n');
+};
+
+const writeLogHeader = (info) => {
+  if (!info.sysLogFile) {
+    return;
+  };
+  runLog.write(renderLogHeader(info));
+};
+
+module.exports = { renderBanner, renderLogHeader, writeLogHeader };

package/workers/IpcTaskStore.js

@@ -0,0 +1,136 @@
+const randomsUtil = require('../utils/randoms');
+const runLog = require('../utils/runLog');
+
+//memory 档 worker 侧存储后端: 把 TaskMgr 的目录操作逐条翻译成 {__storageOp} IPC 消息发 master,
+//与 file 档盘操作逐条同构(禁止捎带/合并消息; 交卷与领卷是两个动作, 严格先交后领)。
+//无返回值操作 fire-and-forget(等价 file 档同步写盘「调用返回即生效」, 同 worker IPC 通道 FIFO 保序);
+//需返回值的操作(incrementTaskRetry/mvTask/mvTaskPromise)带 needAck, 等 master 的 {__storageOpResult} 回执。
+//回执匹配无 opId: master 串行应用按序回执, pending 队列按序 shift 即可对齐。
+//__storageOp/__storageOpResult 是库占用消息字段, 与 __broadcast/__sysEvent/__workerStatus 同级, 勿复用。
+const UNDEFINED_SENTINEL = '__multitasks_undefined__';//JSON 数组里 undefined 会变 null, 用哨兵对象保住语义(file 档 undefined 结果写字面量 'undefined')
+
+let masterTaskId = null;
+let pendingAcks = [];//needAck 回执等待队列, 按发送顺序 shift
+
+const encodeArg = (value)=>{
+  if(typeof value === 'undefined') return {[UNDEFINED_SENTINEL]: true};
+  return value;
+};
+
+const send = (op, args, needAck)=>{
+  let message = {__storageOp: true, op, args: args.map(encodeArg)};
+  if(!needAck){
+    process.send(message);
+    return undefined;
+  };
+  message.needAck = true;
+  return new Promise((resolve, reject)=>{
+    process.send(message);//先送后入队: 回执最快也要下个事件循环才到, 不会错位
+    pendingAcks.push({resolve, reject});
+  });
+};
+
+const load = (config)=>{
+  masterTaskId = config.masterTaskId;
+  return IpcTaskStore;
+};
+
+const saveTaskLog = (subid, log)=>{
+  send('saveLog', [subid, log]);
+};
+
+const saveUserMessage = (message)=>{
+  send('saveUserMessage', [message]);
+};
+
+const saveTaskResult = (subid, hasError, data)=>{
+  //序列化检测必须在 worker 侧(IPC 走 JSON, 不可序列化的结果到不了 master):
+  //占位对象 + warn 用户消息, 语义同 TaskMgr.saveTaskResult 兜底(不翻转任务成败)
+  if(typeof data !== 'undefined'){
+    try{
+      let json = JSON.stringify(data);
+      if(typeof json !== 'string'){
+        throw new Error('result value is not JSON-serializable (function or symbol)');
+      };
+    }catch(serializeErr){
+      runLog.error('[Task Error] failed to serialize task result', subid, serializeErr);//runLog 留证与 file 档(TaskMgr.saveTaskResult)同口径
+      let placeholder = {
+        type: 'result_serialize_error',
+        message: serializeErr && serializeErr.message ? serializeErr.message : String(serializeErr),
+      };
+      data = hasError ? [placeholder] : placeholder;
+      saveUserMessage({
+        type: 'result_serialize_error',
+        level: 'warn',
+        subid,
+        side: hasError ? 'error' : 'result',
+        message: placeholder.message,
+      });
+    };
+  };
+  send('saveResult', [subid, hasError, data]);
+};
+
+const deleteSavedResult = (subid)=>{
+  send('deleteResult', [subid]);
+};
+
+const deleteUserMessage = (subid)=>{
+  send('deleteUserMessage', [subid]);
+};
+
+//动态任务即时生效(禁止攒批): 本地按 TaskMgr 同规则赋 subid 后立即发 createTask 消息
+const createNewTask = (task)=>{
+  task.masterTaskId = masterTaskId;
+  task.subid = `${masterTaskId}--${randomsUtil.get36Id()}`;
+  send('createTask', [task]);
+};
+
+const mvTask = (subid, from, to, callback)=>{
+  send('mvTask', [subid, from, to], true).then((value)=>{
+    callback(value.newPath, value.succ);
+  }).catch((e)=>{
+    runLog.warn('[IpcTaskStore]: mvTask ack failed', subid, e);
+    callback(null, false);
+  });
+};
+
+const mvTaskPromise = (subid, from, to)=>{
+  return send('mvTask', [subid, from, to], true).then((value)=>{
+    if(value.succ) return value.newPath;
+    throw value.newPath;
+  });
+};
+
+const incrementTaskRetry = (subid)=>{
+  return send('incrementRetry', [subid], true);
+};
+
+//内部方法: subWorker 消息链收到 __storageOpResult 时调用, FIFO shift 队首兑付
+const __dispatchStorageOpResult = (message)=>{
+  let pending = pendingAcks.shift();
+  if(!pending) return;//无等待者的回执(乱序/重复): 丢弃
+  if(message.ok){
+    pending.resolve(message.value);
+  }else{
+    let errMsg = (message.error && message.error.message) ? message.error.message : 'storage op failed';
+    pending.reject(new Error(errMsg));
+  };
+};
+
+const IpcTaskStore = {
+  load,
+  saveTaskLog,
+  saveTaskResult,
+  deleteSavedResult,
+  saveUserMessage,
+  deleteUserMessage,
+  createNewTask,
+  mvTask,
+  mvTaskPromise,
+  incrementTaskRetry,
+  __dispatchStorageOpResult,
+  __UNDEFINED_SENTINEL: UNDEFINED_SENTINEL,//master 解码 args 用(单一来源, 勿复制字面量)
+};
+
+module.exports = IpcTaskStore;

package/workers/MemoryStore.js

@@ -0,0 +1,247 @@
+const randomsUtil = require('../utils/randoms');
+const runLog = require('../utils/runLog');
+
+//MemoryStore: memory 档 master 侧的内存任务存储, file 档磁盘三态目录(TaskMgr)的内存等价物。
+//  任务四态(new/running/finished/finished_with_errors)、结果、任务日志、user messages 全部在内存,
+//  操作粒度与 TaskMgr 对齐, 供 Multithread 派发复用。
+const VALID_STATES = ['new', 'running', 'finished', 'finished_with_errors'];
+const TERMINAL_STATES = ['finished', 'finished_with_errors'];
+
+const createMemoryStore = (config) => {
+  const { masterTaskId, taskFolder } = config || {};
+  if (!masterTaskId || !taskFolder) {
+    throw new Error('createMemoryStore: config requires {masterTaskId, taskFolder}');
+  };
+
+  const records = new Map();//subid -> {state, task}
+  const newQueue = [];//new 态任务对象 FIFO 队列(getTasks('new') 返回本体)
+  const finishedOrder = [];//subid 首次进入终态的顺序, forEach 按它遍历
+  const results = new Map();//subid -> {hasError, data}
+  const logs = new Map();//subid -> log
+  const messages = new Map();//文件名 `<type>.<subid>.json` -> message
+  const messageSubids = new Set();//有消息的去重 subid 集合, stats().succButSaveFailed 口径(对齐 TaskMgr 一任务多消息只计一次)
+
+  const assertState = (state) => {
+    if (!VALID_STATES.includes(state)) {
+      throw new Error(`MemoryStore: invalid task state "${state}"`);
+    };
+  };
+
+  //消息文件名规范同 TaskMgr 头部约定: 文件名 = <type>.<subid>.json, type 只允许 [a-z0-9_](不含点)
+  const MESSAGE_TYPE_PATTERN = /^[a-z0-9_]+$/;
+
+  const messageFname = (message) => `${message.type}.${message.subid}.json`;
+
+  const add = (task) => {
+    if (!task.subid) {
+      task.masterTaskId = masterTaskId;
+      task.subid = `${masterTaskId}--${randomsUtil.get36Id()}`;
+    };
+    //同 subid 重复 add 按 file 档「覆盖同名文件」语义处理: records 覆盖;
+    //旧项若还在 new 队列先剔除再入队尾, 防同 subid 双派
+    const existing = records.get(task.subid);
+    if (existing && existing.state === 'new') {
+      const index = newQueue.findIndex((item) => item.subid === task.subid);
+      if (index >= 0) {
+        newQueue.splice(index, 1);
+      };
+    };
+    records.set(task.subid, { state: 'new', task });
+    newQueue.push(task);
+    return task;
+  };
+
+  const addMany = (tasks) => {
+    tasks.forEach(add);
+    return tasks;
+  };
+
+  const pop = () => {
+    const task = newQueue.shift();
+    if (!task) {
+      return null;
+    };
+    records.get(task.subid).state = 'running';
+    return task;
+  };
+
+  const saveLog = (subid, log) => {
+    logs.set(subid, log);
+  };
+
+  const saveResult = (subid, hasError, data) => {
+    results.set(subid, { hasError, data });
+  };
+
+  const deleteResult = (subid) => {
+    results.delete(subid);
+  };
+
+  const move = (subid, from, to) => {
+    assertState(from);
+    assertState(to);
+    const rec = records.get(subid);
+    if (!rec || rec.state !== from) {
+      return false;
+    };
+    if (from === 'new') {
+      const index = newQueue.findIndex((task) => task.subid === subid);
+      if (index >= 0) {
+        newQueue.splice(index, 1);
+      };
+    };
+    rec.state = to;
+    if (to === 'new') {
+      newQueue.push(rec.task);
+    };
+    if (TERMINAL_STATES.includes(to) && !finishedOrder.includes(subid)) {
+      finishedOrder.push(subid);
+    };
+    return true;
+  };
+
+  //对齐 file 档: TaskMgr.incrementTaskRetry 读 running/ 目录里的任务文件, 任务不在 running 即 ENOENT
+  //抛错不增计数——此处同样只在 running 态增计数; 崩溃/超时兜底与 worker 重试竞争时(任务已被对方移回
+  // new)不得重复增计数, 否则配额被虚耗、且超限分支会给仍在排队的任务种下 stale worker_crash 结果
+  const incrementRetry = (subid) => {
+    const rec = records.get(subid);
+    if (!rec) {
+      return 0;
+    };
+    if (rec.state !== 'running') {
+      return rec.task.__retryCount || 0;
+    };
+    rec.task.__retryCount = (rec.task.__retryCount || 0) + 1;
+    return rec.task.__retryCount;
+  };
+
+  const addMessage = (message) => {
+    //对齐 TaskMgr.saveUserMessage 口径: type 不合规拒写(只记日志), level 缺省 info, 补 timestamp
+    if (!MESSAGE_TYPE_PATTERN.test(message.type)) {
+      runLog.error('[Task Error] invalid user message type, skipped', message.type, message.subid);
+      return;
+    };
+    const enriched = { level: 'info', ...message, timestamp: randomsUtil.getDateTimeTxt() };
+    messages.set(messageFname(enriched), enriched);
+    messageSubids.add(enriched.subid);
+  };
+
+  const clearMessages = (subid) => {
+    messages.forEach((message, fname) => {
+      if (message.subid === subid) {
+        messages.delete(fname);
+      };
+    });
+    messageSubids.delete(subid);
+  };
+
+  const getSavedResultType = (subid) => {
+    const entry = results.get(subid);
+    if (!entry) {
+      return null;
+    };
+    return entry.hasError ? 'error' : 'succ';
+  };
+
+  const getState = (subid) => {
+    const rec = records.get(subid);
+    return rec ? rec.state : null;
+  };
+
+  const getTasks = (state) => {
+    assertState(state);
+    if (state === 'new') {
+      return newQueue;
+    };
+    const tasks = [];
+    records.forEach((rec) => {
+      if (rec.state === state) {
+        tasks.push(rec.task);
+      };
+    });
+    return tasks;
+  };
+
+  //口径对齐 file 档进度条注入闭包(Multithread.js `getRemaining: () => TaskMgr.getTasks('new').length`): 只计 new 队列
+  const getRemaining = () => {
+    return newQueue.length;
+  };
+
+  const stats = () => {
+    const failedTasks = [];
+    let succeeded = 0;
+    records.forEach((rec, subid) => {
+      if (rec.state === 'finished') {
+        succeeded++;
+      } else if (rec.state === 'finished_with_errors') {
+        failedTasks.push(subid);
+      };
+    });
+    return {
+      taskFolder,
+      succeeded,
+      failed: failedTasks.length,
+      failedTasks,
+      succButSaveFailed: messageSubids.size,
+    };
+  };
+
+  //按完成顺序遍历某一侧(hasError=false/true)结果, cb(count, task, result);
+  //任务当前不在对应终态(如重试中)时 task 传 null, 对齐 file 档任务文件缺失语义;
+  //result 存的是 undefined 时原样传 undefined, 不跳过
+  const forEachSide = (hasError, state, cb) => {
+    let count = 0;
+    finishedOrder.forEach((subid) => {
+      const entry = results.get(subid);
+      if (!entry || entry.hasError !== hasError) {
+        return;
+      };
+      const rec = records.get(subid);
+      const task = rec && rec.state === state ? rec.task : null;
+      cb(count, task, entry.data);
+      count++;
+    });
+  };
+
+  const forEachSuccResult = (cb) => {
+    forEachSide(false, 'finished', cb);
+  };
+
+  const forEachErrorResult = (cb) => {
+    forEachSide(true, 'finished_with_errors', cb);
+  };
+
+  const forEachResult = (cb) => {//先 succ(succeeded=true) 后 errors(false), count 两段连续编号
+    let offset = 0;
+    forEachSide(false, 'finished', (count, task, result) => {
+      cb(count, task, result, true);
+      offset = count + 1;
+    });
+    forEachSide(true, 'finished_with_errors', (count, task, err) => {
+      cb(offset + count, task, err, false);
+    });
+  };
+
+  return {
+    add,
+    addMany,
+    pop,
+    saveLog,
+    saveResult,
+    deleteResult,
+    move,
+    incrementRetry,
+    addMessage,
+    clearMessages,
+    getSavedResultType,
+    getState,
+    getTasks,
+    getRemaining,
+    stats,
+    forEachSuccResult,
+    forEachErrorResult,
+    forEachResult,
+  };
+};
+
+module.exports = { createMemoryStore };