@eggjs/cluster 3.0.0-beta.0

package/src/utils/messenger.ts

@@ -0,0 +1,199 @@
+import { debuglog } from 'node:util';
+import workerThreads from 'node:worker_threads';
+import type { Master } from '../master.js';
+import type { WorkerManager } from './worker_manager.js';
+
+const debug = debuglog('@eggjs/cluster/messenger');
+
+export type MessageCharacter = 'agent' | 'app' | 'master' | 'parent';
+
+export interface MessageBody {
+  action: string;
+  data?: unknown;
+  to?: MessageCharacter;
+  from?: MessageCharacter;
+  receiverPid?: string;
+}
+
+/**
+ * master messenger, provide communication between parent, master, agent and app.
+ *
+ *             ┌────────┐
+ *             │ parent │
+ *            /└────────┘\
+ *           /     |      \
+ *          /  ┌────────┐  \
+ *         /   │ master │   \
+ *        /    └────────┘    \
+ *       /     /         \    \
+ *     ┌───────┐         ┌───────┐
+ *     │ agent │ ------- │  app  │
+ *     └───────┘         └───────┘
+ *
+ *
+ * in app worker
+ *
+ * ```js
+ * process.send({
+ *   action: 'xxx',
+ *   data: '',
+ *   to: 'agent/master/parent', // default to agent
+ * });
+ * ```
+ *
+ * in agent worker
+ *
+ * ```js
+ * process.send({
+ *   action: 'xxx',
+ *   data: '',
+ *   to: 'app/master/parent', // default to app
+ * });
+ * ```
+ *
+ * in parent
+ *
+ * ```js
+ * process.send({
+ *   action: 'xxx',
+ *   data: '',
+ *   to: 'app/agent/master', // default to master
+ * });
+ * ```
+ */
+export class Messenger {
+  #master: Master;
+  #workerManager: WorkerManager;
+  #hasParent: boolean;
+
+  constructor(master: Master, workerManager: WorkerManager) {
+    this.#master = master;
+    this.#workerManager = workerManager;
+    this.#hasParent = !!workerThreads.parentPort || !!process.send;
+    process.on('message', (msg: MessageBody) => {
+      msg.from = 'parent';
+      this.send(msg);
+    });
+    process.once('disconnect', () => {
+      this.#hasParent = false;
+    });
+  }
+
+  /**
+   * send message
+   * @param {Object} data message body
+   *  - {String} from from who
+   *  - {String} to to who
+   */
+  send(data: MessageBody) {
+    if (!data.from) {
+      data.from = 'master';
+    }
+
+    // https://github.com/eggjs/egg/blob/b6861f1c7548f05a281386050dfeaeb30f236558/lib/core/messenger/ipc.js#L56
+    // recognize receiverPid is to who
+    if (data.receiverPid) {
+      if (data.receiverPid === String(process.pid)) {
+        data.to = 'master';
+      } else if (data.receiverPid === String(this.#workerManager.getAgent()!.workerId)) {
+        data.to = 'agent';
+      } else {
+        data.to = 'app';
+      }
+    }
+
+    // default from -> to rules
+    if (!data.to) {
+      if (data.from === 'agent') {
+        data.to = 'app';
+      }
+      if (data.from === 'app') {
+        data.to = 'agent';
+      }
+      if (data.from === 'parent') {
+        data.to = 'master';
+      }
+    }
+
+    // app -> master
+    // agent -> master
+    if (data.to === 'master') {
+      debug('%s -> master, data: %j', data.from, data);
+      // app/agent to master
+      this.sendToMaster(data);
+      return;
+    }
+
+    // master -> parent
+    // app -> parent
+    // agent -> parent
+    if (data.to === 'parent') {
+      debug('%s -> parent, data: %j', data.from, data);
+      this.sendToParent(data);
+      return;
+    }
+
+    // parent -> master -> app
+    // agent -> master -> app
+    if (data.to === 'app') {
+      debug('%s -> %s, data: %j', data.from, data.to, data);
+      this.sendToAppWorker(data);
+      return;
+    }
+
+    // parent -> master -> agent
+    // app -> master -> agent，可能不指定 to
+    if (data.to === 'agent') {
+      debug('%s -> %s, data: %j', data.from, data.to, data);
+      this.sendToAgentWorker(data);
+      return;
+    }
+  }
+
+  /**
+   * send message to master self
+   * @param {Object} data message body
+   */
+  sendToMaster(data: MessageBody) {
+    this.#master.emit(data.action, data.data);
+  }
+
+  /**
+   * send message to parent process
+   * @param {Object} data message body
+   */
+  sendToParent(data: MessageBody) {
+    if (!this.#hasParent) {
+      return;
+    }
+    process.send!(data);
+  }
+
+  /**
+   * send message to app worker
+   * @param {Object} data message body
+   */
+  sendToAppWorker(data: MessageBody) {
+    for (const worker of this.#workerManager.listWorkers()) {
+      if (worker.state === 'disconnected') {
+        continue;
+      }
+      // check receiverPid
+      if (data.receiverPid && data.receiverPid !== String(worker.workerId)) {
+        continue;
+      }
+      worker.send(data);
+    }
+  }
+
+  /**
+   * send message to agent worker
+   * @param {Object} data message body
+   */
+  sendToAgentWorker(data: MessageBody) {
+    const agent = this.#workerManager.getAgent();
+    if (agent) {
+      agent.send(data);
+    }
+  }
+}

package/src/utils/mode/base/agent.ts

@@ -0,0 +1,90 @@
+import path from 'node:path';
+import { EventEmitter } from 'node:events';
+import type { ChildProcess } from 'node:child_process';
+import type { Worker } from 'node:worker_threads';
+import type { Logger } from 'egg-logger';
+import type { MasterOptions } from '../../../master.js';
+import type { MessageBody, Messenger } from '../../messenger.js';
+import { getSrcDirname } from '../../../dirname.js';
+
+export abstract class BaseAgentWorker<T = ChildProcess | Worker> {
+  instance: T;
+  #instanceId: number;
+  #instanceStatus: string;
+
+  constructor(instance: T) {
+    this.instance = instance;
+  }
+
+  abstract get workerId(): number;
+
+  get id() {
+    return this.#instanceId;
+  }
+
+  set id(id) {
+    this.#instanceId = id;
+  }
+
+  get status() {
+    return this.#instanceStatus;
+  }
+
+  set status(status) {
+    this.#instanceStatus = status;
+  }
+
+  abstract send(message: MessageBody): void;
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  static send(_message: MessageBody) {
+    throw new Error('BaseAgentWorker should implement send.');
+  }
+
+  static kill() {
+    throw new Error('BaseAgentWorker should implement kill.');
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  static gracefulExit(_options: any) {
+    throw new Error('BaseAgentWorker should implement gracefulExit.');
+  }
+}
+
+type LogFun = (msg: any, ...args: any[]) => void;
+
+export abstract class BaseAgentUtils extends EventEmitter {
+  protected options: MasterOptions;
+  protected messenger: Messenger;
+  protected log: LogFun;
+  protected logger: Logger;
+  // public attrs
+  startTime = 0;
+
+  constructor(options: MasterOptions, { log, logger, messenger }: {
+    log: LogFun;
+    logger: Logger;
+    messenger: Messenger;
+  }) {
+    super();
+    this.options = options;
+    this.log = log;
+    this.logger = logger;
+    this.messenger = messenger;
+    // this.instance = null;
+  }
+
+  getAgentWorkerFile() {
+    return path.join(getSrcDirname(), 'agent_worker.js');
+  }
+
+  fork() {
+    throw new Error('BaseAgent should implement fork.');
+  }
+
+  clean() {
+    throw new Error('BaseAgent should implement clean.');
+  }
+
+  abstract kill(timeout: number): Promise<void>;
+}

package/src/utils/mode/base/app.ts

@@ -0,0 +1,115 @@
+import path from 'node:path';
+import { EventEmitter } from 'node:events';
+import type { Worker as ClusterProcessWorker } from 'node:cluster';
+import type { Worker as ThreadWorker } from 'node:worker_threads';
+import type { Logger } from 'egg-logger';
+import type { MessageBody, Messenger } from '../../messenger.js';
+import type { MasterOptions } from '../../../master.js';
+import { getSrcDirname } from '../../../dirname.js';
+
+export abstract class BaseAppWorker<T = ThreadWorker | ClusterProcessWorker> {
+  instance: T;
+  #isDevReload: boolean;
+  #state: string;
+
+  constructor(instance: T) {
+    this.instance = instance;
+  }
+
+  abstract get workerId(): number;
+
+  abstract get id(): number;
+
+  get state() {
+    return this.#state;
+  }
+
+  set state(state) {
+    this.#state = state;
+  }
+
+  abstract get exitedAfterDisconnect(): boolean;
+
+  abstract get exitCode(): number;
+
+  get disableRefork(): boolean {
+    return Reflect.get(this.instance!, 'disableRefork') as boolean;
+  }
+
+  set disableRefork(disableRefork: boolean) {
+    Reflect.set(this.instance!, 'disableRefork', disableRefork);
+  }
+
+  get isDevReload() {
+    return this.#isDevReload;
+  }
+
+  set isDevReload(status) {
+    this.#isDevReload = status;
+  }
+
+  abstract send(data: MessageBody): void;
+
+  clean() {
+    throw new Error('BaseAppWorker should implement clean.');
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  static on(..._args: any[]) {
+    throw new Error('BaseAppWorker should implement on.');
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  static send(_message: MessageBody) {
+    throw new Error('BaseAgentWorker should implement send.');
+  }
+
+  static kill() {
+    throw new Error('BaseAppWorker should implement kill.');
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  static gracefulExit(_options: any) {
+    throw new Error('BaseAgentWorker should implement gracefulExit.');
+  }
+}
+
+type LogFun = (msg: any, ...args: any[]) => void;
+
+export abstract class BaseAppUtils extends EventEmitter {
+  options: MasterOptions;
+  protected messenger: Messenger;
+  protected log: LogFun;
+  protected logger: Logger;
+  protected isProduction: boolean;
+  // public attrs
+  startTime = 0;
+  startSuccessCount = 0;
+  isAllWorkerStarted = false;
+
+  constructor(options: MasterOptions, {
+    log, logger, messenger, isProduction,
+  }: {
+    log: LogFun;
+    logger: Logger;
+    messenger: Messenger;
+    isProduction: boolean;
+  }) {
+    super();
+    this.options = options;
+    this.log = log;
+    this.logger = logger;
+    this.messenger = messenger;
+    this.isProduction = isProduction;
+  }
+
+  getAppWorkerFile() {
+    return path.join(getSrcDirname(), 'app_worker.js');
+  }
+
+  fork() {
+    throw new Error('BaseApp should implement fork.');
+  }
+
+  abstract kill(timeout: number): Promise<void>;
+}

package/src/utils/mode/impl/process/agent.ts

@@ -0,0 +1,118 @@
+import { fork, type ChildProcess, type ForkOptions } from 'node:child_process';
+import { sendmessage } from 'sendmessage';
+import { graceful as gracefulExit, type Options as gracefulExitOptions } from 'graceful-process';
+import { BaseAgentWorker, BaseAgentUtils } from '../../base/agent.js';
+import { terminate } from '../../../terminate.js';
+import type { MessageBody } from '../../../messenger.js';
+import { ClusterAgentWorkerError } from '../../../../error/ClusterAgentWorkerError.js';
+
+export class AgentProcessWorker extends BaseAgentWorker<ChildProcess> {
+  get workerId() {
+    return this.instance.pid!;
+  }
+
+  send(message: MessageBody) {
+    sendmessage(this.instance, message);
+  }
+
+  static send(message: MessageBody) {
+    process.send!(message);
+  }
+
+  static kill() {
+    process.exitCode = 1;
+    process.kill(process.pid);
+  }
+
+  static gracefulExit(options: gracefulExitOptions) {
+    gracefulExit(options);
+  }
+}
+
+export class AgentProcessUtils extends BaseAgentUtils {
+  #agentProcess: ChildProcess;
+  #id = 0;
+  instance: AgentProcessWorker;
+
+  fork() {
+    this.startTime = Date.now();
+
+    const args = [ JSON.stringify(this.options) ];
+    const forkOptions: ForkOptions & { windowsHide?: boolean } = {};
+
+    if (process.platform === 'win32') {
+      forkOptions.windowsHide = true;
+    }
+
+    // add debug execArgv
+    const debugPort = process.env.EGG_AGENT_DEBUG_PORT ?? 5800;
+    if (this.options.isDebug) {
+      forkOptions.execArgv = process.execArgv.concat([ `--inspect-port=${debugPort}` ]);
+    }
+
+    const agentProcess = this.#agentProcess = fork(this.getAgentWorkerFile(), args, forkOptions);
+    const agentWorker = this.instance = new AgentProcessWorker(agentProcess);
+    agentWorker.status = 'starting';
+    agentWorker.id = ++this.#id;
+    this.emit('agent_forked', agentWorker);
+    this.log('[master] agent_worker#%s:%s start with clusterPort:%s',
+      agentWorker.id, agentWorker.workerId, this.options.clusterPort);
+
+    // send debug message
+    if (this.options.isDebug) {
+      this.messenger.send({
+        to: 'parent',
+        from: 'agent',
+        action: 'debug',
+        data: {
+          debugPort,
+          // keep compatibility, should use workerId instead
+          pid: agentWorker.workerId,
+          workerId: agentWorker.workerId,
+        },
+      });
+    }
+    // forwarding agent' message to messenger
+    agentProcess.on('message', (msg: MessageBody | string) => {
+      if (typeof msg === 'string') {
+        msg = {
+          action: msg,
+          data: msg,
+        };
+      }
+      msg.from = 'agent';
+      this.messenger.send(msg);
+    });
+    // logger error event
+    agentProcess.on('error', err => {
+      err.name = 'AgentWorkerError';
+      this.logger.error(new ClusterAgentWorkerError(agentWorker.id, agentWorker.workerId, agentWorker.status, err));
+    });
+    // agent exit message
+    agentProcess.once('exit', (code, signal) => {
+      this.messenger.send({
+        action: 'agent-exit',
+        data: {
+          code,
+          signal,
+        },
+        to: 'master',
+        from: 'agent',
+      });
+    });
+
+    return this;
+  }
+
+  clean() {
+    this.#agentProcess.removeAllListeners();
+  }
+
+  async kill(timeout: number) {
+    if (this.#agentProcess) {
+      this.log('[master] kill agent worker with signal SIGTERM');
+      this.clean();
+      await terminate(this.#agentProcess, timeout);
+    }
+  }
+}

package/src/utils/mode/impl/process/app.ts

@@ -0,0 +1,146 @@
+import cluster, { type Worker as ClusterProcessWorker } from 'node:cluster';
+import { cfork } from 'cfork';
+import { sendmessage } from 'sendmessage';
+import { graceful as gracefulExit, type Options as gracefulExitOptions } from 'graceful-process';
+import { BaseAppWorker, BaseAppUtils } from '../../base/app.js';
+import { terminate } from '../../../terminate.js';
+import type { MessageBody } from '../../../messenger.js';
+
+export class AppProcessWorker extends BaseAppWorker<ClusterProcessWorker> {
+  get id() {
+    return this.instance.id;
+  }
+
+  get workerId() {
+    return this.instance.process.pid!;
+  }
+
+  get exitedAfterDisconnect() {
+    return this.instance.exitedAfterDisconnect;
+  }
+
+  get exitCode() {
+    return this.instance.process.exitCode!;
+  }
+
+  send(message: MessageBody) {
+    sendmessage(this.instance, message);
+  }
+
+  clean() {
+    this.instance.removeAllListeners();
+  }
+
+  static on(event: string, listener: (...args: any[]) => void) {
+    process.on(event, listener);
+  }
+
+  static send(message: MessageBody) {
+    process.send!(message);
+  }
+
+  static kill() {
+    process.exitCode = 1;
+    process.kill(process.pid);
+  }
+
+  static gracefulExit(options: gracefulExitOptions) {
+    gracefulExit(options);
+  }
+}
+
+export class AppProcessUtils extends BaseAppUtils {
+  fork() {
+    this.startTime = Date.now();
+    this.startSuccessCount = 0;
+
+    const args = [ JSON.stringify(this.options) ];
+    this.log('[master] start appWorker with args %j (process)', args);
+    cfork({
+      exec: this.getAppWorkerFile(),
+      args,
+      silent: false,
+      count: this.options.workers,
+      // don't refork in local env
+      refork: this.isProduction,
+      windowsHide: process.platform === 'win32',
+    });
+
+    let debugPort = process.debugPort;
+    cluster.on('fork', worker => {
+      const appWorker = new AppProcessWorker(worker);
+      this.emit('worker_forked', appWorker);
+      appWorker.disableRefork = true;
+      worker.on('message', msg => {
+        if (typeof msg === 'string') {
+          msg = {
+            action: msg,
+            data: msg,
+          };
+        }
+        msg.from = 'app';
+        this.messenger.send(msg);
+      });
+      this.log('[master] app_worker#%s:%s start, state: %s, current workers: %j',
+        appWorker.id, appWorker.workerId, appWorker.state,
+        Object.keys(cluster.workers!));
+
+      // send debug message, due to `brk` scene, send here instead of app_worker.js
+      if (this.options.isDebug) {
+        debugPort++;
+        this.messenger.send({
+          to: 'parent',
+          from: 'app',
+          action: 'debug',
+          data: {
+            debugPort,
+            // keep compatibility, should use workerId instead
+            pid: appWorker.workerId,
+            workerId: appWorker.workerId,
+          },
+        });
+      }
+    });
+    cluster.on('disconnect', worker => {
+      const appWorker = new AppProcessWorker(worker);
+      this.logger.info('[master] app_worker#%s:%s disconnect, suicide: %s, state: %s, current workers: %j',
+        appWorker.id, appWorker.workerId, appWorker.exitedAfterDisconnect, appWorker.state,
+        Object.keys(cluster.workers!));
+    });
+    cluster.on('exit', (worker, code, signal) => {
+      const appWorker = new AppProcessWorker(worker);
+      this.messenger.send({
+        action: 'app-exit',
+        data: {
+          workerId: appWorker.workerId,
+          code,
+          signal,
+        },
+        to: 'master',
+        from: 'app',
+      });
+    });
+    cluster.on('listening', (worker, address) => {
+      const appWorker = new AppProcessWorker(worker);
+      this.messenger.send({
+        action: 'app-start',
+        data: {
+          workerId: appWorker.workerId,
+          address,
+        },
+        to: 'master',
+        from: 'app',
+      });
+    });
+
+    return this;
+  }
+
+  async kill(timeout: number) {
+    await Promise.all(Object.keys(cluster.workers!).map(id => {
+      const worker = cluster.workers![id]!;
+      Reflect.set(worker, 'disableRefork', true);
+      return terminate(worker.process, timeout);
+    }));
+  }
+}