@eggjs/cluster 3.0.0-beta.0

package/src/utils/mode/impl/worker_threads/agent.ts

@@ -0,0 +1,98 @@
+import workerThreads, { type Worker } from 'node:worker_threads';
+import { type Options as gracefulExitOptions } from 'graceful-process';
+import { BaseAgentUtils, BaseAgentWorker } from '../../base/agent.js';
+import type { MessageBody } from '../../../messenger.js';
+import { ClusterAgentWorkerError } from '../../../../error/ClusterAgentWorkerError.js';
+
+export class AgentThreadWorker extends BaseAgentWorker<Worker> {
+  get workerId() {
+    return this.instance.threadId;
+  }
+
+  send(message: MessageBody) {
+    this.instance.postMessage(message);
+  }
+
+  static send(message: MessageBody) {
+    workerThreads.parentPort!.postMessage(message);
+  }
+
+  static kill() {
+    // in worker_threads, process.exit
+    // does not stop the whole program, just the single thread
+    process.exit(1);
+  }
+
+  static gracefulExit(options: gracefulExitOptions) {
+    const { beforeExit } = options;
+    process.on('exit', async code => {
+      if (typeof beforeExit === 'function') {
+        await beforeExit();
+      }
+      process.exit(code);
+    });
+  }
+}
+
+export class AgentThreadUtils extends BaseAgentUtils {
+  #worker: Worker;
+  #id = 0;
+  instance: AgentThreadWorker;
+
+  fork() {
+    this.startTime = Date.now();
+
+    // start agent worker
+    const argv = [ JSON.stringify(this.options) ];
+    const agentPath = this.getAgentWorkerFile();
+    const worker = this.#worker = new workerThreads.Worker(agentPath, { argv });
+
+    // wrap agent worker
+    const agentWorker = this.instance = new AgentThreadWorker(worker);
+    this.emit('agent_forked', agentWorker);
+    agentWorker.status = 'starting';
+    agentWorker.id = ++this.#id;
+    this.log('[master] agent_worker#%s:%s start with worker_threads',
+      agentWorker.id, agentWorker.workerId);
+
+    worker.on('message', msg => {
+      if (typeof msg === 'string') {
+        msg = {
+          action: msg,
+          data: msg,
+        };
+      }
+      msg.from = 'agent';
+      this.messenger.send(msg);
+    });
+
+    worker.on('error', err => {
+      this.logger.error(new ClusterAgentWorkerError(agentWorker.id, agentWorker.workerId, agentWorker.status, err));
+    });
+
+    // agent exit message
+    worker.once('exit', (code: number, signal: string) => {
+      this.messenger.send({
+        action: 'agent-exit',
+        data: {
+          code,
+          signal,
+        },
+        to: 'master',
+        from: 'agent',
+      });
+    });
+  }
+
+  clean() {
+    this.#worker.removeAllListeners();
+  }
+
+  async kill() {
+    if (this.#worker) {
+      this.log(`[master] kill agent worker#${this.#id} (worker_threads) by worker.terminate()`);
+      this.clean();
+      await this.#worker.terminate();
+    }
+  }
+}

package/src/utils/mode/impl/worker_threads/app.ts

@@ -0,0 +1,180 @@
+import { setTimeout as sleep } from 'node:timers/promises';
+import { Worker as ThreadWorker, parentPort, type WorkerOptions } from 'node:worker_threads';
+import type { Options as gracefulExitOptions } from 'graceful-process';
+import { BaseAppWorker, BaseAppUtils } from '../../base/app.js';
+import type { MessageBody } from '../../../messenger.js';
+
+export class AppThreadWorker extends BaseAppWorker<ThreadWorker> {
+  #state = 'none';
+  #id: number;
+
+  constructor(instance: ThreadWorker, id: number) {
+    super(instance);
+    this.#id = id;
+  }
+
+  get id() {
+    return this.#id;
+  }
+
+  get workerId() {
+    return this.instance.threadId;
+  }
+
+  get state() {
+    return this.#state;
+  }
+
+  set state(val) {
+    this.#state = val;
+  }
+
+  get exitedAfterDisconnect() {
+    return true;
+  }
+
+  get exitCode() {
+    return 0;
+    // return this.instance.exitCode;
+  }
+
+  send(message: MessageBody) {
+    this.instance.postMessage(message);
+  }
+
+  clean() {
+    this.instance.removeAllListeners();
+  }
+
+  static on(event: string, listener: (...args: any[]) => void) {
+    parentPort!.on(event, listener);
+  }
+
+  static send(message: MessageBody) {
+    parentPort!.postMessage(message);
+  }
+
+  static kill() {
+    process.exit(1);
+  }
+
+  static gracefulExit(options: gracefulExitOptions) {
+    process.on('exit', async code => {
+      if (typeof options.beforeExit === 'function') {
+        await options.beforeExit();
+      }
+      process.exit(code);
+    });
+  }
+}
+
+export class AppThreadUtils extends BaseAppUtils {
+  #workers: ThreadWorker[] = [];
+
+  #forkSingle(appPath: string, options: WorkerOptions, id: number) {
+    // start app worker
+    const worker = new ThreadWorker(appPath, options);
+    this.#workers.push(worker);
+
+    // wrap app worker
+    const appWorker = new AppThreadWorker(worker, id);
+    this.emit('worker_forked', appWorker);
+    appWorker.disableRefork = true;
+    worker.on('message', (msg: MessageBody) => {
+      if (typeof msg === 'string') {
+        msg = {
+          action: msg,
+          data: msg,
+        };
+      }
+      msg.from = 'app';
+      this.messenger.send(msg);
+    });
+    this.log('[master] app_worker#%s (tid:%s) start', appWorker.id, appWorker.workerId);
+
+    // send debug message, due to `brk` scene, send here instead of app_worker.js
+    let debugPort = process.debugPort;
+    if (this.options.isDebug) {
+      debugPort++;
+      this.messenger.send({
+        to: 'parent',
+        from: 'app',
+        action: 'debug',
+        data: {
+          debugPort,
+          pid: appWorker.workerId,
+          workerId: appWorker.workerId,
+        },
+      });
+    }
+
+    // handle worker listening
+    worker.on('message', ({ action, data: address }) => {
+      if (action !== 'listening') {
+        return;
+      }
+
+      if (!address) {
+        return;
+      }
+
+      appWorker.state = 'listening';
+      this.messenger.send({
+        action: 'app-start',
+        data: {
+          workerId: appWorker.workerId,
+          address,
+        },
+        to: 'master',
+        from: 'app',
+      });
+
+    });
+
+    // handle worker exit
+    worker.on('exit', async code => {
+      appWorker.state = 'dead';
+      this.messenger.send({
+        action: 'app-exit',
+        data: {
+          workerId: appWorker.workerId,
+          code,
+        },
+        to: 'master',
+        from: 'app',
+      });
+
+      // refork app worker
+      await sleep(1000);
+      this.#forkSingle(appPath, options, id);
+    });
+  }
+
+  fork() {
+    this.startTime = Date.now();
+    this.startSuccessCount = 0;
+
+    const ports = this.options.ports ?? [];
+    if (!ports.length) {
+      ports.push(this.options.port!);
+    }
+    this.options.workers = ports.length;
+    let i = 0;
+    do {
+      const options = Object.assign({}, this.options, { port: ports[i] });
+      const argv = [ JSON.stringify(options) ];
+      this.#forkSingle(this.getAppWorkerFile(), { argv }, ++i);
+    } while (i < ports.length);
+
+    return this;
+  }
+
+  async kill() {
+    for (const worker of this.#workers) {
+      const id = Reflect.get(worker, 'id');
+      this.log(`[master] kill app worker#${id} (worker_threads) by worker.terminate()`);
+      worker.removeAllListeners();
+      worker.terminate();
+    }
+  }
+}

package/src/utils/options.ts

@@ -0,0 +1,169 @@
+import os from 'node:os';
+import fs from 'node:fs';
+import path from 'node:path';
+import assert from 'node:assert';
+import { SecureContextOptions } from 'node:tls';
+import { getFrameworkPath, importModule } from '@eggjs/utils';
+
+export interface ClusterHTTPSSecureOptions {
+  key: SecureContextOptions['key'];
+  cert: SecureContextOptions['cert'];
+  ca?: SecureContextOptions['ca'];
+  passphrase?: SecureContextOptions['passphrase'];
+}
+
+export type ClusterStartMode = 'process' | 'worker_threads';
+
+/** Cluster start options */
+export interface ClusterOptions {
+  /**
+   * specify framework that can be absolute path or npm package
+   */
+  framework?: string;
+  /**
+   * @deprecated please use framework instead
+   */
+  customEgg?: string;
+  /** directory of application, default to `process.cwd()` */
+  baseDir?: string;
+  /**
+   * numbers of app workers, default to `os.cpus().length`
+   */
+  workers?: number | string;
+  /**
+   * listening port, default to `7001`(http) or `8443`(https)
+   */
+  port?: number | string | null;
+  /**
+   * listening a debug port on http protocol
+   */
+  debugPort?: number;
+  /**
+   * https options, { key, cert, ca }, full path
+   */
+  https?: ClusterHTTPSSecureOptions | boolean;
+  /**
+   * @deprecated please use `options.https.key` instead
+   */
+  key?: ClusterHTTPSSecureOptions['key'];
+  /**
+   * @deprecated please use `options.https.cert` instead
+   */
+  cert?: ClusterHTTPSSecureOptions['cert'];
+  /**
+   * will inject into worker/agent process
+   */
+  require?: string | string[];
+  /**
+   * will save master pid to this file
+   */
+  pidFile?: string;
+  /**
+   * custom env, default is `process.env.EGG_SERVER_ENV`
+   */
+  env?: string;
+  /**
+   * default is `'process'`, use `'worker_threads'` to start the app & agent worker by worker_threads
+   */
+  startMode?: ClusterStartMode;
+  /**
+   * startup port of each app worker, such as: `[7001, 7002, 7003]`, only effects when the startMode is `'worker_threads'`
+   */
+  ports?: number[];
+  /**
+   * sticky mode server
+   */
+  sticky?: boolean;
+  /** customized plugins, for unittest */
+  plugins?: object;
+  isDebug?: boolean;
+}
+
+export interface ParsedClusterOptions extends ClusterOptions {
+  port?: number;
+  baseDir: string;
+  workers: number;
+  framework: string;
+  startMode: ClusterStartMode;
+}
+
+export async function parseOptions(options?: ClusterOptions) {
+  options = {
+    baseDir: process.cwd(),
+    port: options?.https ? 8443 : undefined,
+    startMode: 'process',
+    // ports: [],
+    env: process.env.EGG_SERVER_ENV,
+    ...options,
+  };
+
+  const pkgPath = path.join(options.baseDir!, 'package.json');
+  assert(fs.existsSync(pkgPath), `${pkgPath} should exist`);
+
+  options.framework = getFrameworkPath({
+    baseDir: options.baseDir!,
+    // compatible customEgg only when call startCluster directly without framework
+    framework: options.framework ?? options.customEgg,
+  });
+
+  const egg = await importModule(options.framework);
+  assert(egg.Application, `should define Application in ${options.framework}`);
+  assert(egg.Agent, `should define Agent in ${options.framework}`);
+
+  if (options.https === true) {
+    // Keep compatible options.key, options.cert
+    console.warn('[@eggjs/cluster:deprecated] [master] Please use `https: { key, cert, ca }` instead of `https: true`');
+    options.https = {
+      key: options.key,
+      cert: options.cert,
+    };
+  }
+
+  // https
+  if (options.https) {
+    assert(options.https.key, 'options.https.key should exists');
+    if (typeof options.https.key === 'string') {
+      assert(fs.existsSync(options.https.key), 'options.https.key file should exists');
+    }
+    assert(options.https.cert, 'options.https.cert should exists');
+    if (typeof options.https.cert === 'string') {
+      assert(fs.existsSync(options.https.cert), 'options.https.cert file should exists');
+    }
+    if (typeof options.https.ca === 'string') {
+      assert(fs.existsSync(options.https.ca), 'options.https.ca file should exists');
+    }
+  }
+
+  if (options.port && typeof options.port === 'string') {
+    options.port = parseInt(options.port);
+  }
+  if (options.port === null) {
+    options.port = undefined;
+  }
+
+  if (options.workers && typeof options.workers === 'string') {
+    options.workers = parseInt(options.workers);
+  }
+  if (!options.workers) {
+    options.workers = os.cpus().length;
+  }
+
+  if (options.require) {
+    if (typeof options.require === 'string') {
+      options.require = [ options.require ];
+    }
+  }
+
+  // don't print deprecated message in production env.
+  // it will print to stderr.
+  if (process.env.NODE_ENV === 'production') {
+    process.env.NO_DEPRECATION = '*';
+  }
+
+  const isDebug = process.execArgv.some(argv => argv.includes('--debug') || argv.includes('--inspect'));
+  if (isDebug) {
+    options.isDebug = isDebug;
+  }
+
+  return options as ParsedClusterOptions;
+}

package/src/utils/terminate.ts

@@ -0,0 +1,97 @@
+import { debuglog } from 'node:util';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { once } from 'node:events';
+import { ChildProcess } from 'node:child_process';
+import { pstree } from '@fengmk2/ps-tree';
+
+const debug = debuglog('@eggjs/cluster/utils/terminate');
+
+interface SubProcess extends ChildProcess {
+  process?: ChildProcess;
+}
+
+export async function terminate(subProcess: SubProcess, timeout: number) {
+  const pid = subProcess.process?.pid ?? subProcess.pid;
+  const childPids = await getChildPids(pid!);
+  await Promise.all([
+    killProcess(subProcess, timeout),
+    killChildren(childPids, timeout),
+  ]);
+}
+
+// kill process, if SIGTERM not work, try SIGKILL
+async function killProcess(subProcess: SubProcess, timeout: number) {
+  // https://github.com/nodejs/node/pull/34312
+  (subProcess.process ?? subProcess).kill('SIGTERM');
+  await Promise.race([
+    once(subProcess, 'exit'),
+    sleep(timeout),
+  ]);
+  if (subProcess.killed) {
+    return;
+  }
+  // SIGKILL: http://man7.org/linux/man-pages/man7/signal.7.html
+  // worker: https://github.com/nodejs/node/blob/master/lib/internal/cluster/worker.js#L22
+  // subProcess.kill is wrapped to subProcess.destroy, it will wait to disconnected.
+  (subProcess.process ?? subProcess).kill('SIGKILL');
+}
+
+// kill all children processes, if SIGTERM not work, try SIGKILL
+async function killChildren(childrenPids: number[], timeout: number) {
+  if (childrenPids.length === 0) {
+    return;
+  }
+  kill(childrenPids, 'SIGTERM');
+
+  const start = Date.now();
+  // if timeout is 1000, it will check twice.
+  const checkInterval = 400;
+  let unterminated: number[] = [];
+
+  while (Date.now() - start < timeout - checkInterval) {
+    await sleep(checkInterval);
+    unterminated = getUnterminatedProcesses(childrenPids);
+    if (unterminated.length === 0) {
+      return;
+    }
+  }
+  kill(unterminated, 'SIGKILL');
+}
+
+async function getChildPids(pid: number) {
+  let childrenPids: number[] = [];
+  try {
+    const children = await pstree(pid);
+    childrenPids = children!.map(c => parseInt(c.PID));
+  } catch (err) {
+    // if get children error, just ignore it
+    debug('pstree %s error: %s, ignore it', pid, err);
+  }
+  return childrenPids;
+}
+
+function kill(pids: number[], signal: string) {
+  for (const pid of pids) {
+    try {
+      process.kill(pid, signal);
+    } catch (err) {
+      // ignore
+      debug('kill %s error: %s, signal: %s, ignore it', pid, err, signal);
+    }
+  }
+}
+
+function getUnterminatedProcesses(pids: number[]) {
+  return pids.filter(pid => {
+    try {
+      // success means it's still alive
+      process.kill(pid, 0);
+      return true;
+    } catch (err) {
+      // error means it's dead
+      debug('kill %s error: %s, it still alive', pid, err);
+      return false;
+    }
+  });
+}
+

package/src/utils/worker_manager.ts

@@ -0,0 +1,87 @@
+import { EventEmitter } from 'node:events';
+import { BaseAgentWorker } from './mode/base/agent.js';
+import { BaseAppWorker } from './mode/base/app.js';
+
+// worker manager to record agent and worker forked by egg-cluster
+// can do some check stuff here to monitor the healthy
+export class WorkerManager extends EventEmitter {
+  agent: BaseAgentWorker | null;
+  workers = new Map<number, BaseAppWorker>();
+  exception = 0;
+  timer: NodeJS.Timeout;
+
+  constructor() {
+    super();
+    this.agent = null;
+  }
+
+  getWorkers() {
+    return Array.from(this.workers.keys());
+  }
+
+  setAgent(agent: BaseAgentWorker) {
+    this.agent = agent;
+  }
+
+  getAgent() {
+    return this.agent;
+  }
+
+  deleteAgent() {
+    this.agent = null;
+  }
+
+  setWorker(worker: BaseAppWorker) {
+    this.workers.set(worker.workerId, worker);
+  }
+
+  getWorker(workerId: number) {
+    return this.workers.get(workerId);
+  }
+
+  deleteWorker(workerId: number) {
+    this.workers.delete(workerId);
+  }
+
+  listWorkerIds() {
+    return Array.from(this.workers.keys());
+  }
+
+  listWorkers() {
+    return Array.from(this.workers.values());
+  }
+
+  getListeningWorkerIds() {
+    const keys = [];
+    for (const [ id, worker ] of this.workers.entries()) {
+      if (worker.state === 'listening') {
+        keys.push(id);
+      }
+    }
+    return keys;
+  }
+
+  count() {
+    return {
+      agent: this.agent?.status === 'started' ? 1 : 0,
+      worker: this.listWorkerIds().length,
+    };
+  }
+
+  // check agent and worker must both alive
+  // if exception appear 3 times, emit an exception event
+  startCheck() {
+    this.timer = setInterval(() => {
+      const count = this.count();
+      if (count.agent > 0 && count.worker > 0) {
+        this.exception = 0;
+        return;
+      }
+      this.exception++;
+      if (this.exception >= 3) {
+        this.emit('exception', count);
+        clearInterval(this.timer);
+      }
+    }, 10000);
+  }
+}