@colyseus/tools 0.16.0-preview.2 → 0.16.0-preview.4

package/pm2/post-deploy-agent.js

@@ -0,0 +1,315 @@
+/**
+ * PM2 Agent for no downtime deployments on Colyseus Cloud.
+ *
+ * How it works:
+ * - New process(es) are spawned (MAX_ACTIVE_PROCESSES/2)
+ * - NGINX configuration is updated so new traffic only goes through the new process
+ * - Old processes are asynchronously and gracefully stopped.
+ * - The rest of the processes are spawned/reactivated.
+ */
+const pm2 = require('pm2');
+const fs = require('fs');
+const cst = require('pm2/constants');
+const io = require('@pm2/io');
+const path = require('path');
+const shared = require('./shared');
+
+const opts = { env: process.env.NODE_ENV || "production", };
+let config = undefined;
+
+io.initModule({
+  pid: path.resolve('/var/run/colyseus-agent.pid'),
+  widget: {
+    type: 'generic',
+    logo: 'https://colyseus.io/images/logos/logo-dark-color.png',
+    theme : ['#9F1414', '#591313', 'white', 'white'],
+  }
+});
+
+pm2.connect(function(err) {
+  if (err) {
+    console.error(err.stack || err);
+    process.exit();
+  }
+  console.log('PM2 post-deploy agent is up and running...');
+
+  /**
+   * Remote actions
+   */
+  io.action('post-deploy', async function (arg0, reply) {
+    const [cwd, ecosystemFilePath] = arg0.split(':');
+    console.log("Received 'post-deploy' action!", { cwd, config: ecosystemFilePath });
+
+    let replied = false;
+
+    //
+    // Override 'reply' to decrement amount of concurrent deployments
+    //
+    const onReply = function() {
+      if (replied) { return; }
+      replied = true;
+      reply.apply(null, arguments);
+    }
+
+    try {
+      config = await shared.getAppConfig(ecosystemFilePath);
+      opts.cwd = cwd;
+      postDeploy(cwd, onReply);
+
+    } catch (err) {
+      onReply({ success: false, message: err?.message });
+    }
+  });
+});
+
+const restartingAppIds = new Set();
+
+function postDeploy(cwd, reply) {
+  shared.listApps(function(err, apps) {
+    if (err) {
+      console.error(err);
+      return reply({ success: false, message: err?.message });
+    }
+
+    // first deploy, start all processes
+    if (apps.length === 0) {
+      return pm2.start(config, {...opts}, (err, result) => {
+        reply({ success: !err, message: err?.message });
+        updateAndSave(err, result);
+      });
+    }
+
+    //
+    // detect if cwd has changed, and restart PM2 if it has
+    //
+    if (apps[0].pm2_env.pm_cwd !== cwd) {
+      console.log("App Root Directory changed. Restarting may take a bit longer...");
+
+      //
+      // remove all and start again with new cwd
+      //
+      return pm2.delete('all', function (err) {
+        logIfError(err);
+
+        // start again
+        pm2.start(config, { ...opts }, (err, result) => {
+          reply({ success: !err, message: err?.message });
+          updateAndSave(err, result);
+        });
+      });
+    }
+
+    /**
+     * Graceful restart logic:
+     * List of PM2 app envs to stop or restart
+     */
+    const appsToStop = [];
+    const appsStopped = [];
+    let numAppsStopping = 0;
+    let numTotalApps = undefined;
+
+    apps.forEach((app) => {
+      const env = app.pm2_env;
+
+      /**
+       * Asynchronously teardown/stop processes with active connections
+       */
+      if (env.status === cst.STOPPED_STATUS) {
+        appsStopped.push(env);
+
+      } else if (env.status !== cst.STOPPING_STATUS) {
+        appsToStop.push(env);
+
+      } else if (!restartingAppIds.has(env.pm_id)) {
+        numAppsStopping++;
+      }
+    });
+
+    /**
+     * - Start new process
+     * - Update NGINX config to expose only the new process
+     * - Stop old processes
+     * - Spawn/reactivate the rest of the processes (shared.MAX_ACTIVE_PROCESSES)
+     */
+    const onFirstAppsStart = async (initialApps, err, result) => {
+      /**
+       * release post-deploy action while proceeding with graceful restart of other processes
+       */
+      reply({ success: !err, message: err?.message });
+
+      if (err) { return console.error(err); }
+
+      let numActiveApps = initialApps.length + restartingAppIds.size;
+
+      /**
+       * - Write NGINX config to expose only the new active process
+       * - The old ones processes will go down asynchronously (or will be restarted)
+       */
+      writeNginxConfig(initialApps);
+
+      //
+      // Wait 1.5 seconds to ensure NGINX is updated & reloaded
+      //
+      await new Promise(resolve => setTimeout(resolve, 1500));
+
+      //
+      // Asynchronously stop/restart apps with active connections
+      // (They make take from minutes up to hours to stop)
+      //
+      appsToStop.forEach((app_env) => {
+        if (numActiveApps < shared.MAX_ACTIVE_PROCESSES) {
+          numActiveApps++;
+
+          restartingAppIds.add(app_env.pm_id);
+          pm2.restart(app_env.pm_id, (err, _) => {
+            restartingAppIds.delete(app_env.pm_id);
+            if (err) { return logIfError(err); }
+
+            // reset counter stats (restart_time=0)
+            pm2.reset(app_env.pm_id, logIfError);
+          });
+
+        } else {
+          pm2.stop(app_env.pm_id, logIfError);
+        }
+      });
+
+      if (numActiveApps < shared.MAX_ACTIVE_PROCESSES) {
+        const missingOnlineApps = shared.MAX_ACTIVE_PROCESSES - numActiveApps;
+
+        // console.log("Active apps is lower than MAX_ACTIVE_PROCESSES, will SCALE again =>", {
+        //   missingOnlineApps,
+        //   numActiveApps,
+        //   newNumTotalApps: numTotalApps + missingOnlineApps
+        // });
+
+        pm2.scale(apps[0].name, numTotalApps + missingOnlineApps, updateAndSaveIfAllRunning);
+      }
+    };
+
+    const numHalfMaxActiveProcesses = Math.ceil(shared.MAX_ACTIVE_PROCESSES / 2);
+
+    /**
+     * Re-use previously stopped apps if available
+     */
+    if (appsStopped.length >= numHalfMaxActiveProcesses) {
+      const initialApps = appsStopped.splice(0, numHalfMaxActiveProcesses);
+
+      let numSucceeded = 0;
+      initialApps.forEach((app_env) => {
+        // console.log("pm2.restart => ", app_env.pm_id);
+
+        restartingAppIds.add(app_env.pm_id);
+        pm2.restart(app_env.pm_id, (err) => {
+          restartingAppIds.delete(app_env.pm_id);
+          if (err) { return replyIfError(err, reply); }
+
+          // reset counter stats (restart_time=0)
+          pm2.reset(app_env.pm_id, logIfError);
+
+          // TODO: set timeout here to exit if some processes are not restarting
+
+          numSucceeded++;
+          if (numSucceeded === initialApps.length) {
+            onFirstAppsStart(initialApps);
+          }
+        });
+      });
+
+    } else {
+      /**
+       * Increment to +(MAX/2) processes
+       */
+      let LAST_NODE_APP_INSTANCE = apps[apps.length - 1].pm2_env.NODE_APP_INSTANCE;
+      const initialApps = Array.from({ length: numHalfMaxActiveProcesses }).map((_, i) => {
+        const new_app_env = Object.assign({}, apps[0].pm2_env);
+        new_app_env.NODE_APP_INSTANCE = ++LAST_NODE_APP_INSTANCE;
+        return new_app_env;
+      });
+
+      numTotalApps = apps.length + numHalfMaxActiveProcesses;
+
+      // Ensure to scale to a number of processes where `numHalfMaxActiveProcesses` can start immediately.
+      pm2.scale(apps[0].name, numTotalApps, onFirstAppsStart.bind(undefined, initialApps));
+    }
+  });
+}
+
+function updateAndSave() {
+  // console.log("updateAndExit");
+  updateAndReloadNginx(() => complete());
+}
+
+function updateAndSaveIfAllRunning(err) {
+  if (err) { return console.error(err); }
+
+  updateAndReloadNginx((app_envs) => {
+    // console.log("updateAndExitIfAllRunning, app_ids (", app_envs.map(app_env => app_env.NODE_APP_INSTANCE) ,") => ", app_envs.length, "/", shared.MAX_ACTIVE_PROCESSES);
+
+    //
+    // TODO: add timeout to exit here, in case some processes are not starting
+    //
+    if (app_envs.length === shared.MAX_ACTIVE_PROCESSES) {
+      complete();
+    }
+  });
+}
+
+function updateAndReloadNginx(cb) {
+  //
+  // If you are self-hosting and reading this file, consider using the
+  // following in your self-hosted environment:
+  //
+  // #!/bin/bash
+  // # Requires fswatch (`apt install fswatch`)
+  // # Reload NGINX when colyseus_servers.conf changes
+  // fswatch /etc/nginx/colyseus_servers.conf -m poll_monitor --event=Updated | while read event
+  // do
+  //     service nginx reload
+  // done
+
+  shared.listApps(function(err, apps) {
+    if (apps.length === 0) { err = "no apps running."; }
+    if (err) { return console.error(err); }
+
+    const app_envs = apps
+      .filter(app => app.pm2_env.status !== cst.STOPPING_STATUS && app.pm2_env.status !== cst.STOPPED_STATUS)
+      .map((app) => app.pm2_env);
+
+    writeNginxConfig(app_envs);
+
+    cb?.(app_envs);
+  });
+}
+
+function writeNginxConfig(app_envs) {
+  // console.log("writeNginxConfig: ", app_envs.map(app_env => app_env.NODE_APP_INSTANCE));
+
+  const port = 2567;
+  const addresses = [];
+
+  app_envs.forEach(function(app_env) {
+    addresses.push(`unix:${shared.PROCESS_UNIX_SOCK_PATH}${port + app_env.NODE_APP_INSTANCE}.sock`);
+  });
+
+  // write NGINX config
+  fs.writeFileSync(shared.NGINX_SERVERS_CONFIG_FILE, addresses.map(address => `server ${address};`).join("\n"), logIfError);
+}
+
+function complete() {
+  // "pm2 save"
+  pm2.dump(logIfError);
+}
+
+function logIfError (err) {
+  if (err) {
+    console.error(err);
+  }
+}
+
+function replyIfError(err, reply) {
+  if (err) {
+    console.error(err);
+    reply({ success: false, message: err?.message });
+  }
+}

package/pm2/shared.js

@@ -0,0 +1,73 @@
+const pm2 = require('pm2');
+const os = require('os');
+
+const NAMESPACE = 'cloud';
+const MAX_ACTIVE_PROCESSES = os.cpus().length;
+
+function listApps(callback) {
+  pm2.list((err, apps) => {
+    if (err) { return callback(err);; }
+
+    // Filter out @colyseus/tools module (PM2 post-deploy agent)
+    apps = apps.filter(app => app.name !== '@colyseus/tools');
+
+    callback(err, apps);
+  });
+}
+
+async function getAppConfig(ecosystemFilePath) {
+  const module = await import(ecosystemFilePath);
+  const config = module.default;
+
+  /**
+   * Tune PM2 app config
+   */
+  if (config.apps && config.apps.length >= 0) {
+    const app = config.apps[0];
+
+    // app.name = "colyseus-app";
+    app.namespace = NAMESPACE;
+    app.exec_mode = "fork";
+
+    app.instances = MAX_ACTIVE_PROCESSES;
+
+    app.time = true;
+    app.wait_ready = true;
+    app.watch = false;
+
+    // default: merge logs into a single file
+    if (app.merge_logs === undefined) {
+      app.merge_logs = true;
+    }
+
+    // default: wait for 30 minutes before forcibly killing
+    // (prevent forcibly killing while rooms are still active)
+    if (!app.kill_timeout) {
+      app.kill_timeout = 30 * 60 * 1000;
+    }
+
+    // default: retry kill after 1 second
+    if (!app.kill_retry_time) {
+      app.kill_retry_time = 5000;
+    }
+  }
+
+  return config;
+}
+
+module.exports = {
+  /**
+   * Constants
+   */
+  NGINX_SERVERS_CONFIG_FILE: '/etc/nginx/colyseus_servers.conf',
+  PROCESS_UNIX_SOCK_PATH: '/run/colyseus/',
+
+  MAX_ACTIVE_PROCESSES,
+  NAMESPACE,
+
+  /**
+   * Shared methods
+   */
+  listApps,
+  getAppConfig,
+}

package/post-deploy.js

@@ -1,17 +1,10 @@
 #!/usr/bin/env node
 const pm2 = require('pm2');
-const os = require('os');
 const fs = require('fs');
 const path = require('path');
+const shared = require('./pm2/shared');
 
-const opts = { env: process.env.NODE_ENV || "production" };
-const maxCPU = os.cpus().length;
-
-// // allow deploying from other path as root.
-// if (process.env.npm_config_local_prefix) {
-//   process.chdir(process.env.npm_config_local_prefix);
-//   pm2.cwd = process.env.npm_config_local_prefix;
-// }
+const opts = { env: process.env.NODE_ENV || "production", };
 
 const CONFIG_FILE = [
   'ecosystem.config.cjs',
@@ -20,88 +13,125 @@ const CONFIG_FILE = [
   'pm2.config.js',
 ].find((filename) => fs.existsSync(path.resolve(pm2.cwd, filename)));
 
+/**
+ * TODO: if not provided, auto-detect entry-point & dynamically generate ecosystem config
+ */
 if (!CONFIG_FILE) {
   throw new Error('missing ecosystem config file. make sure to provide one with a valid "script" entrypoint file path.');
 }
 
-pm2.list(function(err, apps) {
-  bailOnErr(err);
+const CONFIG_FILE_PATH = `${pm2.cwd}/${CONFIG_FILE}`;
 
-  if (apps.length === 0) {
-    // first deploy
-    pm2.start(CONFIG_FILE, {...opts}, onAppRunning);
+/**
+ * Try to handle post-deploy via PM2 module first (pm2 install @colyseus/tools)
+ * If not available, fallback to legacy post-deploy script.
+ */
+pm2.trigger('@colyseus/tools', 'post-deploy', `${pm2.cwd}:${CONFIG_FILE_PATH}`, async function (err, result) {
+  if (err) {
+    console.log("Proceeding with legacy post-deploy script...");
+    postDeploy();
 
   } else {
+    if (result[0].data?.return?.success === false) {
+      console.error(result[0].data?.return?.message);
+      process.exit(1);
+    } else {
+      console.log("Post-deploy success.");
+      process.exit();
+    }
+  }
+});
 
-    //
-    // detect if cwd has changed, and restart PM2 if it has
-    //
-    if (apps[0].pm2_env.pm_cwd !== pm2.cwd) {
+async function postDeploy() {
+  shared.listApps(function (err, apps) {
+    bailOnErr(err);
 
+    if (apps.length === 0) {
       //
-      // remove all and start again with new cwd
+      // first deploy
       //
-      pm2.delete('all', function(err) {
-
-        // kill & start again
-        pm2.kill(function() {
-          pm2.start(CONFIG_FILE, { ...opts }, onAppRunning);
-        });
-
-      });
+      pm2.start(CONFIG_FILE_PATH, { ...opts }, () => onAppRunning());
 
     } else {
+
       //
-      // reload existing apps
+      // detect if cwd has changed, and restart PM2 if it has
       //
-      pm2.reload(CONFIG_FILE, {...opts}, function(err, apps) {
-        bailOnErr(err);
-
-        const name = apps[0].name;
-
-        // scale app to use all CPUs available
-        if (apps.length !== maxCPU) {
-          pm2.scale(name, maxCPU, onAppRunning);
-
-        } else {
-          onAppRunning();
-        }
-      });
+      if (apps[0].pm2_env.pm_cwd !== pm2.cwd) {
+        //
+        // remove all and start again with new cwd
+        //
+        restartAll();
+
+      } else {
+        //
+        // reload existing apps
+        //
+        reloadAll();
+      }
     }
+  });
+}
 
+function onAppRunning(reloadedAppIds) {
+  // reset reloaded app stats
+  if (reloadedAppIds) {
+    resetAppStats(reloadedAppIds);
   }
-});
-
-function onAppRunning() {
-  updateColyseusBootService();
   updateAndReloadNginx();
 }
 
-function updateColyseusBootService() {
-  //
-  // Update colyseus-boot.service to use the correct paths for the application
-  //
+function restartAll () {
+  pm2.delete('all', function (err) {
+    // kill & start again
+    pm2.kill(function () {
+      pm2.start(CONFIG_FILE_PATH, { ...opts }, () => onAppRunning());
+    });
+  });
+}
 
-  const COLYSEUS_CLOUD_BOOT_SERVICE = '/etc/systemd/system/colyseus-boot.service';
+function reloadAll(retry = 0) {
+  pm2.reload(CONFIG_FILE_PATH, { ...opts }, function (err, apps) {
+    if (err) {
+      //
+      // Retry in case of "Reload in progress" error.
+      //
+      if (err.message === 'Reload in progress' && retry < 5) {
+        console.warn(err.message, ", retrying...");
+        setTimeout(() => reloadAll(retry + 1), 1000);
 
-  // ignore if no boot service found
-  if (!fs.existsSync(COLYSEUS_CLOUD_BOOT_SERVICE)) {
-    return;
-  }
+      } else {
+        bailOnErr(err);
+      }
 
-  const workingDirectory = pm2.cwd;
-  const execStart = `${detectPackageManager()} colyseus-post-deploy`;
+      return;
+    }
 
-  const contents = fs.readFileSync(COLYSEUS_CLOUD_BOOT_SERVICE, 'utf8');
-  try {
-    fs.writeFileSync(COLYSEUS_CLOUD_BOOT_SERVICE, contents
-      .replace(/WorkingDirectory=(.*)/, `WorkingDirectory=${workingDirectory}`)
-      .replace(/ExecStart=(.*)/, `ExecStart=${execStart}`));
-  } catch (e) {
-    // couldn't write to file
-  }
+    const name = apps[0].name;
+    const reloadedAppIds = apps.map(app => app.pm_id);
+
+    // scale app to use all CPUs available
+    if (apps.length !== shared.MAX_ACTIVE_PROCESSES) {
+      pm2.scale(name, shared.MAX_ACTIVE_PROCESSES, () => onAppRunning(reloadedAppIds));
+
+    } else {
+      onAppRunning(reloadedAppIds);
+    }
+  });
 }
 
+function resetAppStats (reloadedAppIds) {
+  reloadedAppIds.forEach((pm_id) => {
+    pm2.reset(pm_id, (err, _) => {
+      if (err) {
+        console.error(err);
+      } else {
+        console.log(`metrics re-set for app_id: ${pm_id}`);
+      }
+    });
+  });
+};
+
 function updateAndReloadNginx() {
   //
   // If you are self-hosting and reading this file, consider using the
@@ -115,9 +145,7 @@ function updateAndReloadNginx() {
   //     service nginx reload
   // done
 
-  const NGINX_SERVERS_CONFIG_FILE = '/etc/nginx/colyseus_servers.conf';
-
-  pm2.list(function(err, apps) {
+  shared.listApps(function(err, apps) {
     if (apps.length === 0) {
       err = "no apps running.";
     }
@@ -127,11 +155,11 @@ function updateAndReloadNginx() {
     const addresses = [];
 
     apps.forEach(function(app) {
-      addresses.push(`unix:/run/colyseus/${ port + app.pm2_env.NODE_APP_INSTANCE }.sock`);
+      addresses.push(`unix:${shared.PROCESS_UNIX_SOCK_PATH}${port + app.pm2_env.NODE_APP_INSTANCE}.sock`);
     });
 
     // write NGINX config
-    fs.writeFileSync(NGINX_SERVERS_CONFIG_FILE, addresses.map(address => `server ${address};`).join("\n"), bailOnErr);
+    fs.writeFileSync(shared.NGINX_SERVERS_CONFIG_FILE, addresses.map(address => `server ${address};`).join("\n"), bailOnErr);
 
     // "pm2 save"
     pm2.dump(function (err, ret) {
@@ -144,30 +172,6 @@ function updateAndReloadNginx() {
   });
 }
 
-function detectPackageManager() {
-  const lockfiles = {
-    // npm
-    "npm exec": path.resolve(pm2.cwd, 'package-lock.json'),
-
-    // yarn
-    "yarn exec": path.resolve(pm2.cwd, 'yarn.lock'),
-
-    // pnpm
-    "pnpm exec": path.resolve(pm2.cwd, 'pnpm-lock.yaml'),
-
-    // bun
-    "bunx": path.resolve(pm2.cwd, 'bun.lockb'),
-  };
-
-  for (const [key, value] of Object.entries(lockfiles)) {
-    if (fs.existsSync(value)) {
-      return key;
-    }
-  }
-
-  return "npm";
-}
-
 function bailOnErr(err) {
   if (err) {
     console.error(err);
@@ -176,4 +180,3 @@ function bailOnErr(err) {
     process.exit(1);
   }
 }
-