@colyseus/tools 0.16.0-preview.2 → 0.16.0-preview.4

package/report-stats.js

@@ -0,0 +1,178 @@
+#!/usr/bin/env node
+
+const fs = require('fs');
+const net = require('net');
+const pm2 = require('pm2');
+
+const COLYSEUS_CLOUD_URL = `${process.env.ENDPOINT}/vultr/stats`;
+
+const FAILED_ATTEMPS_FILE = "/var/tmp/pm2-stats-attempts.txt";
+const FETCH_TIMEOUT = 7000;
+
+async function retryFailedAttempts() {
+  /**
+   * Retry cached failed attempts
+   */
+  if (!fs.existsSync(FAILED_ATTEMPS_FILE)) {
+    return;
+  }
+
+  // Retry up to last 30 attempts
+  const failedAttempts = fs.readFileSync(FAILED_ATTEMPS_FILE, "utf8").split("\n").slice(-30);
+
+  for (const body of failedAttempts) {
+    // skip if empty
+    if (!body) { continue; }
+
+    try {
+      await fetch(COLYSEUS_CLOUD_URL, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "Authorization": `Bearer ${process.env.COLYSEUS_SECRET}`,
+          "ngrok-skip-browser-warning": "yes",
+        },
+        body,
+        signal: AbortSignal.timeout(FETCH_TIMEOUT)
+      });
+    } catch (e) {
+      console.error(e);
+    }
+  }
+
+  fs.unlinkSync(FAILED_ATTEMPS_FILE);
+}
+
+async function fetchRetry(url, options, retries = 3) {
+  try {
+    return await fetch(url, options)
+  } catch (err) {
+    if (retries === 1) throw err;
+    return await fetchRetry(url, options, retries - 1);
+  }
+};
+
+pm2.Client.executeRemote('getMonitorData', {}, async function(err, list) {
+  if (err) { return console.error(err); }
+
+  //
+  // Filter out:
+  // - @colyseus/tools module (PM2 agent)
+  // - "stopped" processes (gracefully shut down)
+  //
+  list = list.filter((item) => {
+    return (
+      item.name !== '@colyseus/tools' &&
+      item.pm2_env.status !== 'stopped'
+    );
+  });
+
+  const aggregate = { ccu: 0, roomcount: 0, };
+  const apps = {};
+
+  await Promise.all(list.map(async (item) => {
+    const env = item.pm2_env;
+    const app_id = env.pm_id;
+    const uptime = new Date(env.pm_uptime); // uptime in milliseconds
+    const axm_monitor = env.axm_monitor;
+    const restart_time = env.restart_time;
+    const status = env.status; // 'online', 'stopped', 'stopping', 'waiting restart', 'launching', 'errored'
+    const node_version = env.node_version;
+
+    const monit = {
+      cpu: item.monit.cpu, // 0 ~ 100 (%)
+      memory: item.monit.memory / 1024 / 1024, // in MB
+    };
+
+    const version = {};
+    if (env.versioning) {
+      version.revision = env.versioning.revision;
+      version.comment = env.versioning.comment;
+      version.branch = env.versioning.branch;
+      version.update_time = env.versioning.update_time;
+    }
+
+    aggregate.ccu += axm_monitor.ccu?.value ?? 0;
+    aggregate.roomcount += axm_monitor.roomcount?.value ?? 0;
+
+    const custom_monitor = {};
+    for (const originalKey in axm_monitor) {
+      const key = (originalKey.indexOf(" ") !== -1)
+        ? axm_monitor[originalKey].type
+        : originalKey;
+      custom_monitor[key] = Number(axm_monitor[originalKey].value);
+    }
+
+    // check if process .sock file is active
+    const socket_is_active = await checkSocketIsActive(`/run/colyseus/${(2567 + env.NODE_APP_INSTANCE)}.sock`);
+
+    apps[app_id] = {
+      pid: item.pid,
+      uptime,
+      app_id,
+      status,
+      restart_time,
+      ...monit,
+      ...custom_monitor,
+      version,
+      node_version,
+      socket_is_active,
+    };
+  }));
+
+  const fetchipv4 = await fetch("http://169.254.169.254/v1.json");
+  const ip = (await fetchipv4.json()).interfaces[0].ipv4.address;
+
+  const body = {
+    version: 1,
+    ip,
+    time: new Date(),
+    aggregate,
+    apps,
+  };
+
+  console.log(body);
+
+  try {
+    const response = await fetchRetry(COLYSEUS_CLOUD_URL, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${process.env.COLYSEUS_SECRET}`,
+        "ngrok-skip-browser-warning": "yes",
+      },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(FETCH_TIMEOUT)
+    });
+
+    if (response.status !== 200) {
+      throw new Error(`Failed to send stats to Colyseus Cloud. Status: ${response.status}`);
+    }
+
+    console.log("OK");
+
+    // Only retry failed attempts if the current attempt was successful
+    await retryFailedAttempts();
+
+  } catch (e) {
+    console.error("Failed to send stats to Colyseus Cloud. ");
+
+    // cache failed attempts
+    fs.appendFileSync(FAILED_ATTEMPS_FILE, JSON.stringify(body) + "\n");
+
+  } finally {
+    process.exit();
+  }
+});
+
+function checkSocketIsActive(sockFilePath) {
+  return new Promise((resolve, _) => {
+    const client = net.createConnection({ path: sockFilePath, timeout: 5000 })
+      .on('connect', () => {
+        client.end(); // close the connection
+        resolve(true);
+      })
+      .on('error', () => resolve(false))
+      .on('timeout', () => resolve(false));
+  });
+}

package/system-boot.js

@@ -2,56 +2,110 @@
 
 const os = require('os');
 const fs = require('fs');
-const { exec } = require('child_process');
+const { execSync } = require('child_process');
 
 const NGINX_LIMITS_CONFIG_FILE = '/etc/nginx/colyseus_limits.conf';
-const LIMITS_CONF_FILE = "/etc/security/limits.conf";
+const LIMITS_CONF_FILE = '/etc/security/limits.d/colyseus.conf';
+const SYSCTL_FILE = '/etc/sysctl.d/local.conf'
 
-// update file descriptor limits systemwide + nginx worker connections
+const MAX_CCU_PER_CPU = 8000;
 
-function bailOnErr(err) {
-  if (err) {
-    console.error(err);
+/**
+ * System-wide limits configuration for high-CCU environments
+ * @param {Object} options
+ * @param {number} options.maxCCUPerCPU - Maximum concurrent users per CPU core
+ * @param {number} options.connectionsMultiplier - Multiplier for worker connections (default: 3)
+ * @param {number} options.fileDescriptorMultiplier - Multiplier for max file descriptors (default: 4)
+ */
+function configureSystemLimits(options = {}) {
+  const {
+    connectionsMultiplier = 3,
+    fileDescriptorMultiplier = 4
+  } = options;
 
-    // exit with error!
-    process.exit(1);
-  }
-}
+  const numCPU = os.cpus().length;
+  const maxCCU = numCPU * MAX_CCU_PER_CPU;
 
-function updateNOFileConfig(cb) {
-  // const numCPU = os.cpus().length;
-  const totalmemMB = os.totalmem() / 1024 / 1024;
-  const estimatedCCUPerGB = 5000;
+  // Calculate limits
+  const workerConnections = maxCCU * connectionsMultiplier;
+  const maxFileDescriptors = maxCCU * fileDescriptorMultiplier;
 
-  const maxCCU = Math.floor((totalmemMB / 1024) * estimatedCCUPerGB);
-  const workerConnections = maxCCU * 2; // 2x because of proxy_pass
-  const nginxMaxNOFileLimit = workerConnections * 2; // 2x
-  const systemMaxNOFileLimit = workerConnections * 3; // 3x for other system operations
+  // Nginx-specific calculations
+  const workerRlimitNofile = (workerConnections / numCPU) * 2;
 
-  // immediatelly apply new nofile limit
-  // (apparently this has no effect)
-  exec(`ulimit -n ${systemMaxNOFileLimit}`, bailOnErr);
+  // Validation checks
+  if (workerConnections > 65535) {
+    console.warn(`Warning: worker_connections (${workerConnections}) exceeds typical max_socket_backlog (65535)`);
+  }
 
-  // update "/etc/security/limits.conf" file.
-  fs.writeFileSync(LIMITS_CONF_FILE, `
-* - nofile ${systemMaxNOFileLimit}
-`, bailOnErr);
+  if (maxFileDescriptors > 1048576) {
+    console.warn(`Warning: Very high file descriptor limit (${maxFileDescriptors}). Verify system capabilities.`);
+  }
 
-  if (fs.existsSync(NGINX_LIMITS_CONFIG_FILE)) {
-    fs.writeFileSync(NGINX_LIMITS_CONFIG_FILE, `
-worker_rlimit_nofile ${nginxMaxNOFileLimit};
+  if (process.argv.includes('--dry-run')) {
+    console.log({
+      maxCCU,
+      workerConnections,
+      maxFileDescriptors,
+      workerRlimitNofile
+    })
+    process.exit();
+  }
 
+  // Configuration updates
+  try {
+    // Update Nginx limits
+    if (fs.existsSync(NGINX_LIMITS_CONFIG_FILE)) {
+      fs.writeFileSync(NGINX_LIMITS_CONFIG_FILE, `
+worker_rlimit_nofile ${workerRlimitNofile};
 events {
+    use epoll;
     worker_connections ${workerConnections};
-    # multi_accept on;
+    multi_accept on;
 }
-`, cb);
-    console.log("new nofile limit:", { workerConnections, systemMaxNOFileLimit, nginxMaxNOFileLimit });
+`);
+    }
+
+    // Update system-wide limits
+    fs.writeFileSync(LIMITS_CONF_FILE, `
+# System-wide file descriptor limits
+* - nofile ${maxFileDescriptors}
+nginx soft nofile ${maxFileDescriptors}
+nginx hard nofile ${maxFileDescriptors}
+`);
 
-  } else {
-    console.warn(NGINX_LIMITS_CONFIG_FILE, "not found.");
+    // Update sysctl with doubled file-max for safety margin
+    fs.writeFileSync(SYSCTL_FILE, `
+# System-wide file descriptor limit
+fs.file-max = ${maxFileDescriptors * 2}
+
+# TCP buffer optimization
+net.core.rmem_max = 16777216
+net.core.wmem_max = 16777216
+net.ipv4.tcp_rmem = 4096 87380 16777216
+net.ipv4.tcp_wmem = 4096 87380 16777216
+
+# Connection handling optimization
+net.core.netdev_max_backlog = 50000
+net.core.somaxconn = 65535
+net.ipv4.tcp_tw_reuse = 1
+net.ipv4.ip_local_port_range = 1024 65535
+
+# TCP timeout optimization
+net.ipv4.tcp_fin_timeout = 30
+net.ipv4.tcp_keepalive_time = 30
+net.ipv4.tcp_keepalive_intvl = 10
+net.ipv4.tcp_keepalive_probes = 3
+`);
+
+    // Apply sysctl changes
+    execSync("sysctl -p", { stdio: 'inherit' });
+    console.log(`System limits configured successfully for ${maxCCU} CCU (${MAX_CCU_PER_CPU}/CPU)`);
+
+  } catch (error) {
+    console.error('Failed to update system limits:', error);
+    process.exit(1);
   }
 }
 
-
-updateNOFileConfig(bailOnErr);
+configureSystemLimits();