@colyseus/tools 0.15.44 → 0.15.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +13 -4
- package/pm2/post-deploy-agent.js +312 -0
- package/pm2/shared.js +73 -0
- package/post-deploy.js +53 -90
- package/report-stats.js +14 -0
- package/system-boot.js +89 -35
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@colyseus/tools",
|
|
3
|
-
"version": "0.15.
|
|
3
|
+
"version": "0.15.46",
|
|
4
4
|
"description": "Colyseus Tools for Production",
|
|
5
5
|
"input": "./src/index.ts",
|
|
6
6
|
"main": "./build/index.js",
|
|
@@ -26,17 +26,18 @@
|
|
|
26
26
|
"url": "https://github.com/colyseus/colyseus/issues"
|
|
27
27
|
},
|
|
28
28
|
"files": [
|
|
29
|
+
"pm2",
|
|
29
30
|
"html",
|
|
30
31
|
"build",
|
|
31
32
|
"LICENSE",
|
|
32
33
|
"README.md"
|
|
33
34
|
],
|
|
34
|
-
"homepage": "https
|
|
35
|
+
"homepage": "https://colyseus.io",
|
|
35
36
|
"devDependencies": {
|
|
36
37
|
"@types/cors": "^2.8.10",
|
|
37
38
|
"@types/dotenv": "^8.2.0",
|
|
38
39
|
"uwebsockets-express": "^1.1.10",
|
|
39
|
-
"@colyseus/uwebsockets-transport": "^0.15.
|
|
40
|
+
"@colyseus/uwebsockets-transport": "^0.15.12"
|
|
40
41
|
},
|
|
41
42
|
"dependencies": {
|
|
42
43
|
"@pm2/io": "^6.0.1",
|
|
@@ -44,12 +45,20 @@
|
|
|
44
45
|
"cors": "^2.8.5",
|
|
45
46
|
"dotenv": "^8.2.0",
|
|
46
47
|
"express": "^4.16.2",
|
|
47
|
-
"@colyseus/core": "^0.15.
|
|
48
|
+
"@colyseus/core": "^0.15.54",
|
|
48
49
|
"@colyseus/ws-transport": "^0.15.2"
|
|
49
50
|
},
|
|
50
51
|
"publishConfig": {
|
|
51
52
|
"access": "public"
|
|
52
53
|
},
|
|
54
|
+
"apps": [
|
|
55
|
+
{
|
|
56
|
+
"merge_logs": true,
|
|
57
|
+
"max_memory_restart": "256M",
|
|
58
|
+
"script": "./pm2/post-deploy-agent.js",
|
|
59
|
+
"instance_var": "INSTANCE_ID"
|
|
60
|
+
}
|
|
61
|
+
],
|
|
53
62
|
"scripts": {
|
|
54
63
|
"start": "tsx example/app.ts"
|
|
55
64
|
}
|
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PM2 Agent for no downtime deployments on Colyseus Cloud.
|
|
3
|
+
*
|
|
4
|
+
* How it works:
|
|
5
|
+
* - New process(es) are spawned (MAX_ACTIVE_PROCESSES/2)
|
|
6
|
+
* - NGINX configuration is updated so new traffic only goes through the new process
|
|
7
|
+
* - Old processes are asynchronously and gracefully stopped.
|
|
8
|
+
* - The rest of the processes are spawned/reactivated.
|
|
9
|
+
*/
|
|
10
|
+
const pm2 = require('pm2');
|
|
11
|
+
const fs = require('fs');
|
|
12
|
+
const cst = require('pm2/constants');
|
|
13
|
+
const io = require('@pm2/io');
|
|
14
|
+
const path = require('path');
|
|
15
|
+
const shared = require('./shared');
|
|
16
|
+
|
|
17
|
+
const opts = { env: process.env.NODE_ENV || "production", };
|
|
18
|
+
let config = undefined;
|
|
19
|
+
|
|
20
|
+
io.initModule({
|
|
21
|
+
pid: path.resolve('/var/run/colyseus-agent.pid'),
|
|
22
|
+
widget: {
|
|
23
|
+
type: 'generic',
|
|
24
|
+
logo: 'https://colyseus.io/images/logos/logo-dark-color.png',
|
|
25
|
+
theme : ['#9F1414', '#591313', 'white', 'white'],
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
pm2.connect(function(err) {
|
|
30
|
+
if (err) {
|
|
31
|
+
console.error(err.stack || err);
|
|
32
|
+
process.exit();
|
|
33
|
+
}
|
|
34
|
+
console.log('PM2 post-deploy agent is up and running...');
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Remote actions
|
|
38
|
+
*/
|
|
39
|
+
io.action('post-deploy', async function (arg0, reply) {
|
|
40
|
+
const [cwd, ecosystemFilePath] = arg0.split(':');
|
|
41
|
+
console.log("Received 'post-deploy' action!", { cwd, config: ecosystemFilePath });
|
|
42
|
+
|
|
43
|
+
let replied = false;
|
|
44
|
+
|
|
45
|
+
//
|
|
46
|
+
// Override 'reply' to decrement amount of concurrent deployments
|
|
47
|
+
//
|
|
48
|
+
const onReply = function() {
|
|
49
|
+
if (replied) { return; }
|
|
50
|
+
replied = true;
|
|
51
|
+
reply.apply(null, arguments);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
try {
|
|
55
|
+
config = await shared.getAppConfig(ecosystemFilePath);
|
|
56
|
+
opts.cwd = cwd;
|
|
57
|
+
postDeploy(cwd, onReply);
|
|
58
|
+
|
|
59
|
+
} catch (err) {
|
|
60
|
+
onReply({ success: false, message: err?.message });
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
const restartingAppIds = new Set();
|
|
66
|
+
|
|
67
|
+
function postDeploy(cwd, reply) {
|
|
68
|
+
shared.listApps(function(err, apps) {
|
|
69
|
+
if (err) {
|
|
70
|
+
console.error(err);
|
|
71
|
+
return reply({ success: false, message: err?.message });
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// first deploy, start all processes
|
|
75
|
+
if (apps.length === 0) {
|
|
76
|
+
return pm2.start(config, {...opts}, (err, result) => {
|
|
77
|
+
reply({ success: !err, message: err?.message });
|
|
78
|
+
updateAndSave(err, result);
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
console.log("apps[0].pm2_env.pm_cwd =>", apps[0].pm2_env.pm_cwd);
|
|
83
|
+
console.log("cwd =>", cwd);
|
|
84
|
+
|
|
85
|
+
//
|
|
86
|
+
// detect if cwd has changed, and restart PM2 if it has
|
|
87
|
+
//
|
|
88
|
+
if (apps[0].pm2_env.pm_cwd !== cwd) {
|
|
89
|
+
console.log("cwd has changed. restarting PM2...");
|
|
90
|
+
|
|
91
|
+
//
|
|
92
|
+
// remove all and start again with new cwd
|
|
93
|
+
//
|
|
94
|
+
return pm2.delete(shared.NAMESPACE, function(err) {
|
|
95
|
+
// start again
|
|
96
|
+
// (TODO: make sure CWD is actually changed after this...)
|
|
97
|
+
pm2.start(config, { ...opts }, (err, result) => {
|
|
98
|
+
reply({ success: !err, message: err?.message });
|
|
99
|
+
updateAndSave(err, result);
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Graceful restart logic:
|
|
106
|
+
* List of PM2 app envs to stop or restart
|
|
107
|
+
*/
|
|
108
|
+
const appsToStop = [];
|
|
109
|
+
const appsStopped = [];
|
|
110
|
+
let numAppsStopping = 0;
|
|
111
|
+
let numTotalApps = undefined;
|
|
112
|
+
|
|
113
|
+
apps.forEach((app) => {
|
|
114
|
+
const env = app.pm2_env;
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Asynchronously teardown/stop processes with active connections
|
|
118
|
+
*/
|
|
119
|
+
if (env.status === cst.STOPPED_STATUS) {
|
|
120
|
+
appsStopped.push(env);
|
|
121
|
+
|
|
122
|
+
} else if (env.status !== cst.STOPPING_STATUS) {
|
|
123
|
+
appsToStop.push(env);
|
|
124
|
+
|
|
125
|
+
} else if (!restartingAppIds.has(env.pm_id)) {
|
|
126
|
+
numAppsStopping++;
|
|
127
|
+
}
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* - Start new process
|
|
132
|
+
* - Update NGINX config to expose only the new process
|
|
133
|
+
* - Stop old processes
|
|
134
|
+
* - Spawn/reactivate the rest of the processes (shared.MAX_ACTIVE_PROCESSES)
|
|
135
|
+
*/
|
|
136
|
+
const onFirstAppsStart = (initialApps, err, result) => {
|
|
137
|
+
/**
|
|
138
|
+
* release post-deploy action while proceeding with graceful restart of other processes
|
|
139
|
+
*/
|
|
140
|
+
reply({ success: !err, message: err?.message });
|
|
141
|
+
|
|
142
|
+
if (err) { return console.error(err); }
|
|
143
|
+
|
|
144
|
+
let numActiveApps = initialApps.length + restartingAppIds.size;
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* - Write NGINX config to expose only the new active process
|
|
148
|
+
* - The old ones processes will go down asynchronously (or will be restarted)
|
|
149
|
+
*/
|
|
150
|
+
writeNginxConfig(initialApps);
|
|
151
|
+
|
|
152
|
+
//
|
|
153
|
+
// Asynchronously stop/restart apps with active connections
|
|
154
|
+
// (They make take from minutes up to hours to stop)
|
|
155
|
+
//
|
|
156
|
+
appsToStop.forEach((app_env) => {
|
|
157
|
+
if (numActiveApps < shared.MAX_ACTIVE_PROCESSES) {
|
|
158
|
+
numActiveApps++;
|
|
159
|
+
|
|
160
|
+
restartingAppIds.add(app_env.pm_id);
|
|
161
|
+
pm2.restart(app_env.pm_id, (err, _) => {
|
|
162
|
+
restartingAppIds.delete(app_env.pm_id);
|
|
163
|
+
if (err) { return logIfError(err); }
|
|
164
|
+
|
|
165
|
+
// reset counter stats (restart_time=0)
|
|
166
|
+
pm2.reset(app_env.pm_id, logIfError);
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
} else {
|
|
170
|
+
pm2.stop(app_env.pm_id, logIfError);
|
|
171
|
+
}
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
if (numActiveApps < shared.MAX_ACTIVE_PROCESSES) {
|
|
175
|
+
const missingOnlineApps = shared.MAX_ACTIVE_PROCESSES - numActiveApps;
|
|
176
|
+
|
|
177
|
+
// console.log("Active apps is lower than MAX_ACTIVE_PROCESSES, will SCALE again =>", {
|
|
178
|
+
// missingOnlineApps,
|
|
179
|
+
// numActiveApps,
|
|
180
|
+
// newNumTotalApps: numTotalApps + missingOnlineApps
|
|
181
|
+
// });
|
|
182
|
+
|
|
183
|
+
pm2.scale(apps[0].name, numTotalApps + missingOnlineApps, updateAndSaveIfAllRunning);
|
|
184
|
+
}
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
const numHalfMaxActiveProcesses = Math.ceil(shared.MAX_ACTIVE_PROCESSES / 2);
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Re-use previously stopped apps if available
|
|
191
|
+
*/
|
|
192
|
+
if (appsStopped.length >= numHalfMaxActiveProcesses) {
|
|
193
|
+
const initialApps = appsStopped.splice(0, numHalfMaxActiveProcesses);
|
|
194
|
+
|
|
195
|
+
let numSucceeded = 0;
|
|
196
|
+
initialApps.forEach((app_env) => {
|
|
197
|
+
// console.log("pm2.restart => ", app_env.pm_id);
|
|
198
|
+
|
|
199
|
+
restartingAppIds.add(app_env.pm_id);
|
|
200
|
+
pm2.restart(app_env.pm_id, (err) => {
|
|
201
|
+
restartingAppIds.delete(app_env.pm_id);
|
|
202
|
+
if (err) { return replyIfError(err, reply); }
|
|
203
|
+
|
|
204
|
+
// reset counter stats (restart_time=0)
|
|
205
|
+
pm2.reset(app_env.pm_id, logIfError);
|
|
206
|
+
|
|
207
|
+
// TODO: set timeout here to exit if some processes are not restarting
|
|
208
|
+
|
|
209
|
+
numSucceeded++;
|
|
210
|
+
if (numSucceeded === initialApps.length) {
|
|
211
|
+
onFirstAppsStart(initialApps);
|
|
212
|
+
}
|
|
213
|
+
});
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
} else {
|
|
217
|
+
/**
|
|
218
|
+
* Increment to +(MAX/2) processes
|
|
219
|
+
*/
|
|
220
|
+
let LAST_NODE_APP_INSTANCE = apps[apps.length - 1].pm2_env.NODE_APP_INSTANCE;
|
|
221
|
+
const initialApps = Array.from({ length: numHalfMaxActiveProcesses }).map((_, i) => {
|
|
222
|
+
const new_app_env = Object.assign({}, apps[0].pm2_env);
|
|
223
|
+
new_app_env.NODE_APP_INSTANCE = ++LAST_NODE_APP_INSTANCE;
|
|
224
|
+
return new_app_env;
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
numTotalApps = apps.length + numHalfMaxActiveProcesses;
|
|
228
|
+
|
|
229
|
+
// Ensure to scale to a number of processes where `numHalfMaxActiveProcesses` can start immediately.
|
|
230
|
+
pm2.scale(apps[0].name, numTotalApps, onFirstAppsStart.bind(undefined, initialApps));
|
|
231
|
+
}
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
function updateAndSave() {
|
|
236
|
+
// console.log("updateAndExit");
|
|
237
|
+
updateAndReloadNginx(() => complete());
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function updateAndSaveIfAllRunning(err) {
|
|
241
|
+
if (err) { return console.error(err); }
|
|
242
|
+
|
|
243
|
+
updateAndReloadNginx((app_envs) => {
|
|
244
|
+
// console.log("updateAndExitIfAllRunning, app_ids (", app_envs.map(app_env => app_env.NODE_APP_INSTANCE) ,") => ", app_envs.length, "/", shared.MAX_ACTIVE_PROCESSES);
|
|
245
|
+
|
|
246
|
+
//
|
|
247
|
+
// TODO: add timeout to exit here, in case some processes are not starting
|
|
248
|
+
//
|
|
249
|
+
if (app_envs.length === shared.MAX_ACTIVE_PROCESSES) {
|
|
250
|
+
complete();
|
|
251
|
+
}
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
function updateAndReloadNginx(cb) {
|
|
256
|
+
//
|
|
257
|
+
// If you are self-hosting and reading this file, consider using the
|
|
258
|
+
// following in your self-hosted environment:
|
|
259
|
+
//
|
|
260
|
+
// #!/bin/bash
|
|
261
|
+
// # Requires fswatch (`apt install fswatch`)
|
|
262
|
+
// # Reload NGINX when colyseus_servers.conf changes
|
|
263
|
+
// fswatch /etc/nginx/colyseus_servers.conf -m poll_monitor --event=Updated | while read event
|
|
264
|
+
// do
|
|
265
|
+
// service nginx reload
|
|
266
|
+
// done
|
|
267
|
+
|
|
268
|
+
shared.listApps(function(err, apps) {
|
|
269
|
+
if (apps.length === 0) { err = "no apps running."; }
|
|
270
|
+
if (err) { return console.error(err); }
|
|
271
|
+
|
|
272
|
+
const app_envs = apps
|
|
273
|
+
.filter(app => app.pm2_env.status !== cst.STOPPING_STATUS && app.pm2_env.status !== cst.STOPPED_STATUS)
|
|
274
|
+
.map((app) => app.pm2_env);
|
|
275
|
+
|
|
276
|
+
writeNginxConfig(app_envs);
|
|
277
|
+
|
|
278
|
+
cb?.(app_envs);
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
function writeNginxConfig(app_envs) {
|
|
283
|
+
// console.log("writeNginxConfig: ", app_envs.map(app_env => app_env.NODE_APP_INSTANCE));
|
|
284
|
+
|
|
285
|
+
const port = 2567;
|
|
286
|
+
const addresses = [];
|
|
287
|
+
|
|
288
|
+
app_envs.forEach(function(app_env) {
|
|
289
|
+
addresses.push(`unix:${shared.PROCESS_UNIX_SOCK_PATH}${port + app_env.NODE_APP_INSTANCE}.sock`);
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
// write NGINX config
|
|
293
|
+
fs.writeFileSync(shared.NGINX_SERVERS_CONFIG_FILE, addresses.map(address => `server ${address};`).join("\n"), logIfError);
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function complete() {
|
|
297
|
+
// "pm2 save"
|
|
298
|
+
pm2.dump(logIfError);
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
function logIfError (err) {
|
|
302
|
+
if (err) {
|
|
303
|
+
console.error(err);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function replyIfError(err, reply) {
|
|
308
|
+
if (err) {
|
|
309
|
+
console.error(err);
|
|
310
|
+
reply({ success: false, message: err?.message });
|
|
311
|
+
}
|
|
312
|
+
}
|
package/pm2/shared.js
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
const pm2 = require('pm2');
|
|
2
|
+
const os = require('os');
|
|
3
|
+
|
|
4
|
+
const NAMESPACE = 'cloud';
|
|
5
|
+
const MAX_ACTIVE_PROCESSES = os.cpus().length;
|
|
6
|
+
|
|
7
|
+
function listApps(callback) {
|
|
8
|
+
pm2.list((err, apps) => {
|
|
9
|
+
if (err) { return callback(err);; }
|
|
10
|
+
|
|
11
|
+
// Filter out @colyseus/tools module (PM2 post-deploy agent)
|
|
12
|
+
apps = apps.filter(app => app.name !== '@colyseus/tools');
|
|
13
|
+
|
|
14
|
+
callback(err, apps);
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
async function getAppConfig(ecosystemFilePath) {
|
|
19
|
+
const module = await import(ecosystemFilePath);
|
|
20
|
+
const config = module.default;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Tune PM2 app config
|
|
24
|
+
*/
|
|
25
|
+
if (config.apps && config.apps.length >= 0) {
|
|
26
|
+
const app = config.apps[0];
|
|
27
|
+
|
|
28
|
+
// app.name = "colyseus-app";
|
|
29
|
+
app.namespace = NAMESPACE;
|
|
30
|
+
app.exec_mode = "fork";
|
|
31
|
+
|
|
32
|
+
app.instances = MAX_ACTIVE_PROCESSES;
|
|
33
|
+
|
|
34
|
+
app.time = true;
|
|
35
|
+
app.wait_ready = true;
|
|
36
|
+
app.watch = false;
|
|
37
|
+
|
|
38
|
+
// default: merge logs into a single file
|
|
39
|
+
if (app.merge_logs === undefined) {
|
|
40
|
+
app.merge_logs = true;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// default: wait for 30 minutes before forcibly killing
|
|
44
|
+
// (prevent forcibly killing while rooms are still active)
|
|
45
|
+
if (!app.kill_timeout) {
|
|
46
|
+
app.kill_timeout = 30 * 60 * 1000;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// default: retry kill after 1 second
|
|
50
|
+
if (!app.kill_retry_time) {
|
|
51
|
+
app.kill_retry_time = 5000;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return config;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
module.exports = {
|
|
59
|
+
/**
|
|
60
|
+
* Constants
|
|
61
|
+
*/
|
|
62
|
+
NGINX_SERVERS_CONFIG_FILE: '/etc/nginx/colyseus_servers.conf',
|
|
63
|
+
PROCESS_UNIX_SOCK_PATH: '/run/colyseus/',
|
|
64
|
+
|
|
65
|
+
MAX_ACTIVE_PROCESSES,
|
|
66
|
+
NAMESPACE,
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Shared methods
|
|
70
|
+
*/
|
|
71
|
+
listApps,
|
|
72
|
+
getAppConfig,
|
|
73
|
+
}
|
package/post-deploy.js
CHANGED
|
@@ -1,67 +1,81 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
const pm2 = require('pm2');
|
|
3
|
-
const os = require('os');
|
|
4
3
|
const fs = require('fs');
|
|
5
4
|
const path = require('path');
|
|
5
|
+
const shared = require('./pm2/shared');
|
|
6
6
|
|
|
7
|
-
const opts = { env: process.env.NODE_ENV || "production" };
|
|
8
|
-
const maxCPU = os.cpus().length;
|
|
7
|
+
const opts = { env: process.env.NODE_ENV || "production", };
|
|
9
8
|
|
|
10
|
-
|
|
11
|
-
// if (process.env.npm_config_local_prefix) {
|
|
12
|
-
// process.chdir(process.env.npm_config_local_prefix);
|
|
13
|
-
// pm2.cwd = process.env.npm_config_local_prefix;
|
|
14
|
-
// }
|
|
15
|
-
|
|
16
|
-
const CONFIG_FILE = [
|
|
9
|
+
const CONFIG_FILE = pm2.cwd + "/" + [
|
|
17
10
|
'ecosystem.config.cjs',
|
|
18
11
|
'ecosystem.config.js',
|
|
19
12
|
'pm2.config.cjs',
|
|
20
13
|
'pm2.config.js',
|
|
21
14
|
].find((filename) => fs.existsSync(path.resolve(pm2.cwd, filename)));
|
|
22
15
|
|
|
16
|
+
let config = undefined;
|
|
17
|
+
|
|
23
18
|
if (!CONFIG_FILE) {
|
|
24
19
|
throw new Error('missing ecosystem config file. make sure to provide one with a valid "script" entrypoint file path.');
|
|
25
20
|
}
|
|
26
21
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
//
|
|
37
|
-
pm2.start(CONFIG_FILE, { ...opts }, () => onAppRunning());
|
|
22
|
+
/**
|
|
23
|
+
* Try to handle post-deploy via PM2 module first (pm2 install @colyseus/tools)
|
|
24
|
+
* If not available, fallback to legacy post-deploy script.
|
|
25
|
+
*/
|
|
26
|
+
pm2.trigger('@colyseus/tools', 'post-deploy', `${pm2.cwd}:${CONFIG_FILE}`, async function (err, result) {
|
|
27
|
+
if (err) {
|
|
28
|
+
console.log("Proceeding with legacy post-deploy script...");
|
|
29
|
+
config = await shared.getAppConfig(CONFIG_FILE);
|
|
30
|
+
postDeploy();
|
|
38
31
|
|
|
39
32
|
} else {
|
|
33
|
+
if (result[0].data?.return?.success === false) {
|
|
34
|
+
console.error(result[0].data?.return?.message);
|
|
35
|
+
process.exit(1);
|
|
36
|
+
} else {
|
|
37
|
+
console.log("Post-deploy success.");
|
|
38
|
+
process.exit();
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
});
|
|
40
42
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
async function postDeploy() {
|
|
44
|
+
shared.listApps(function (err, apps) {
|
|
45
|
+
bailOnErr(err);
|
|
46
|
+
|
|
47
|
+
if (apps.length === 0) {
|
|
45
48
|
//
|
|
46
|
-
//
|
|
49
|
+
// first deploy
|
|
47
50
|
//
|
|
48
|
-
|
|
51
|
+
pm2.start(config, { ...opts }, () => onAppRunning());
|
|
49
52
|
|
|
50
53
|
} else {
|
|
54
|
+
|
|
51
55
|
//
|
|
52
|
-
//
|
|
56
|
+
// detect if cwd has changed, and restart PM2 if it has
|
|
53
57
|
//
|
|
54
|
-
|
|
58
|
+
if (apps[0].pm2_env.pm_cwd !== pm2.cwd) {
|
|
59
|
+
//
|
|
60
|
+
// remove all and start again with new cwd
|
|
61
|
+
//
|
|
62
|
+
restartAll();
|
|
63
|
+
|
|
64
|
+
} else {
|
|
65
|
+
//
|
|
66
|
+
// reload existing apps
|
|
67
|
+
//
|
|
68
|
+
reloadAll();
|
|
69
|
+
}
|
|
55
70
|
}
|
|
56
|
-
}
|
|
57
|
-
}
|
|
71
|
+
});
|
|
72
|
+
}
|
|
58
73
|
|
|
59
74
|
function onAppRunning(reloadedAppIds) {
|
|
60
75
|
// reset reloaded app stats
|
|
61
76
|
if (reloadedAppIds) {
|
|
62
77
|
resetAppStats(reloadedAppIds);
|
|
63
78
|
}
|
|
64
|
-
updateColyseusBootService();
|
|
65
79
|
updateAndReloadNginx();
|
|
66
80
|
}
|
|
67
81
|
|
|
@@ -69,13 +83,13 @@ function restartAll () {
|
|
|
69
83
|
pm2.delete('all', function (err) {
|
|
70
84
|
// kill & start again
|
|
71
85
|
pm2.kill(function () {
|
|
72
|
-
pm2.start(
|
|
86
|
+
pm2.start(config, { ...opts }, () => onAppRunning());
|
|
73
87
|
});
|
|
74
88
|
});
|
|
75
89
|
}
|
|
76
90
|
|
|
77
91
|
function reloadAll(retry = 0) {
|
|
78
|
-
pm2.reload(
|
|
92
|
+
pm2.reload(config, { ...opts }, function (err, apps) {
|
|
79
93
|
if (err) {
|
|
80
94
|
//
|
|
81
95
|
// Retry in case of "Reload in progress" error.
|
|
@@ -95,8 +109,8 @@ function reloadAll(retry = 0) {
|
|
|
95
109
|
const reloadedAppIds = apps.map(app => app.pm_id);
|
|
96
110
|
|
|
97
111
|
// scale app to use all CPUs available
|
|
98
|
-
if (apps.length !==
|
|
99
|
-
pm2.scale(name,
|
|
112
|
+
if (apps.length !== shared.MAX_ACTIVE_PROCESSES) {
|
|
113
|
+
pm2.scale(name, shared.MAX_ACTIVE_PROCESSES, () => onAppRunning(reloadedAppIds));
|
|
100
114
|
|
|
101
115
|
} else {
|
|
102
116
|
onAppRunning(reloadedAppIds);
|
|
@@ -116,30 +130,6 @@ function resetAppStats (reloadedAppIds) {
|
|
|
116
130
|
});
|
|
117
131
|
};
|
|
118
132
|
|
|
119
|
-
function updateColyseusBootService() {
|
|
120
|
-
//
|
|
121
|
-
// Update colyseus-boot.service to use the correct paths for the application
|
|
122
|
-
//
|
|
123
|
-
const COLYSEUS_CLOUD_BOOT_SERVICE = '/etc/systemd/system/colyseus-boot.service';
|
|
124
|
-
|
|
125
|
-
// ignore if no boot service found
|
|
126
|
-
if (!fs.existsSync(COLYSEUS_CLOUD_BOOT_SERVICE)) {
|
|
127
|
-
return;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
const workingDirectory = pm2.cwd;
|
|
131
|
-
const execStart = `${detectPackageManager()} colyseus-post-deploy`;
|
|
132
|
-
|
|
133
|
-
const contents = fs.readFileSync(COLYSEUS_CLOUD_BOOT_SERVICE, 'utf8');
|
|
134
|
-
try {
|
|
135
|
-
fs.writeFileSync(COLYSEUS_CLOUD_BOOT_SERVICE, contents
|
|
136
|
-
.replace(/WorkingDirectory=(.*)/, `WorkingDirectory=${workingDirectory}`)
|
|
137
|
-
.replace(/ExecStart=(.*)/, `ExecStart=${execStart}`));
|
|
138
|
-
} catch (e) {
|
|
139
|
-
// couldn't write to file
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
|
|
143
133
|
function updateAndReloadNginx() {
|
|
144
134
|
//
|
|
145
135
|
// If you are self-hosting and reading this file, consider using the
|
|
@@ -153,9 +143,7 @@ function updateAndReloadNginx() {
|
|
|
153
143
|
// service nginx reload
|
|
154
144
|
// done
|
|
155
145
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
pm2.list(function(err, apps) {
|
|
146
|
+
shared.listApps(function(err, apps) {
|
|
159
147
|
if (apps.length === 0) {
|
|
160
148
|
err = "no apps running.";
|
|
161
149
|
}
|
|
@@ -165,11 +153,11 @@ function updateAndReloadNginx() {
|
|
|
165
153
|
const addresses = [];
|
|
166
154
|
|
|
167
155
|
apps.forEach(function(app) {
|
|
168
|
-
addresses.push(`unix
|
|
156
|
+
addresses.push(`unix:${shared.PROCESS_UNIX_SOCK_PATH}${port + app.pm2_env.NODE_APP_INSTANCE}.sock`);
|
|
169
157
|
});
|
|
170
158
|
|
|
171
159
|
// write NGINX config
|
|
172
|
-
fs.writeFileSync(NGINX_SERVERS_CONFIG_FILE, addresses.map(address => `server ${address};`).join("\n"), bailOnErr);
|
|
160
|
+
fs.writeFileSync(shared.NGINX_SERVERS_CONFIG_FILE, addresses.map(address => `server ${address};`).join("\n"), bailOnErr);
|
|
173
161
|
|
|
174
162
|
// "pm2 save"
|
|
175
163
|
pm2.dump(function (err, ret) {
|
|
@@ -182,30 +170,6 @@ function updateAndReloadNginx() {
|
|
|
182
170
|
});
|
|
183
171
|
}
|
|
184
172
|
|
|
185
|
-
function detectPackageManager() {
|
|
186
|
-
const lockfiles = {
|
|
187
|
-
// npm
|
|
188
|
-
"npm exec": path.resolve(pm2.cwd, 'package-lock.json'),
|
|
189
|
-
|
|
190
|
-
// yarn
|
|
191
|
-
"yarn exec": path.resolve(pm2.cwd, 'yarn.lock'),
|
|
192
|
-
|
|
193
|
-
// pnpm
|
|
194
|
-
"pnpm exec": path.resolve(pm2.cwd, 'pnpm-lock.yaml'),
|
|
195
|
-
|
|
196
|
-
// bun
|
|
197
|
-
"bunx": path.resolve(pm2.cwd, 'bun.lockb'),
|
|
198
|
-
};
|
|
199
|
-
|
|
200
|
-
for (const [key, value] of Object.entries(lockfiles)) {
|
|
201
|
-
if (fs.existsSync(value)) {
|
|
202
|
-
return key;
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
return "npm";
|
|
207
|
-
}
|
|
208
|
-
|
|
209
173
|
function bailOnErr(err) {
|
|
210
174
|
if (err) {
|
|
211
175
|
console.error(err);
|
|
@@ -214,4 +178,3 @@ function bailOnErr(err) {
|
|
|
214
178
|
process.exit(1);
|
|
215
179
|
}
|
|
216
180
|
}
|
|
217
|
-
|
package/report-stats.js
CHANGED
|
@@ -53,6 +53,20 @@ async function fetchRetry(url, options, retries = 3) {
|
|
|
53
53
|
};
|
|
54
54
|
|
|
55
55
|
pm2.Client.executeRemote('getMonitorData', {}, async function(err, list) {
|
|
56
|
+
if (err) { return console.error(err); }
|
|
57
|
+
|
|
58
|
+
//
|
|
59
|
+
// Filter out:
|
|
60
|
+
// - @colyseus/tools module (PM2 agent)
|
|
61
|
+
// - "stopped" processes (gracefully shut down)
|
|
62
|
+
//
|
|
63
|
+
list = list.filter((item) => {
|
|
64
|
+
return (
|
|
65
|
+
item.name !== '@colyseus/tools' &&
|
|
66
|
+
item.pm2_env.status !== 'stopped'
|
|
67
|
+
);
|
|
68
|
+
});
|
|
69
|
+
|
|
56
70
|
const aggregate = { ccu: 0, roomcount: 0, };
|
|
57
71
|
const apps = {};
|
|
58
72
|
|
package/system-boot.js
CHANGED
|
@@ -2,56 +2,110 @@
|
|
|
2
2
|
|
|
3
3
|
const os = require('os');
|
|
4
4
|
const fs = require('fs');
|
|
5
|
-
const {
|
|
5
|
+
const { execSync } = require('child_process');
|
|
6
6
|
|
|
7
7
|
const NGINX_LIMITS_CONFIG_FILE = '/etc/nginx/colyseus_limits.conf';
|
|
8
|
-
const LIMITS_CONF_FILE =
|
|
8
|
+
const LIMITS_CONF_FILE = '/etc/security/limits.d/colyseus.conf';
|
|
9
|
+
const SYSCTL_FILE = '/etc/sysctl.d/local.conf'
|
|
9
10
|
|
|
10
|
-
|
|
11
|
+
const MAX_CCU_PER_CPU = 8000;
|
|
11
12
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
/**
|
|
14
|
+
* System-wide limits configuration for high-CCU environments
|
|
15
|
+
* @param {Object} options
|
|
16
|
+
* @param {number} options.maxCCUPerCPU - Maximum concurrent users per CPU core
|
|
17
|
+
* @param {number} options.connectionsMultiplier - Multiplier for worker connections (default: 3)
|
|
18
|
+
* @param {number} options.fileDescriptorMultiplier - Multiplier for max file descriptors (default: 4)
|
|
19
|
+
*/
|
|
20
|
+
function configureSystemLimits(options = {}) {
|
|
21
|
+
const {
|
|
22
|
+
connectionsMultiplier = 3,
|
|
23
|
+
fileDescriptorMultiplier = 4
|
|
24
|
+
} = options;
|
|
15
25
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
}
|
|
19
|
-
}
|
|
26
|
+
const numCPU = os.cpus().length;
|
|
27
|
+
const maxCCU = numCPU * MAX_CCU_PER_CPU;
|
|
20
28
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
const
|
|
24
|
-
const estimatedCCUPerGB = 5000;
|
|
29
|
+
// Calculate limits
|
|
30
|
+
const workerConnections = maxCCU * connectionsMultiplier;
|
|
31
|
+
const maxFileDescriptors = maxCCU * fileDescriptorMultiplier;
|
|
25
32
|
|
|
26
|
-
|
|
27
|
-
const
|
|
28
|
-
const nginxMaxNOFileLimit = workerConnections * 2; // 2x
|
|
29
|
-
const systemMaxNOFileLimit = workerConnections * 3; // 3x for other system operations
|
|
33
|
+
// Nginx-specific calculations
|
|
34
|
+
const workerRlimitNofile = (workerConnections / numCPU) * 2;
|
|
30
35
|
|
|
31
|
-
//
|
|
32
|
-
|
|
33
|
-
|
|
36
|
+
// Validation checks
|
|
37
|
+
if (workerConnections > 65535) {
|
|
38
|
+
console.warn(`Warning: worker_connections (${workerConnections}) exceeds typical max_socket_backlog (65535)`);
|
|
39
|
+
}
|
|
34
40
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
`, bailOnErr);
|
|
41
|
+
if (maxFileDescriptors > 1048576) {
|
|
42
|
+
console.warn(`Warning: Very high file descriptor limit (${maxFileDescriptors}). Verify system capabilities.`);
|
|
43
|
+
}
|
|
39
44
|
|
|
40
|
-
if (
|
|
41
|
-
|
|
42
|
-
|
|
45
|
+
if (process.argv.includes('--dry-run')) {
|
|
46
|
+
console.log({
|
|
47
|
+
maxCCU,
|
|
48
|
+
workerConnections,
|
|
49
|
+
maxFileDescriptors,
|
|
50
|
+
workerRlimitNofile
|
|
51
|
+
})
|
|
52
|
+
process.exit();
|
|
53
|
+
}
|
|
43
54
|
|
|
55
|
+
// Configuration updates
|
|
56
|
+
try {
|
|
57
|
+
// Update Nginx limits
|
|
58
|
+
if (fs.existsSync(NGINX_LIMITS_CONFIG_FILE)) {
|
|
59
|
+
fs.writeFileSync(NGINX_LIMITS_CONFIG_FILE, `
|
|
60
|
+
worker_rlimit_nofile ${workerRlimitNofile};
|
|
44
61
|
events {
|
|
62
|
+
use epoll;
|
|
45
63
|
worker_connections ${workerConnections};
|
|
46
|
-
|
|
64
|
+
multi_accept on;
|
|
47
65
|
}
|
|
48
|
-
|
|
49
|
-
|
|
66
|
+
`);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Update system-wide limits
|
|
70
|
+
fs.writeFileSync(LIMITS_CONF_FILE, `
|
|
71
|
+
# System-wide file descriptor limits
|
|
72
|
+
* - nofile ${maxFileDescriptors}
|
|
73
|
+
nginx soft nofile ${maxFileDescriptors}
|
|
74
|
+
nginx hard nofile ${maxFileDescriptors}
|
|
75
|
+
`);
|
|
50
76
|
|
|
51
|
-
|
|
52
|
-
|
|
77
|
+
// Update sysctl with doubled file-max for safety margin
|
|
78
|
+
fs.writeFileSync(SYSCTL_FILE, `
|
|
79
|
+
# System-wide file descriptor limit
|
|
80
|
+
fs.file-max = ${maxFileDescriptors * 2}
|
|
81
|
+
|
|
82
|
+
# TCP buffer optimization
|
|
83
|
+
net.core.rmem_max = 16777216
|
|
84
|
+
net.core.wmem_max = 16777216
|
|
85
|
+
net.ipv4.tcp_rmem = 4096 87380 16777216
|
|
86
|
+
net.ipv4.tcp_wmem = 4096 87380 16777216
|
|
87
|
+
|
|
88
|
+
# Connection handling optimization
|
|
89
|
+
net.core.netdev_max_backlog = 50000
|
|
90
|
+
net.core.somaxconn = 65535
|
|
91
|
+
net.ipv4.tcp_tw_reuse = 1
|
|
92
|
+
net.ipv4.ip_local_port_range = 1024 65535
|
|
93
|
+
|
|
94
|
+
# TCP timeout optimization
|
|
95
|
+
net.ipv4.tcp_fin_timeout = 30
|
|
96
|
+
net.ipv4.tcp_keepalive_time = 30
|
|
97
|
+
net.ipv4.tcp_keepalive_intvl = 10
|
|
98
|
+
net.ipv4.tcp_keepalive_probes = 3
|
|
99
|
+
`);
|
|
100
|
+
|
|
101
|
+
// Apply sysctl changes
|
|
102
|
+
execSync("sysctl -p", { stdio: 'inherit' });
|
|
103
|
+
console.log(`System limits configured successfully for ${maxCCU} CCU (${MAX_CCU_PER_CPU}/CPU)`);
|
|
104
|
+
|
|
105
|
+
} catch (error) {
|
|
106
|
+
console.error('Failed to update system limits:', error);
|
|
107
|
+
process.exit(1);
|
|
53
108
|
}
|
|
54
109
|
}
|
|
55
110
|
|
|
56
|
-
|
|
57
|
-
updateNOFileConfig(bailOnErr);
|
|
111
|
+
configureSystemLimits();
|