@cocreate/server 1.6.1 → 1.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +435 -55
- package/package.json +29 -10
- package/src/index.js +245 -393
- package/src/ipc.js +157 -0
- package/src/log.js +207 -0
- package/src/state.js +436 -0
- package/src/storage.js +104 -0
- package/src/getIp.js +0 -113
package/src/index.js
CHANGED
|
@@ -19,14 +19,13 @@
|
|
|
19
19
|
import { EventEmitter } from 'node:events';
|
|
20
20
|
import Http from 'node:http';
|
|
21
21
|
import Https from 'node:https';
|
|
22
|
+
import cluster from 'node:cluster';
|
|
23
|
+
import os from 'node:os';
|
|
24
|
+
|
|
22
25
|
import Acme from '@cocreate/acme';
|
|
23
26
|
import ServerMesh from '@cocreate/server-mesh';
|
|
24
27
|
import ServerAutoscaler from '@cocreate/server-autoscaler';
|
|
25
28
|
import ServerTelemetry from '@cocreate/server-telemetry';
|
|
26
|
-
|
|
27
|
-
import cluster from 'node:cluster';
|
|
28
|
-
import os from 'node:os';
|
|
29
|
-
import uid from '@cocreate/uuid';
|
|
30
29
|
import Config from '@cocreate/config';
|
|
31
30
|
import authenticate from '@cocreate/authenticate';
|
|
32
31
|
import authorize from '@cocreate/authorize';
|
|
@@ -37,172 +36,121 @@ import LazyLoader from '@cocreate/lazy-loader';
|
|
|
37
36
|
import FileServer from '@cocreate/file-server';
|
|
38
37
|
import ApiServer from '@cocreate/api';
|
|
39
38
|
|
|
40
|
-
import { ObjectId } from '@cocreate/utils';
|
|
39
|
+
import { ObjectId, getClientInfo } from '@cocreate/utils';
|
|
41
40
|
|
|
42
|
-
|
|
43
|
-
import
|
|
44
|
-
import
|
|
41
|
+
import getInfrastructure from './getInfrastructure.js';
|
|
42
|
+
import IPC from './ipc.js';
|
|
43
|
+
import Storage from './storage.js';
|
|
44
|
+
import State from './state.js';
|
|
45
|
+
import LogManager from './log.js';
|
|
45
46
|
|
|
46
|
-
const
|
|
47
|
-
const id = process.env.SERVER_ID || ObjectId().toString();
|
|
47
|
+
const isMaster = cluster.isMaster;
|
|
48
48
|
|
|
49
|
-
const sharedMap = new Map();
|
|
50
49
|
const pendingPromises = new Map();
|
|
51
50
|
const activeWorkers = new Map();
|
|
52
51
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
52
|
+
/**
|
|
53
|
+
* CoCreateServer orchestrates cluster process management, socket handling,
|
|
54
|
+
* API endpoint dispatching, SSL certificates, and mesh cluster synchronization.
|
|
55
|
+
*/
|
|
58
56
|
export class CoCreateServer extends EventEmitter {
|
|
59
57
|
constructor() {
|
|
60
58
|
super();
|
|
61
|
-
this.id =
|
|
62
|
-
|
|
59
|
+
this.id = process.env.SERVER_ID || null;
|
|
60
|
+
|
|
61
|
+
this.isMaster = isMaster;
|
|
63
62
|
|
|
64
|
-
if (
|
|
63
|
+
if (this.isMaster) {
|
|
65
64
|
this.workers = activeWorkers;
|
|
66
65
|
} else {
|
|
67
66
|
this.workerId = parseInt(process.env.WORKER_ID, 10) || 1;
|
|
68
67
|
}
|
|
69
68
|
|
|
70
69
|
this.totalWorkers = 0;
|
|
71
|
-
this.workerStorage = null;
|
|
72
70
|
this.isInitialized = false;
|
|
73
|
-
|
|
74
|
-
this.organization_id = null;
|
|
75
|
-
this.host = null;
|
|
76
|
-
this.ip = null;
|
|
77
|
-
this.infrastructure = null;
|
|
78
|
-
|
|
79
|
-
// Ingress state management flags
|
|
80
|
-
this.isCordon = false;
|
|
81
|
-
this.isShedding = false;
|
|
82
|
-
this.isDraining = false;
|
|
83
|
-
|
|
84
|
-
this.on('cordon', (payload) => {
|
|
85
|
-
const active = payload?.active !== false;
|
|
86
|
-
|
|
87
|
-
// Core Logic: Telemetry or callers cannot uncordon the node if shedding or draining is active
|
|
88
|
-
if (!active && (this.isShedding || this.isDraining)) {
|
|
89
|
-
console.warn(`[@cocreate/server] Blocked un-cordon request. Overriding active states: isShedding=${this.isShedding}, isDraining=${this.isDraining}`);
|
|
90
|
-
return;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
this.isCordon = active;
|
|
94
|
-
const processLabel = this.isPrimary ? `Primary (${process.pid})` : `Worker ${process.pid} (ID: ${this.workerId})`;
|
|
95
|
-
console.log(`[@cocreate/server] ${processLabel} isCordon updated to: ${this.isCordon}`);
|
|
96
|
-
});
|
|
97
71
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
if (this.isShedding) {
|
|
103
|
-
this.isCordon = true;
|
|
104
|
-
|
|
105
|
-
// Selective Shedding Integration: Target specified organizations inside the socket server
|
|
106
|
-
if (payload?.organizations && this.wsManager && typeof this.wsManager.destroyOrganizations === 'function') {
|
|
107
|
-
const orgList = Array.isArray(payload.organizations) ? payload.organizations : [payload.organizations];
|
|
108
|
-
console.log(`[@cocreate/server] Shedding event triggered. Selective eviction of org IDs:`, orgList);
|
|
109
|
-
this.wsManager.destroyOrganizations(orgList);
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
const processLabel = this.isPrimary ? `Primary (${process.pid})` : `Worker ${process.pid} (ID: ${this.workerId})`;
|
|
113
|
-
console.log(`[@cocreate/server] ${processLabel} isShedding updated to: ${this.isShedding} (isCordon: ${this.isCordon})`);
|
|
114
|
-
});
|
|
72
|
+
// Initialize reactive cluster storage
|
|
73
|
+
this.storage = new Storage(this, pendingPromises);
|
|
74
|
+
this.store = this.storage;
|
|
75
|
+
this.workerStorage = this.storage;
|
|
115
76
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
this.isCordon = true;
|
|
122
|
-
|
|
123
|
-
// Complete Worker Drain Integration: Instantly purge all sockets and loaded maps
|
|
124
|
-
if (this.wsManager && typeof this.wsManager.destroyAll === 'function') {
|
|
125
|
-
console.log(`[@cocreate/server] Worker-level drain event triggered. Purging all tenant configurations.`);
|
|
126
|
-
this.wsManager.destroyAll();
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
const processLabel = this.isPrimary ? `Primary (${process.pid})` : `Worker ${process.pid} (ID: ${this.workerId})`;
|
|
130
|
-
console.log(`[@cocreate/server] ${processLabel} isDraining updated to: ${this.isDraining} (isCordon: ${this.isCordon})`);
|
|
131
|
-
});
|
|
77
|
+
// Attach State Manager module (attaches cordon, shed, drain, publishState, process signals)
|
|
78
|
+
State.init(this);
|
|
79
|
+
|
|
80
|
+
// Attach Log Manager module
|
|
81
|
+
this.logManager = LogManager.init(this);
|
|
132
82
|
|
|
83
|
+
// Master unencrypted HTTP edge listener for ACME and redirect handling
|
|
133
84
|
this.http = Http.createServer((req, res) => {
|
|
134
85
|
const requestUrl = req.url || '/';
|
|
86
|
+
|
|
135
87
|
// 1. Allow ACME challenges to pass through over unencrypted HTTP
|
|
136
88
|
if (requestUrl.startsWith('/.well-known/acme-challenge/')) {
|
|
137
89
|
return this.handleEdgeRequest(req, res);
|
|
138
90
|
}
|
|
139
91
|
|
|
92
|
+
// 2. Force a 301 redirect to HTTPS for all other traffic
|
|
140
93
|
const host = req.headers.host ? req.headers.host.split(':')[0] : 'localhost';
|
|
141
|
-
|
|
142
|
-
// 2. Intercept with a 307 Temporary Redirect if cordon is active
|
|
143
|
-
if (this.isCordon) {
|
|
144
|
-
res.writeHead(307, { "Location": `https://${host}${requestUrl}` });
|
|
145
|
-
return res.end();
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
// 3. Force a 301 redirect to HTTPS for all other traffic
|
|
149
94
|
res.writeHead(301, { "Location": `https://${host}${requestUrl}` });
|
|
150
95
|
res.end();
|
|
151
96
|
});
|
|
152
|
-
|
|
153
|
-
this.https = null;
|
|
154
|
-
this.mesh = null;
|
|
155
|
-
this.autoscaler = null;
|
|
97
|
+
|
|
156
98
|
this.authenticate = authenticate;
|
|
157
99
|
this.authorize = authorize;
|
|
158
|
-
this.wsManager = null;
|
|
159
|
-
this.crud = null;
|
|
160
|
-
this.telemetry = null;
|
|
161
|
-
this.files = null;
|
|
162
|
-
this.loader = null;
|
|
163
100
|
}
|
|
164
101
|
|
|
165
|
-
|
|
166
|
-
if (
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
102
|
+
closeEdgeListeners() {
|
|
103
|
+
if (this.isMaster) return;
|
|
104
|
+
if (this.http && this.http.listening) this.http.close();
|
|
105
|
+
if (this.https && this.https.listening) this.https.close();
|
|
106
|
+
}
|
|
170
107
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
}
|
|
108
|
+
restoreEdgeListeners() {
|
|
109
|
+
if (this.isMaster || this.state.isDraining) return;
|
|
110
|
+
const HTTP_PORT = process.env.HTTP_PORT || 80;
|
|
111
|
+
const HTTPS_PORT = process.env.HTTPS_PORT || 443;
|
|
112
|
+
const BIND_IP = process.env.BIND_IP || '0.0.0.0';
|
|
177
113
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
114
|
+
if (!this.http.listening) this.http.listen(HTTP_PORT, BIND_IP);
|
|
115
|
+
if (this.https && !this.https.listening) this.https.listen(HTTPS_PORT, BIND_IP);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
evaluateMasterFallback() {
|
|
119
|
+
if (!this.isMaster) return;
|
|
120
|
+
|
|
121
|
+
const allWorkersDown = Array.from(this.workers.values()).every(w => w.isListening === false);
|
|
122
|
+
|
|
123
|
+
if (allWorkersDown) {
|
|
124
|
+
if (!this.fallbackHttpServer) {
|
|
125
|
+
const HTTP_PORT = process.env.HTTP_PORT || 80;
|
|
126
|
+
const HTTPS_PORT = process.env.HTTPS_PORT || 443;
|
|
127
|
+
const BIND_IP = process.env.BIND_IP || '0.0.0.0';
|
|
128
|
+
|
|
129
|
+
const handleFallback = (req, res) => {
|
|
130
|
+
res.writeHead(503, {
|
|
131
|
+
"Content-Type": "text/plain",
|
|
132
|
+
"Retry-After": "5"
|
|
133
|
+
});
|
|
134
|
+
res.end("Service Unavailable: All workers cordoned or inactive");
|
|
135
|
+
};
|
|
136
|
+
this.fallbackHttpServer = Http.createServer(handleFallback);
|
|
137
|
+
this.fallbackHttpServer.listen(HTTP_PORT, BIND_IP);
|
|
138
|
+
|
|
139
|
+
if (this.httpsOptions) {
|
|
140
|
+
this.fallbackHttpsServer = Https.createServer(this.httpsOptions, handleFallback);
|
|
141
|
+
this.fallbackHttpsServer.listen(HTTPS_PORT, BIND_IP);
|
|
192
142
|
}
|
|
143
|
+
console.log(`[@cocreate/server] [MASTER] All workers cordoned/down.`);
|
|
193
144
|
}
|
|
194
145
|
} else {
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
try {
|
|
202
|
-
process.send({ ipcPayload: true, payload });
|
|
203
|
-
} catch (err) {
|
|
204
|
-
console.error("[@cocreate/server] Failed sending upstream IPC payload to Master:", err);
|
|
146
|
+
if (this.fallbackHttpServer) {
|
|
147
|
+
this.fallbackHttpServer.close();
|
|
148
|
+
this.fallbackHttpServer = null;
|
|
149
|
+
if (this.fallbackHttpsServer) {
|
|
150
|
+
this.fallbackHttpsServer.close();
|
|
151
|
+
this.fallbackHttpsServer = null;
|
|
205
152
|
}
|
|
153
|
+
console.log(`[@cocreate/server] [MASTER] Workers active.`);
|
|
206
154
|
}
|
|
207
155
|
}
|
|
208
156
|
}
|
|
@@ -213,27 +161,32 @@ export class CoCreateServer extends EventEmitter {
|
|
|
213
161
|
}
|
|
214
162
|
|
|
215
163
|
this.isInitialized = true;
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
this.workerStorage = {
|
|
219
|
-
set: (key, value) => workerStorageManager(this, 'set', key, value),
|
|
220
|
-
get: (key) => workerStorageManager(this, 'get', key),
|
|
221
|
-
delete: (key) => workerStorageManager(this, 'delete', key),
|
|
222
|
-
size: () => workerStorageManager(this, 'size'),
|
|
223
|
-
clear: () => workerStorageManager(this, 'clear')
|
|
224
|
-
};
|
|
164
|
+
this.send = IPC.init(this, pendingPromises);
|
|
225
165
|
|
|
226
|
-
const config = await Config({
|
|
166
|
+
const config = await Config.request({
|
|
227
167
|
'organization_id': '',
|
|
228
168
|
'host': '',
|
|
229
|
-
'workers': ''
|
|
169
|
+
'workers': '',
|
|
170
|
+
'server_id': ''
|
|
230
171
|
});
|
|
231
172
|
|
|
232
173
|
this.organization_id = config.organization_id || null;
|
|
233
174
|
this.host = config.host || null;
|
|
234
175
|
|
|
235
|
-
|
|
176
|
+
if (this.isMaster) {
|
|
177
|
+
if (config.server_id) {
|
|
178
|
+
this.id = config.server_id;
|
|
179
|
+
} else {
|
|
180
|
+
this.id = process.env.SERVER_ID || ObjectId().toString();
|
|
181
|
+
await Config.request({ server_id: { value: this.id } });
|
|
182
|
+
console.log(`[@cocreate/server] [MASTER] Generated & persisted new Server ID: ${this.id}`);
|
|
183
|
+
}
|
|
184
|
+
process.env.SERVER_ID = this.id;
|
|
185
|
+
} else {
|
|
186
|
+
this.id = process.env.SERVER_ID;
|
|
187
|
+
}
|
|
236
188
|
|
|
189
|
+
let configuredWorkers = config.workers;
|
|
237
190
|
if (configuredWorkers !== false && configuredWorkers !== 'false') {
|
|
238
191
|
const parsedWorkers = parseInt(configuredWorkers, 10);
|
|
239
192
|
if (parsedWorkers === -1) {
|
|
@@ -243,7 +196,7 @@ export class CoCreateServer extends EventEmitter {
|
|
|
243
196
|
}
|
|
244
197
|
}
|
|
245
198
|
|
|
246
|
-
if (this.
|
|
199
|
+
if (this.isMaster && this.totalWorkers <= 1) {
|
|
247
200
|
this.workerId = 1;
|
|
248
201
|
}
|
|
249
202
|
|
|
@@ -251,43 +204,74 @@ export class CoCreateServer extends EventEmitter {
|
|
|
251
204
|
this.mesh = ServerMesh;
|
|
252
205
|
this.autoscaler = ServerAutoscaler;
|
|
253
206
|
|
|
254
|
-
if (this.
|
|
255
|
-
console.log(`[@cocreate/server] [
|
|
256
|
-
|
|
257
|
-
this.ip =
|
|
258
|
-
this.infrastructure = await
|
|
207
|
+
if (this.isMaster) {
|
|
208
|
+
console.log(`[@cocreate/server] [MASTER] Orchestrator running with PID: ${process.pid} (Server ID: ${this.id})`);
|
|
209
|
+
this.location = await getClientInfo()
|
|
210
|
+
this.ip = this.location.ip;
|
|
211
|
+
this.infrastructure = await getInfrastructure(this.id);
|
|
259
212
|
await this.mesh.init(this);
|
|
260
213
|
await this.autoscaler.init(this);
|
|
261
214
|
|
|
262
|
-
//
|
|
263
|
-
|
|
215
|
+
// Reset logs array for a fresh logging session on master startup
|
|
216
|
+
await this.crud.send(this.getServerObject({ logs: [] }));
|
|
217
|
+
|
|
218
|
+
// Register initial running profile to DB for Master
|
|
219
|
+
await this.publishState();
|
|
220
|
+
|
|
221
|
+
// Zero-out worker states in DB upon master initialization
|
|
222
|
+
if (this.totalWorkers > 0) {
|
|
223
|
+
const initialWorkerStates = {};
|
|
224
|
+
for (let i = 1; i <= this.totalWorkers; i++) {
|
|
225
|
+
initialWorkerStates[`worker${i}`] = {
|
|
226
|
+
pid: null,
|
|
227
|
+
status: 'stopped',
|
|
228
|
+
isCordoned: false,
|
|
229
|
+
isShedding: false,
|
|
230
|
+
isDraining: false
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
await this.crud.send(this.getServerObject(initialWorkerStates));
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// Route mesh commands to IPC
|
|
238
|
+
const systemEvents = ['restart', 'reboot', 'shutdown', 'cordon', 'shed', 'drain'];
|
|
239
|
+
for (const eventName of systemEvents) {
|
|
240
|
+
this.mesh.on(eventName, (payload) => {
|
|
241
|
+
console.log(`[@cocreate/server] [MASTER] Received remote mesh command: "${eventName}"`);
|
|
242
|
+
this.send(payload);
|
|
243
|
+
});
|
|
244
|
+
}
|
|
264
245
|
|
|
265
246
|
if (this.totalWorkers > 1) {
|
|
266
|
-
console.log(`[@cocreate/server] [
|
|
247
|
+
console.log(`[@cocreate/server] [MASTER] Orchestrator creating ${this.totalWorkers} workers.`);
|
|
248
|
+
|
|
249
|
+
const generateWorkerEnv = (wId) => ({
|
|
250
|
+
WORKER_ID: String(wId),
|
|
251
|
+
SERVER_ID: String(this.id || ''),
|
|
252
|
+
SERVER_IP: String(this.ip || '127.0.0.1'),
|
|
253
|
+
SERVER_INFRASTRUCTURE: JSON.stringify(this.infrastructure || {}),
|
|
254
|
+
SERVER_LOCATION: JSON.stringify(this.location || {})
|
|
255
|
+
});
|
|
256
|
+
|
|
267
257
|
for (let i = 0; i < this.totalWorkers; i++) {
|
|
268
258
|
const currentWorkerId = i + 1;
|
|
269
|
-
const worker = cluster.fork(
|
|
270
|
-
WORKER_ID: currentWorkerId,
|
|
271
|
-
SERVER_ID: this.id,
|
|
272
|
-
SERVER_IP: this.ip,
|
|
273
|
-
SERVER_INFRASTRUCTURE: JSON.stringify(this.infrastructure)
|
|
274
|
-
});
|
|
259
|
+
const worker = cluster.fork(generateWorkerEnv(currentWorkerId));
|
|
275
260
|
|
|
276
|
-
// Protect against unhandled 'error' events on the cluster worker wrapper
|
|
277
261
|
worker.on('error', (err) => {
|
|
278
|
-
console.error(`[@cocreate/server] [
|
|
262
|
+
console.error(`[@cocreate/server] [MASTER] Error on Worker ${currentWorkerId}:`, err.message);
|
|
279
263
|
});
|
|
280
264
|
|
|
281
265
|
activeWorkers.set(currentWorkerId, {
|
|
282
266
|
id: currentWorkerId,
|
|
283
267
|
pid: worker.process.pid,
|
|
284
|
-
process: worker
|
|
268
|
+
process: worker,
|
|
269
|
+
isListening: false
|
|
285
270
|
});
|
|
286
271
|
}
|
|
287
272
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
console.log(`[@cocreate/server] [PRIMARY] Worker ${worker.process.pid} exited (Code: ${code}, Signal: ${signal})`);
|
|
273
|
+
cluster.on('exit', async (worker, code, signal) => {
|
|
274
|
+
console.log(`[@cocreate/server] [MASTER] Worker ${worker.process.pid} exited (Code: ${code}, Signal: ${signal})`);
|
|
291
275
|
|
|
292
276
|
let deadWorkerId = null;
|
|
293
277
|
for (let [id, w] of activeWorkers.entries()) {
|
|
@@ -297,87 +281,124 @@ export class CoCreateServer extends EventEmitter {
|
|
|
297
281
|
break;
|
|
298
282
|
}
|
|
299
283
|
}
|
|
284
|
+
this.evaluateMasterFallback();
|
|
285
|
+
|
|
286
|
+
if (deadWorkerId) {
|
|
287
|
+
await this.crud.send(this.getServerObject({
|
|
288
|
+
[`worker${deadWorkerId}`]: {
|
|
289
|
+
pid: null,
|
|
290
|
+
status: 'stopped',
|
|
291
|
+
isCordoned: false,
|
|
292
|
+
isShedding: false,
|
|
293
|
+
isDraining: false
|
|
294
|
+
}
|
|
295
|
+
}));
|
|
296
|
+
console.log(`[@cocreate/server] [MASTER] Reset worker${deadWorkerId} state in DB to stopped`);
|
|
297
|
+
}
|
|
300
298
|
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
console.log(`[@cocreate/server] [PRIMARY] Worker ${worker.process.pid} removed. Draining active workers: ${activeWorkers.size} remaining.`);
|
|
299
|
+
if (this.state.isShuttingDown || this.state.isDraining) {
|
|
300
|
+
console.log(`[@cocreate/server] [MASTER] Worker ${worker.process.pid} removed. Active workers remaining: ${activeWorkers.size}`);
|
|
304
301
|
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
if (
|
|
309
|
-
|
|
302
|
+
if (activeWorkers.size === 0) {
|
|
303
|
+
console.log("[@cocreate/server] [MASTER] All workers successfully drained.");
|
|
304
|
+
if (this.state.masterShutdownTimeout) clearTimeout(this.state.masterShutdownTimeout);
|
|
305
|
+
if (typeof this.state.shutdownCompletionCallback === 'function') {
|
|
306
|
+
this.state.shutdownCompletionCallback();
|
|
307
|
+
} else {
|
|
308
|
+
process.exit(0);
|
|
310
309
|
}
|
|
311
|
-
process.exit(0);
|
|
312
310
|
}
|
|
313
311
|
return;
|
|
314
312
|
}
|
|
315
313
|
|
|
316
314
|
const targetId = deadWorkerId || (activeWorkers.size + 1);
|
|
317
315
|
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
});
|
|
316
|
+
setTimeout(() => {
|
|
317
|
+
console.log(`[@cocreate/server] [MASTER] Respawning Worker ${targetId}...`);
|
|
318
|
+
const newWorker = cluster.fork(generateWorkerEnv(targetId));
|
|
319
|
+
|
|
320
|
+
newWorker.on('error', (err) => {
|
|
321
|
+
console.error(`[@cocreate/server] [MASTER] Error on replacement Worker ${targetId}:`, err.message);
|
|
322
|
+
});
|
|
323
|
+
|
|
324
|
+
activeWorkers.set(targetId, {
|
|
325
|
+
id: targetId,
|
|
326
|
+
pid: newWorker.process.pid,
|
|
327
|
+
process: newWorker,
|
|
328
|
+
isListening: false
|
|
329
|
+
});
|
|
330
|
+
}, 2000);
|
|
334
331
|
});
|
|
335
332
|
|
|
336
|
-
|
|
337
|
-
return this;
|
|
338
|
-
}
|
|
333
|
+
return this;
|
|
339
334
|
}
|
|
340
335
|
}
|
|
341
|
-
|
|
342
|
-
this.ip = process.env.SERVER_IP || await getIP();
|
|
343
|
-
|
|
344
|
-
try {
|
|
345
|
-
this.infrastructure = process.env.SERVER_INFRASTRUCTURE
|
|
346
|
-
? JSON.parse(process.env.SERVER_INFRASTRUCTURE)
|
|
347
|
-
: await resolveInfrastructure(this.id);
|
|
348
|
-
} catch (error) {
|
|
349
|
-
console.error(`[@cocreate/server] Worker ${process.pid} failed to parse pre-resolved infrastructure. Falling back...`, error);
|
|
350
|
-
this.infrastructure = await resolveInfrastructure(this.id);
|
|
351
|
-
}
|
|
352
336
|
|
|
353
|
-
this.
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
await this.certificate.init(this);
|
|
337
|
+
if (!this.isMaster || this.totalWorkers <= 1) {
|
|
338
|
+
if (!this.isMaster) {
|
|
339
|
+
console.log(`[@cocreate/server] [WORKER ${this.workerId}] running with PID: ${process.pid} (Server ID: ${this.id})`);
|
|
340
|
+
}
|
|
358
341
|
|
|
359
|
-
|
|
360
|
-
this.api = await ApiServer.init(this);
|
|
361
|
-
this.files = FileServer.init(this);
|
|
362
|
-
this.loader = await LazyLoader.init(this);
|
|
342
|
+
this.telemetry = await ServerTelemetry.init(this);
|
|
363
343
|
|
|
364
|
-
|
|
344
|
+
if (!this.ip) {
|
|
345
|
+
this.ip = process.env.SERVER_IP || (await getClientInfo()).ip;
|
|
346
|
+
}
|
|
365
347
|
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
348
|
+
if (!this.infrastructure) {
|
|
349
|
+
try {
|
|
350
|
+
this.infrastructure = process.env.SERVER_INFRASTRUCTURE
|
|
351
|
+
? JSON.parse(process.env.SERVER_INFRASTRUCTURE)
|
|
352
|
+
: await getInfrastructure(this.id);
|
|
353
|
+
} catch (error) {
|
|
354
|
+
this.infrastructure = await getInfrastructure(this.id);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
if (!this.location) {
|
|
359
|
+
try {
|
|
360
|
+
this.location = process.env.SERVER_LOCATION
|
|
361
|
+
? JSON.parse(process.env.SERVER_LOCATION)
|
|
362
|
+
: await getClientInfo(this.id);
|
|
363
|
+
} catch (error) {
|
|
364
|
+
this.location = await getClientInfo(this.id);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
this.certificate = Acme;
|
|
369
|
+
this.httpsOptions = {
|
|
370
|
+
SNICallback: (domain, cb) => this.certificate.sniCallback(domain, cb)
|
|
371
|
+
};
|
|
372
|
+
this.https = Https.createServer(this.httpsOptions, (req, res) => this.handleEdgeRequest(req, res));
|
|
373
|
+
await this.certificate.init(this);
|
|
374
|
+
|
|
375
|
+
this.wsManager = new SocketServer(this);
|
|
376
|
+
this.api = await ApiServer.init(this);
|
|
377
|
+
this.files = FileServer.init(this);
|
|
378
|
+
this.loader = await LazyLoader.init(this);
|
|
379
|
+
|
|
380
|
+
const HTTP_PORT = process.env.HTTP_PORT || 80;
|
|
381
|
+
const HTTPS_PORT = process.env.HTTPS_PORT || 443;
|
|
382
|
+
const BIND_IP = process.env.BIND_IP || '0.0.0.0';
|
|
383
|
+
|
|
384
|
+
const reportListening = (state) => {
|
|
385
|
+
if (process.connected) {
|
|
386
|
+
process.send({ type: 'listening_state', state });
|
|
387
|
+
}
|
|
388
|
+
};
|
|
370
389
|
|
|
371
|
-
|
|
390
|
+
this.http.on('listening', () => reportListening(true));
|
|
391
|
+
this.http.on('close', () => reportListening(false));
|
|
372
392
|
this.http.listen(HTTP_PORT, BIND_IP, () => {
|
|
373
|
-
console.log(`[@cocreate/server] ${
|
|
393
|
+
console.log(`[@cocreate/server] [WORKER ${this.workerId}] HTTP listening on ${BIND_IP}:${HTTP_PORT}`);
|
|
374
394
|
});
|
|
375
|
-
}
|
|
376
395
|
|
|
377
|
-
if (this.https) {
|
|
378
396
|
this.https.listen(HTTPS_PORT, BIND_IP, () => {
|
|
379
|
-
console.log(`[@cocreate/server] ${
|
|
397
|
+
console.log(`[@cocreate/server] [WORKER ${this.workerId}] HTTPS listening on ${BIND_IP}:${HTTPS_PORT}`);
|
|
380
398
|
});
|
|
399
|
+
|
|
400
|
+
// Publish worker active running state object to DB upon creation/startup
|
|
401
|
+
await this.publishState('running');
|
|
381
402
|
}
|
|
382
403
|
|
|
383
404
|
return this;
|
|
@@ -387,183 +408,14 @@ export class CoCreateServer extends EventEmitter {
|
|
|
387
408
|
try {
|
|
388
409
|
const requestUrl = req.url || '/';
|
|
389
410
|
|
|
390
|
-
// Check if cordon is active (exclude critical ACME challenges from loop-breaking)
|
|
391
|
-
if (this.isCordon && !requestUrl.startsWith('/.well-known/acme-challenge/')) {
|
|
392
|
-
const host = req.headers.host ? req.headers.host.split(':')[0] : 'localhost';
|
|
393
|
-
res.writeHead(307, { "Location": `https://${host}${requestUrl}` });
|
|
394
|
-
return res.end();
|
|
395
|
-
}
|
|
396
|
-
|
|
397
411
|
if (requestUrl.startsWith("/webhooks/")) {
|
|
398
|
-
|
|
399
|
-
}
|
|
400
|
-
if (this.files) return this.files.send(req, res);
|
|
401
|
-
|
|
402
|
-
res.writeHead(500, { "Content-Type": "text/plain" });
|
|
403
|
-
res.end("Internal Error: Files component missing");
|
|
404
|
-
} catch (error) {
|
|
405
|
-
res.writeHead(400, { "Content-Type": "text/plain" });
|
|
406
|
-
res.end("Invalid request routing context");
|
|
407
|
-
}
|
|
408
|
-
}
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
function executeLocalWorkerStorageOperation(method, key, value) {
|
|
412
|
-
switch (method) {
|
|
413
|
-
case 'set': sharedMap.set(key, value); return true;
|
|
414
|
-
case 'get': return sharedMap.get(key);
|
|
415
|
-
case 'delete': return sharedMap.delete(key);
|
|
416
|
-
case 'size': return sharedMap.size;
|
|
417
|
-
case 'clear': sharedMap.clear(); return true;
|
|
418
|
-
default: throw new Error(`Unsupported storage key operation: "${method}"`);
|
|
419
|
-
}
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
function workerStorageManager(serverCtx, method, key, value) {
|
|
423
|
-
return new Promise((resolve, reject) => {
|
|
424
|
-
try {
|
|
425
|
-
if (serverCtx.isPrimary) {
|
|
426
|
-
resolve(executeLocalWorkerStorageOperation(method, key, value));
|
|
427
|
-
} else {
|
|
428
|
-
const messageId = uid.generate();
|
|
429
|
-
pendingPromises.set(messageId, { resolve, reject });
|
|
430
|
-
process.send({ id: serverCtx.id, messageId, method, key, value });
|
|
412
|
+
return this.loader.request(req, res);
|
|
431
413
|
}
|
|
414
|
+
return this.files.send(req, res);
|
|
432
415
|
} catch (error) {
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
});
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
function setupPrimaryShutdownHooks(serverCtx) {
|
|
439
|
-
const handleSignal = async (signal) => {
|
|
440
|
-
if (isPrimaryShuttingDown) return;
|
|
441
|
-
isPrimaryShuttingDown = true;
|
|
442
|
-
serverCtx.isCordon = true;
|
|
443
|
-
serverCtx.isDraining = true;
|
|
444
|
-
|
|
445
|
-
console.log(`\n[@cocreate/server] [PRIMARY] Intercepted process termination signal: ${signal}. Commencing cluster-wide graceful shutdown...`);
|
|
446
|
-
|
|
447
|
-
// STEP 1: Close Mesh FIRST. Wipes the server record profile from the database while CRUD connection pools are completely active.
|
|
448
|
-
if (serverCtx.mesh && typeof serverCtx.mesh.close === 'function') {
|
|
449
|
-
try {
|
|
450
|
-
console.log("[@cocreate/server] [PRIMARY] Step 1: Initiating Server Mesh close & server record database removal...");
|
|
451
|
-
await serverCtx.mesh.close();
|
|
452
|
-
console.log("[@cocreate/server] [PRIMARY] Success: Server Mesh closed and server record cleanly wiped from database.");
|
|
453
|
-
} catch (err) {
|
|
454
|
-
console.error("[@cocreate/server] [PRIMARY] Failed to close Server Mesh or wipe server profile cleanly:", err);
|
|
455
|
-
}
|
|
456
|
-
}
|
|
457
|
-
isMeshClosed = true;
|
|
458
|
-
|
|
459
|
-
// Dual-Lock Resolution: If all workers finished exiting while the database prune was running,
|
|
460
|
-
// terminate the Master process cleanly right now instead of waiting for the safety timeout.
|
|
461
|
-
if (activeWorkers.size === 0) {
|
|
462
|
-
console.log("[@cocreate/server] [PRIMARY] All workers successfully drained and Mesh closed. Terminating master process cleanly.");
|
|
463
|
-
if (primaryShutdownTimeout) {
|
|
464
|
-
clearTimeout(primaryShutdownTimeout);
|
|
465
|
-
}
|
|
466
|
-
process.exit(0);
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
// STEP 2: Now that the database profile is clean, tell all worker processes over IPC to begin local graceful drains.
|
|
470
|
-
console.log("[@cocreate/server] [PRIMARY] Step 2: Informing worker processes to start local connection drains...");
|
|
471
|
-
for (const [wId, worker] of activeWorkers.entries()) {
|
|
472
|
-
const workerInstance = worker.process;
|
|
473
|
-
if (workerInstance && typeof workerInstance.send === 'function' && workerInstance.isConnected()) {
|
|
474
|
-
console.log(`[@cocreate/server] [PRIMARY] Sending 'shutdown' instruction to Worker ${wId} (PID: ${worker.pid})`);
|
|
475
|
-
try {
|
|
476
|
-
workerInstance.send({ type: 'shutdown' });
|
|
477
|
-
} catch (err) {
|
|
478
|
-
console.error(`[@cocreate/server] Failed sending shutdown IPC instruction to worker ${wId}:`, err);
|
|
479
|
-
}
|
|
480
|
-
} else {
|
|
481
|
-
console.log(`[@cocreate/server] [PRIMARY] Skipping disconnected Worker ${wId} (PID: ${worker.pid})`);
|
|
482
|
-
}
|
|
416
|
+
res.writeHead(500, { "Content-Type": "text/plain" });
|
|
417
|
+
res.end("Internal Error: Request routing failed");
|
|
483
418
|
}
|
|
484
|
-
|
|
485
|
-
// STEP 3: Set Cooperative Grace Fallback Timer (5 Seconds)
|
|
486
|
-
// If workers finish early, the exit event triggers process exit immediately. If they hang, we force-kill.
|
|
487
|
-
primaryShutdownTimeout = setTimeout(() => {
|
|
488
|
-
console.warn("[@cocreate/server] [PRIMARY] Safety grace window expired! Force-terminating remaining processes...");
|
|
489
|
-
|
|
490
|
-
for (const [wId, worker] of activeWorkers.entries()) {
|
|
491
|
-
if (worker?.process) {
|
|
492
|
-
try {
|
|
493
|
-
console.log(`[@cocreate/server] [PRIMARY] Force-killing hanging Worker ${wId} (PID: ${worker.pid}).`);
|
|
494
|
-
worker.process.kill('SIGKILL');
|
|
495
|
-
} catch (err) {
|
|
496
|
-
console.error(`[@cocreate/server] Error force-killing Worker ${wId}:`, err);
|
|
497
|
-
}
|
|
498
|
-
}
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
console.log("[@cocreate/server] [PRIMARY] All worker processes forcefully reaped. Terminating Master process.");
|
|
502
|
-
process.exit(0);
|
|
503
|
-
}, 5000);
|
|
504
|
-
};
|
|
505
|
-
|
|
506
|
-
process.on('SIGTERM', () => handleSignal('SIGTERM'));
|
|
507
|
-
process.on('SIGINT', () => handleSignal('SIGINT'));
|
|
508
|
-
process.on('SIGBREAK', () => handleSignal('SIGBREAK'));
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
function setupIPC(serverCtx) {
|
|
512
|
-
if (serverCtx.isPrimary) {
|
|
513
|
-
cluster.on('message', (worker, message) => {
|
|
514
|
-
if (message && message.id === serverCtx.id) {
|
|
515
|
-
try {
|
|
516
|
-
message.value = executeLocalWorkerStorageOperation(message.method, message.key, message.value);
|
|
517
|
-
worker.send(message);
|
|
518
|
-
} catch (error) {
|
|
519
|
-
message.error = error.error;
|
|
520
|
-
worker.send(message);
|
|
521
|
-
}
|
|
522
|
-
}
|
|
523
|
-
else if (message && message.ipcPayload && message.payload) {
|
|
524
|
-
const { payload } = message;
|
|
525
|
-
|
|
526
|
-
let senderWorkerId = null;
|
|
527
|
-
for (const [wId, w] of serverCtx.workers.entries()) {
|
|
528
|
-
if (w.pid === worker.process.pid) {
|
|
529
|
-
senderWorkerId = wId;
|
|
530
|
-
break;
|
|
531
|
-
}
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
payload.senderWorkerId = senderWorkerId;
|
|
535
|
-
serverCtx.send(payload);
|
|
536
|
-
}
|
|
537
|
-
});
|
|
538
|
-
} else {
|
|
539
|
-
process.on('message', (message) => {
|
|
540
|
-
// Intercept Master-directed shutdown instruction over the worker's IPC channel
|
|
541
|
-
if (message && message.type === 'shutdown') {
|
|
542
|
-
console.log(`[@cocreate/server] [Worker ${process.pid}] Received master coordinated shutdown signal. Commencing local drain...`);
|
|
543
|
-
|
|
544
|
-
// Immediately cordon worker traffic and trigger the internal SocketServer destroyAll logic
|
|
545
|
-
serverCtx.isCordon = true;
|
|
546
|
-
serverCtx.isDraining = true;
|
|
547
|
-
|
|
548
|
-
if (serverCtx.wsManager && typeof serverCtx.wsManager.destroyAll === 'function') {
|
|
549
|
-
serverCtx.wsManager.destroyAll();
|
|
550
|
-
}
|
|
551
|
-
}
|
|
552
|
-
else if (message && message.id === serverCtx.id) {
|
|
553
|
-
const pending = pendingPromises.get(message.messageId);
|
|
554
|
-
if (pending) {
|
|
555
|
-
pendingPromises.delete(message.messageId);
|
|
556
|
-
if (message.error) {
|
|
557
|
-
pending.reject(new Error(message.error));
|
|
558
|
-
} else {
|
|
559
|
-
pending.resolve(message.value);
|
|
560
|
-
}
|
|
561
|
-
}
|
|
562
|
-
}
|
|
563
|
-
else if (message && message.ipcPayload && message.payload) {
|
|
564
|
-
serverCtx.emit(message.payload.method, message.payload);
|
|
565
|
-
}
|
|
566
|
-
});
|
|
567
419
|
}
|
|
568
420
|
}
|
|
569
421
|
|