@cocreate/server-mesh 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/FUNDING.yml +3 -0
- package/.github/workflows/automated.yml +56 -0
- package/CHANGELOG.md +24 -0
- package/CONTRIBUTING.md +97 -0
- package/CoCreate.config.js +23 -0
- package/LICENSE +683 -0
- package/README.md +65 -0
- package/demo/server.js +24 -0
- package/docs/index.html +217 -0
- package/package.json +37 -0
- package/prettier.config.js +16 -0
- package/release.config.js +30 -0
- package/src/index.js +731 -0
package/src/index.js
ADDED
|
@@ -0,0 +1,731 @@
|
|
|
1
|
+
/********************************************************************************
|
|
2
|
+
* Copyright (C) 2023 CoCreate and Contributors.
|
|
3
|
+
*
|
|
4
|
+
* This program is free software: you can redistribute it and/or modify
|
|
5
|
+
* it under the terms of the GNU Affero General Public License as published
|
|
6
|
+
* by the Free Software Foundation, either version 3 of the License, or
|
|
7
|
+
* (at your option) any later version.
|
|
8
|
+
*
|
|
9
|
+
* This program is distributed in the hope that it will be useful,
|
|
10
|
+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
12
|
+
* GNU Affero General Public License for more details.
|
|
13
|
+
*
|
|
14
|
+
* You should have received a copy of the GNU Affero General Public License
|
|
15
|
+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
16
|
+
*
|
|
17
|
+
********************************************************************************/
|
|
18
|
+
|
|
19
|
+
import { WebSocketServer, WebSocket } from 'ws';
|
|
20
|
+
import https from 'node:https';
|
|
21
|
+
import fs from 'node:fs';
|
|
22
|
+
import path from 'node:path';
|
|
23
|
+
import os from 'node:os';
|
|
24
|
+
import cluster from 'node:cluster';
|
|
25
|
+
import { execSync } from 'node:child_process';
|
|
26
|
+
import { EventEmitter } from 'node:events';
|
|
27
|
+
import crypto from 'node:crypto';
|
|
28
|
+
|
|
29
|
+
const activeConnections = new Map(); // serverId/IP -> WebSocket P2P tunnels
|
|
30
|
+
const retryTimers = new Map(); // peerIp -> Timeout ID
|
|
31
|
+
const pruneTimers = new Map(); // peerIp -> Timeout ID
|
|
32
|
+
const orgWorkers = new Map(); // organization_id -> Set(workerId)
|
|
33
|
+
const orgServers = new Map(); // organization_id -> Set(WebSocket)
|
|
34
|
+
|
|
35
|
+
let isGracefulExitRunning = false;
|
|
36
|
+
let wsPingInterval = null;
|
|
37
|
+
|
|
38
|
+
let serverCtx = null;
|
|
39
|
+
const MESH_PORT = process.env.MESH_PORT || 8090;
|
|
40
|
+
let meshWss = null;
|
|
41
|
+
let secureMeshServer = null;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Safely loops and sends JSON payloads to one or multiple sockets.
|
|
45
|
+
* Wraps individual operations in independent try-catch layers to avoid process instability.
|
|
46
|
+
*
|
|
47
|
+
* @param {WebSocket|WebSocket[]} sockets - Target WebSocket(s)
|
|
48
|
+
* @param {Object} data - Payload content to dispatch
|
|
49
|
+
*/
|
|
50
|
+
function socketSend(sockets, data) {
|
|
51
|
+
const targetSockets = Array.isArray(sockets) ? sockets : [sockets];
|
|
52
|
+
const dataStr = typeof data === 'string' ? data : JSON.stringify(data);
|
|
53
|
+
|
|
54
|
+
for (const socket of targetSockets) {
|
|
55
|
+
try {
|
|
56
|
+
if (socket && socket.readyState === WebSocket.OPEN) {
|
|
57
|
+
socket.send(dataStr);
|
|
58
|
+
}
|
|
59
|
+
} catch (err) {
|
|
60
|
+
console.error(`[@cocreate/server-mesh] Error executing socketSend transaction:`, err.message);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* ServerMesh acts as a pure transport fabric. It blindly dispatches
|
|
67
|
+
* all inbound network payloads as standard EventEmitter events.
|
|
68
|
+
*/
|
|
69
|
+
class ServerMesh extends EventEmitter {
|
|
70
|
+
constructor() {
|
|
71
|
+
super();
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Sends a payload across the secure network mesh.
|
|
76
|
+
* Automatically appends a unique ID (uid) to guarantee end-to-end tracking.
|
|
77
|
+
*
|
|
78
|
+
* @param {Object} payload - Object to transmit containing a required 'method'
|
|
79
|
+
*/
|
|
80
|
+
send(payload) {
|
|
81
|
+
if (!payload || !payload.method) {
|
|
82
|
+
console.error("[@cocreate/server-mesh] [MESH ERROR] Cannot transmit mesh payload without a defined 'method' key.");
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Standardize: Every outgoing message must have a unique identifier (uid)
|
|
87
|
+
if (!payload.uid) {
|
|
88
|
+
payload.uid = crypto.randomUUID();
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const isPrimary = serverCtx?.isPrimary;
|
|
92
|
+
|
|
93
|
+
if (!isPrimary) {
|
|
94
|
+
// Worker Thread: Forward to Master process via native child IPC to relay
|
|
95
|
+
if (typeof process.send === 'function') {
|
|
96
|
+
process.send({
|
|
97
|
+
method: 'mesh.outbound_mesh_relay',
|
|
98
|
+
payload: payload
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
} else {
|
|
102
|
+
// Master Process: Determine target sockets dynamically based on payload directives
|
|
103
|
+
const targets = [];
|
|
104
|
+
|
|
105
|
+
if (payload.broadcast === true || !payload.servers) {
|
|
106
|
+
// Symmetrical Cluster Broadcast: Gather all active socket tunnels
|
|
107
|
+
targets.push(...activeConnections.values());
|
|
108
|
+
} else {
|
|
109
|
+
// Multicast/Unicast Routing: Handle polymorphic servers filter (single string or array)
|
|
110
|
+
const serversList = Array.isArray(payload.servers)
|
|
111
|
+
? payload.servers
|
|
112
|
+
: [payload.servers];
|
|
113
|
+
|
|
114
|
+
for (const id of serversList) {
|
|
115
|
+
const socket = activeConnections.get(id);
|
|
116
|
+
if (socket) {
|
|
117
|
+
targets.push(socket);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Deduplicate target sockets since activeConnections maps both serverId and IP to the same socket
|
|
123
|
+
const uniqueSockets = Array.from(new Set(targets));
|
|
124
|
+
|
|
125
|
+
// Dispatch safely using standard wrapper loop
|
|
126
|
+
socketSend(uniqueSockets, payload);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const serverMeshInstance = new ServerMesh();
|
|
132
|
+
|
|
133
|
+
// Listener A: Handle peer registration of an organization subscription
|
|
134
|
+
serverMeshInstance.on('mesh.orgAdded', (message, ws) => {
|
|
135
|
+
if (!ws) return;
|
|
136
|
+
const organization_id = message.organization_id;
|
|
137
|
+
const ip = ws.remoteAddress || ws._socket?.remoteAddress;
|
|
138
|
+
const peerId = message.senderId || ip;
|
|
139
|
+
|
|
140
|
+
if (!orgServers.has(organization_id)) {
|
|
141
|
+
orgServers.set(organization_id, new Set());
|
|
142
|
+
}
|
|
143
|
+
orgServers.get(organization_id).add(ws);
|
|
144
|
+
console.log(`[@cocreate/server-mesh] Peer ${peerId} registered subscription for org "${organization_id}"`);
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
// Listener B: Handle peer de-registration of an organization subscription
|
|
148
|
+
serverMeshInstance.on('mesh.orgDeleted', (message, ws) => {
|
|
149
|
+
if (!ws) return;
|
|
150
|
+
const organization_id = message.organization_id;
|
|
151
|
+
const ip = ws.remoteAddress || ws._socket?.remoteAddress;
|
|
152
|
+
const peerId = message.senderId || ip;
|
|
153
|
+
|
|
154
|
+
const sockets = orgServers.get(organization_id);
|
|
155
|
+
if (sockets) {
|
|
156
|
+
sockets.delete(ws);
|
|
157
|
+
if (sockets.size === 0) {
|
|
158
|
+
orgServers.delete(organization_id);
|
|
159
|
+
}
|
|
160
|
+
console.log(`[@cocreate/server-mesh] Peer ${peerId} de-registered subscription from org "${organization_id}"`);
|
|
161
|
+
}
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
// Listener C: Handle incoming real-time network unicast transaction data
|
|
165
|
+
serverMeshInstance.on('mesh.orgData', (message) => {
|
|
166
|
+
const { organization_id, data } = message;
|
|
167
|
+
|
|
168
|
+
// Local Unicast: Relay directly to local child worker threads hosting this tenant
|
|
169
|
+
const activeWorkerIds = orgWorkers.get(organization_id);
|
|
170
|
+
if (activeWorkerIds && activeWorkerIds.size > 0 && serverCtx) {
|
|
171
|
+
try {
|
|
172
|
+
serverCtx.send({
|
|
173
|
+
method: 'mesh.inbound',
|
|
174
|
+
workers: Array.from(activeWorkerIds),
|
|
175
|
+
master: false,
|
|
176
|
+
data: typeof data === 'string' ? data : JSON.stringify(data)
|
|
177
|
+
});
|
|
178
|
+
} catch (err) {
|
|
179
|
+
console.error(`[@cocreate/server-mesh] Failed to relay incoming orgData to workers of org "${organization_id}":`, err.message);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
// Listener D: Handle cluster coordination to cancel DB prune schedules on peer consensus
|
|
185
|
+
serverMeshInstance.on('mesh.peer_pruned', (message) => {
|
|
186
|
+
if (message.deadIp) {
|
|
187
|
+
const activeTimerId = pruneTimers.get(message.deadIp);
|
|
188
|
+
if (activeTimerId) {
|
|
189
|
+
clearTimeout(activeTimerId);
|
|
190
|
+
pruneTimers.delete(message.deadIp);
|
|
191
|
+
console.log(`[@cocreate/server-mesh] Cancelled DB prune for ${message.deadIp} due to peer consensus.`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
// Listener E: Handle fallback of unhandled incoming network packets
|
|
197
|
+
serverMeshInstance.on('mesh.raw_message', (message) => {
|
|
198
|
+
if (serverCtx) {
|
|
199
|
+
try {
|
|
200
|
+
serverCtx.send({
|
|
201
|
+
method: 'mesh.inbound',
|
|
202
|
+
broadcast: true,
|
|
203
|
+
master: false,
|
|
204
|
+
data: JSON.stringify(message)
|
|
205
|
+
});
|
|
206
|
+
} catch (err) {
|
|
207
|
+
console.error("[@cocreate/server-mesh] Failed to relay raw fallback message to workers:", err.message);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Extracts raw dataset arrays from standard database responses.
|
|
214
|
+
*/
|
|
215
|
+
function normalizePeersList(result) {
|
|
216
|
+
if (!result) return [];
|
|
217
|
+
if (Array.isArray(result)) return result;
|
|
218
|
+
if (Array.isArray(result.object)) return result.object;
|
|
219
|
+
if (Array.isArray(result.data)) return result.data;
|
|
220
|
+
return [];
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Generates secure, ephemeral TLS credentials on-the-fly for inter-node communication.
|
|
225
|
+
*/
|
|
226
|
+
function generateTLSCredentials() {
|
|
227
|
+
const tmpDir = os.tmpdir();
|
|
228
|
+
const keyPath = path.join(tmpDir, `mesh-key-${Date.now()}-${crypto.randomBytes(4).toString('hex')}.pem`);
|
|
229
|
+
const certPath = path.join(tmpDir, `mesh-cert-${Date.now()}-${crypto.randomBytes(4).toString('hex')}.pem`);
|
|
230
|
+
|
|
231
|
+
try {
|
|
232
|
+
execSync(`openssl req -x509 -newkey rsa:2048 -keyout "${keyPath}" -out "${certPath}" -days 1 -nodes -subj "/CN=localhost"`, { stdio: 'ignore' });
|
|
233
|
+
const key = fs.readFileSync(keyPath);
|
|
234
|
+
const cert = fs.readFileSync(certPath);
|
|
235
|
+
|
|
236
|
+
// Cleanup generated temporary certificate files
|
|
237
|
+
try { fs.unlinkSync(keyPath); } catch {}
|
|
238
|
+
try { fs.unlinkSync(certPath); } catch {}
|
|
239
|
+
|
|
240
|
+
return { key, cert };
|
|
241
|
+
} catch (err) {
|
|
242
|
+
console.error('[@cocreate/server-mesh] Failed to generate ephemeral TLS credentials:', err.message);
|
|
243
|
+
return null;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Safely removes a closed socket from all organization maps.
|
|
249
|
+
*/
|
|
250
|
+
function removeSocketFromAllOrgServers(ws) {
|
|
251
|
+
for (const [orgId, sockets] of orgServers.entries()) {
|
|
252
|
+
if (sockets.has(ws)) {
|
|
253
|
+
sockets.delete(ws);
|
|
254
|
+
if (sockets.size === 0) {
|
|
255
|
+
orgServers.delete(orgId);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Boots the socket mesh and configures routing tunnels.
|
|
263
|
+
*
|
|
264
|
+
* @param {Object} server - CoCreateServer instance context
|
|
265
|
+
*/
|
|
266
|
+
export async function init(server) {
|
|
267
|
+
serverCtx = server;
|
|
268
|
+
|
|
269
|
+
process.on('SIGINT', handleGracefulShutdown);
|
|
270
|
+
process.on('SIGTERM', handleGracefulShutdown);
|
|
271
|
+
|
|
272
|
+
const isPrimary = serverCtx.isPrimary;
|
|
273
|
+
|
|
274
|
+
if (!isPrimary) {
|
|
275
|
+
// --- WORKER EVENT PIPELINE ---
|
|
276
|
+
// Monitor process IPC from Primary
|
|
277
|
+
process.on('message', (message) => {
|
|
278
|
+
if (message && message.method) {
|
|
279
|
+
serverMeshInstance.emit(message.method, message);
|
|
280
|
+
}
|
|
281
|
+
});
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
// --- MASTER TUNNEL & UNICAST ROUTING PIPELINE ---
|
|
286
|
+
// Handle incoming IPC registrations from child workers
|
|
287
|
+
cluster.on('message', (workerRef, message) => {
|
|
288
|
+
if (message && message.method) {
|
|
289
|
+
if (message.method === 'mesh.registerOrg') {
|
|
290
|
+
const { organization_id, workerId } = message;
|
|
291
|
+
let isNewOrg = false;
|
|
292
|
+
|
|
293
|
+
if (!orgWorkers.has(organization_id)) {
|
|
294
|
+
orgWorkers.set(organization_id, new Set());
|
|
295
|
+
isNewOrg = true;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
orgWorkers.get(organization_id).add(workerId);
|
|
299
|
+
console.log(`[@cocreate/server-mesh] [LOCAL-MAP] Tenant "${organization_id}" registered on Worker ${workerId}`);
|
|
300
|
+
|
|
301
|
+
if (isNewOrg) {
|
|
302
|
+
serverMeshInstance.send({
|
|
303
|
+
method: 'mesh.orgAdded',
|
|
304
|
+
organization_id,
|
|
305
|
+
senderId: serverCtx.id
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
else if (message.method === 'mesh.deregisterOrg') {
|
|
310
|
+
const { organization_id, workerId } = message;
|
|
311
|
+
const workersSet = orgWorkers.get(organization_id);
|
|
312
|
+
|
|
313
|
+
if (workersSet) {
|
|
314
|
+
workersSet.delete(workerId);
|
|
315
|
+
if (workersSet.size === 0) {
|
|
316
|
+
orgWorkers.delete(organization_id);
|
|
317
|
+
console.log(`[@cocreate/server-mesh] [LOCAL-MAP] Tenant "${organization_id}" evicted. No remaining local workers.`);
|
|
318
|
+
serverMeshInstance.send({
|
|
319
|
+
method: 'mesh.orgDeleted',
|
|
320
|
+
organization_id,
|
|
321
|
+
senderId: serverCtx.id
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
else if (message.method === 'mesh.orgData') {
|
|
327
|
+
const { organization_id, data, senderWorkerId } = message;
|
|
328
|
+
|
|
329
|
+
// Local Unicast: Relay directly to child worker cores hosting the tenant (ignoring sender)
|
|
330
|
+
const activeWorkerIds = orgWorkers.get(organization_id);
|
|
331
|
+
if (activeWorkerIds && activeWorkerIds.size > 0) {
|
|
332
|
+
const siblingWorkerIds = Array.from(activeWorkerIds).filter(wId => wId !== senderWorkerId);
|
|
333
|
+
if (siblingWorkerIds.length > 0) {
|
|
334
|
+
try {
|
|
335
|
+
serverCtx.send({
|
|
336
|
+
method: 'mesh.inbound',
|
|
337
|
+
workers: siblingWorkerIds,
|
|
338
|
+
master: false,
|
|
339
|
+
data: typeof data === 'string' ? data : JSON.stringify(data)
|
|
340
|
+
});
|
|
341
|
+
} catch (err) {
|
|
342
|
+
console.error("[@cocreate/server-mesh] Failed to propagate orgData to local siblings:", err.message);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// Network Unicast: Relay over secure tunnels to peer servers hosting the tenant
|
|
348
|
+
const targetSockets = orgServers.get(organization_id);
|
|
349
|
+
if (targetSockets && targetSockets.size > 0) {
|
|
350
|
+
const envelope = {
|
|
351
|
+
method: 'mesh.orgData',
|
|
352
|
+
organization_id,
|
|
353
|
+
data
|
|
354
|
+
};
|
|
355
|
+
socketSend(Array.from(targetSockets), envelope);
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
else if (message.method === 'mesh.outbound_mesh_relay') {
|
|
359
|
+
// Relay child worker message out to peer nodes
|
|
360
|
+
if (message.payload) {
|
|
361
|
+
serverMeshInstance.send(message.payload);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
// Mirror all incoming process IPC commands to local listeners
|
|
366
|
+
serverMeshInstance.emit(message.method, message, workerRef);
|
|
367
|
+
}
|
|
368
|
+
});
|
|
369
|
+
|
|
370
|
+
// Boot HTTPS Secure Server
|
|
371
|
+
const credentials = generateTLSCredentials();
|
|
372
|
+
if (credentials) {
|
|
373
|
+
secureMeshServer = https.createServer(credentials);
|
|
374
|
+
secureMeshServer.on('error', (err) => {
|
|
375
|
+
if (err.code === 'EADDRINUSE') {
|
|
376
|
+
console.error(`\n[@cocreate/server-mesh] [FATAL] Port ${MESH_PORT} is already in use!`);
|
|
377
|
+
process.exit(1);
|
|
378
|
+
} else {
|
|
379
|
+
console.error(`[@cocreate/server-mesh] Mesh server error:`, err);
|
|
380
|
+
}
|
|
381
|
+
});
|
|
382
|
+
|
|
383
|
+
secureMeshServer.listen(MESH_PORT, '0.0.0.0', () => {
|
|
384
|
+
console.log(`[@cocreate/server-mesh] Mesh securely listening on 0.0.0.0:${MESH_PORT}`);
|
|
385
|
+
});
|
|
386
|
+
} else {
|
|
387
|
+
throw new Error("Unable to boot secure mesh: TLS credential generation failed.");
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
meshWss = new WebSocketServer({ server: secureMeshServer });
|
|
391
|
+
|
|
392
|
+
meshWss.on('connection', (ws, req) => {
|
|
393
|
+
const ip = req.socket.remoteAddress;
|
|
394
|
+
let remoteServerId = null;
|
|
395
|
+
|
|
396
|
+
const protocolHeader = req.headers['sec-websocket-protocol'];
|
|
397
|
+
if (protocolHeader) {
|
|
398
|
+
try {
|
|
399
|
+
const parsed = JSON.parse(decodeURIComponent(protocolHeader));
|
|
400
|
+
remoteServerId = parsed.id;
|
|
401
|
+
} catch (err) {
|
|
402
|
+
// Ignore decoding anomalies gracefully
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
const identifier = remoteServerId || ip;
|
|
407
|
+
console.log(`[@cocreate/server-mesh] Secure peer connection established: ${identifier}`);
|
|
408
|
+
|
|
409
|
+
ws.isAlive = true;
|
|
410
|
+
ws.on('pong', () => { ws.isAlive = true; });
|
|
411
|
+
|
|
412
|
+
// Inlined Blind Event-Dispatch Loop: If no method, do nothing.
|
|
413
|
+
ws.on('message', (rawData) => {
|
|
414
|
+
try {
|
|
415
|
+
const message = JSON.parse(rawData.toString());
|
|
416
|
+
if (message && message.method) {
|
|
417
|
+
serverMeshInstance.emit(message.method, message, ws);
|
|
418
|
+
}
|
|
419
|
+
} catch (err) {
|
|
420
|
+
// Safely catch parsing failures
|
|
421
|
+
}
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
ws.on('close', () => {
|
|
425
|
+
console.log(`[@cocreate/server-mesh] Peer connection ${identifier} dropped.`);
|
|
426
|
+
activeConnections.delete(identifier);
|
|
427
|
+
if (remoteServerId) activeConnections.delete(ip);
|
|
428
|
+
removeSocketFromAllOrgServers(ws);
|
|
429
|
+
});
|
|
430
|
+
|
|
431
|
+
ws.on('error', (err) => {
|
|
432
|
+
console.error(`[@cocreate/server-mesh] Connection error (${identifier}):`, err.message);
|
|
433
|
+
});
|
|
434
|
+
|
|
435
|
+
activeConnections.set(identifier, ws);
|
|
436
|
+
if (remoteServerId) activeConnections.set(ip, ws);
|
|
437
|
+
});
|
|
438
|
+
|
|
439
|
+
try {
|
|
440
|
+
await getServers();
|
|
441
|
+
} catch (err) {
|
|
442
|
+
console.error("[@cocreate/server-mesh] Peer discovery reconciliation cycle failed on start:", err.message);
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
wsPingInterval = setInterval(pingActiveWebsockets, 30000);
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
/**
|
|
450
|
+
* Registers this server instance once on startup to allow discovery.
|
|
451
|
+
*/
|
|
452
|
+
async function registerServer() {
|
|
453
|
+
if (isGracefulExitRunning || !serverCtx?.crud) return;
|
|
454
|
+
|
|
455
|
+
const registerPayload = {
|
|
456
|
+
method: 'object.update',
|
|
457
|
+
array: 'servers',
|
|
458
|
+
upsert: true,
|
|
459
|
+
object: {
|
|
460
|
+
_id: serverCtx.id,
|
|
461
|
+
ip: serverCtx.ip,
|
|
462
|
+
host: serverCtx.host,
|
|
463
|
+
status: 'active',
|
|
464
|
+
infrastructure: serverCtx.infrastructure
|
|
465
|
+
},
|
|
466
|
+
organization_id: serverCtx.organization_id,
|
|
467
|
+
host: serverCtx.host
|
|
468
|
+
};
|
|
469
|
+
|
|
470
|
+
try {
|
|
471
|
+
const registerResult = await serverCtx.crud.send(registerPayload);
|
|
472
|
+
if (registerResult && registerResult.error) {
|
|
473
|
+
console.error(`[@cocreate/server-mesh] Startup registration rejected: ${JSON.stringify(registerResult.error)}`);
|
|
474
|
+
} else {
|
|
475
|
+
console.log(`[@cocreate/server-mesh] Successfully registered server profile in db.`);
|
|
476
|
+
}
|
|
477
|
+
} catch (err) {
|
|
478
|
+
console.error("[@cocreate/server-mesh] Startup registration network error:", err.message);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* Monitors responsiveness of active WebSocket connections.
|
|
484
|
+
*/
|
|
485
|
+
function pingActiveWebsockets() {
|
|
486
|
+
if (isGracefulExitRunning || !meshWss) return;
|
|
487
|
+
|
|
488
|
+
meshWss.clients.forEach((ws) => {
|
|
489
|
+
if (ws.isAlive === false) {
|
|
490
|
+
console.warn(`[@cocreate/server-mesh] Client socket unresponsive. Terminating connection.`);
|
|
491
|
+
return ws.terminate();
|
|
492
|
+
}
|
|
493
|
+
ws.isAlive = false;
|
|
494
|
+
try {
|
|
495
|
+
ws.ping();
|
|
496
|
+
} catch (err) {
|
|
497
|
+
console.error("[@cocreate/server-mesh] Error during websocket ping:", err.message);
|
|
498
|
+
}
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
/**
|
|
503
|
+
* Cleanly handles node evictions and releases network bindings.
|
|
504
|
+
*/
|
|
505
|
+
async function handleGracefulShutdown() {
|
|
506
|
+
if (isGracefulExitRunning) return;
|
|
507
|
+
isGracefulExitRunning = true;
|
|
508
|
+
|
|
509
|
+
console.log(`\n[@cocreate/server-mesh] Graceful shutdown initiated. Cleaning up mesh state...`);
|
|
510
|
+
|
|
511
|
+
clearInterval(wsPingInterval);
|
|
512
|
+
|
|
513
|
+
if (serverCtx?.crud && serverCtx.id && serverCtx.organization_id) {
|
|
514
|
+
try {
|
|
515
|
+
await serverCtx.crud.send({
|
|
516
|
+
method: 'object.delete',
|
|
517
|
+
array: 'servers',
|
|
518
|
+
object: { _id: serverCtx.id },
|
|
519
|
+
organization_id: serverCtx.organization_id,
|
|
520
|
+
host: serverCtx.host
|
|
521
|
+
});
|
|
522
|
+
console.log(`[@cocreate/server-mesh] Server record wiped from database.`);
|
|
523
|
+
} catch (err) {
|
|
524
|
+
console.error("[@cocreate/server-mesh] Failed to wipe server status cleanly from DB:", err.message);
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
for (const timerId of retryTimers.values()) clearTimeout(timerId);
|
|
529
|
+
for (const timerId of pruneTimers.values()) clearTimeout(timerId);
|
|
530
|
+
retryTimers.clear();
|
|
531
|
+
pruneTimers.clear();
|
|
532
|
+
|
|
533
|
+
const uniqueSockets = new Set(activeConnections.values());
|
|
534
|
+
for (const ws of uniqueSockets) {
|
|
535
|
+
if (ws && ws.readyState === WebSocket.OPEN) {
|
|
536
|
+
try { ws.close(); } catch {}
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
activeConnections.clear();
|
|
540
|
+
|
|
541
|
+
if (meshWss) {
|
|
542
|
+
try { meshWss.close(); } catch {}
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
if (secureMeshServer) {
|
|
546
|
+
secureMeshServer.close(() => {
|
|
547
|
+
console.log(`[@cocreate/server-mesh] Port ${MESH_PORT} successfully released.`);
|
|
548
|
+
process.exit(0);
|
|
549
|
+
});
|
|
550
|
+
} else {
|
|
551
|
+
process.exit(0);
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
setTimeout(() => {
|
|
555
|
+
console.warn('[@cocreate/server-mesh] Force-exiting gracefully hung process.');
|
|
556
|
+
process.exit(1);
|
|
557
|
+
}, 3000).unref();
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/**
|
|
561
|
+
* Scans for peer nodes, establishes direct connections, and schedules lazy prunes.
|
|
562
|
+
*/
|
|
563
|
+
async function getServers() {
|
|
564
|
+
if (isGracefulExitRunning || !serverCtx?.crud) return;
|
|
565
|
+
|
|
566
|
+
const instanceId = serverCtx.infrastructure?.instanceId;
|
|
567
|
+
if (instanceId) {
|
|
568
|
+
try {
|
|
569
|
+
await serverCtx.crud.send({
|
|
570
|
+
method: 'object.delete',
|
|
571
|
+
array: 'servers',
|
|
572
|
+
$filter: {
|
|
573
|
+
query: { 'infrastructure.instanceId': instanceId },
|
|
574
|
+
limit: 0
|
|
575
|
+
},
|
|
576
|
+
organization_id: serverCtx.organization_id,
|
|
577
|
+
host: serverCtx.host
|
|
578
|
+
});
|
|
579
|
+
} catch (err) {
|
|
580
|
+
console.warn("[@cocreate/server-mesh] Startup instance-eviction run failed/skipped:", err.message);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
// Register profile once on boot
|
|
585
|
+
await registerServer();
|
|
586
|
+
|
|
587
|
+
const readPayload = {
|
|
588
|
+
method: 'object.read',
|
|
589
|
+
array: 'servers',
|
|
590
|
+
$filter: {
|
|
591
|
+
query: { _id: { $lt: serverCtx.id } },
|
|
592
|
+
sort: { _id: 1 },
|
|
593
|
+
limit: 0
|
|
594
|
+
},
|
|
595
|
+
organization_id: serverCtx.organization_id,
|
|
596
|
+
host: serverCtx.host
|
|
597
|
+
};
|
|
598
|
+
|
|
599
|
+
let readResult;
|
|
600
|
+
try {
|
|
601
|
+
readResult = await serverCtx.crud.send(readPayload);
|
|
602
|
+
} catch (err) {
|
|
603
|
+
console.error("[@cocreate/server-mesh] Failed to fetch active peer server records:", err.message);
|
|
604
|
+
return;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
const peers = normalizePeersList(readResult);
|
|
608
|
+
|
|
609
|
+
for (const targetPeer of peers) {
|
|
610
|
+
const targetIp = targetPeer.ip;
|
|
611
|
+
|
|
612
|
+
if (targetIp === serverCtx.ip) continue;
|
|
613
|
+
if (activeConnections.has(targetIp)) continue;
|
|
614
|
+
|
|
615
|
+
console.log(`[@cocreate/server-mesh] Initiating secure connection stream to target: ${targetIp}`);
|
|
616
|
+
|
|
617
|
+
let retryCount = 0;
|
|
618
|
+
const maxRetries = 10;
|
|
619
|
+
|
|
620
|
+
const connectToPeer = () => {
|
|
621
|
+
if (isGracefulExitRunning) return;
|
|
622
|
+
|
|
623
|
+
const targetUrl = `wss://${targetIp}:${MESH_PORT}/${serverCtx.organization_id}`;
|
|
624
|
+
const clientSocket = new WebSocket(targetUrl, [
|
|
625
|
+
encodeURIComponent(JSON.stringify({
|
|
626
|
+
type: 'peer',
|
|
627
|
+
ip: serverCtx.ip,
|
|
628
|
+
id: serverCtx.id,
|
|
629
|
+
token: process.env.CLUSTER_SECRET,
|
|
630
|
+
sniHost: serverCtx.host
|
|
631
|
+
}))
|
|
632
|
+
], {
|
|
633
|
+
servername: serverCtx.host,
|
|
634
|
+
rejectUnauthorized: false
|
|
635
|
+
});
|
|
636
|
+
|
|
637
|
+
clientSocket.isAlive = true;
|
|
638
|
+
clientSocket.on('pong', () => { clientSocket.isAlive = true; });
|
|
639
|
+
clientSocket.on('ping', () => { clientSocket.pong(); });
|
|
640
|
+
|
|
641
|
+
clientSocket.on('open', () => {
|
|
642
|
+
console.log(`[@cocreate/server-mesh] Secure link complete to peer: ${targetIp}`);
|
|
643
|
+
retryCount = 0;
|
|
644
|
+
|
|
645
|
+
if (retryTimers.has(targetIp)) clearTimeout(retryTimers.get(targetIp));
|
|
646
|
+
if (pruneTimers.has(targetIp)) clearTimeout(pruneTimers.get(targetIp));
|
|
647
|
+
|
|
648
|
+
retryTimers.delete(targetIp);
|
|
649
|
+
pruneTimers.delete(targetIp);
|
|
650
|
+
|
|
651
|
+
clientSocket.isPeer = true;
|
|
652
|
+
clientSocket.isPlatformTunnel = true;
|
|
653
|
+
|
|
654
|
+
activeConnections.set(targetPeer._id, clientSocket);
|
|
655
|
+
activeConnections.set(targetIp, clientSocket);
|
|
656
|
+
});
|
|
657
|
+
|
|
658
|
+
clientSocket.on('close', () => {
|
|
659
|
+
activeConnections.delete(targetPeer._id);
|
|
660
|
+
activeConnections.delete(targetIp);
|
|
661
|
+
retryTimers.delete(targetIp);
|
|
662
|
+
removeSocketFromAllOrgServers(clientSocket);
|
|
663
|
+
|
|
664
|
+
if (isGracefulExitRunning) return;
|
|
665
|
+
|
|
666
|
+
if (retryCount < maxRetries) {
|
|
667
|
+
retryCount++;
|
|
668
|
+
const delay = 5000 + Math.floor(Math.random() * 2000);
|
|
669
|
+
console.warn(`[@cocreate/server-mesh] Peer disconnected. Retrying in ${delay}ms...`);
|
|
670
|
+
const timerId = setTimeout(connectToPeer, delay);
|
|
671
|
+
retryTimers.set(targetIp, timerId);
|
|
672
|
+
} else {
|
|
673
|
+
const staggerDelay = peers.length * 1000;
|
|
674
|
+
console.warn(`[@cocreate/server-mesh] Peer dead. Scheduling db prune in ${staggerDelay}ms...`);
|
|
675
|
+
|
|
676
|
+
const pruneTimerId = setTimeout(async () => {
|
|
677
|
+
console.error(`[@cocreate/server-mesh] Pruning unresponsive node: ${targetIp}`);
|
|
678
|
+
|
|
679
|
+
try {
|
|
680
|
+
await serverCtx.crud.send({
|
|
681
|
+
method: 'object.delete',
|
|
682
|
+
array: 'servers',
|
|
683
|
+
object: { _id: targetPeer._id },
|
|
684
|
+
organization_id: serverCtx.organization_id,
|
|
685
|
+
host: serverCtx.host
|
|
686
|
+
});
|
|
687
|
+
} catch (err) {
|
|
688
|
+
console.error(`[@cocreate/server-mesh] Failed to prune node ${targetIp} from database:`, err.message);
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
// Symmetrical Consensus Broadcast: Emits with standard uuid tracking
|
|
692
|
+
serverMeshInstance.send({
|
|
693
|
+
method: 'mesh.peer_pruned',
|
|
694
|
+
deadIp: targetIp,
|
|
695
|
+
isForwarded: true,
|
|
696
|
+
senderId: serverCtx.id
|
|
697
|
+
});
|
|
698
|
+
|
|
699
|
+
pruneTimers.delete(targetIp);
|
|
700
|
+
}, staggerDelay);
|
|
701
|
+
|
|
702
|
+
pruneTimers.set(targetIp, pruneTimerId);
|
|
703
|
+
}
|
|
704
|
+
});
|
|
705
|
+
|
|
706
|
+
// Inlined Blind Event-Dispatch Loop
|
|
707
|
+
clientSocket.on('message', (rawData) => {
|
|
708
|
+
try {
|
|
709
|
+
const message = JSON.parse(rawData.toString());
|
|
710
|
+
if (message && message.method) {
|
|
711
|
+
serverMeshInstance.emit(message.method, message, clientSocket);
|
|
712
|
+
}
|
|
713
|
+
} catch (err) {
|
|
714
|
+
// Safely catch parsing failures
|
|
715
|
+
}
|
|
716
|
+
});
|
|
717
|
+
|
|
718
|
+
clientSocket.on('error', (err) => {
|
|
719
|
+
console.log(`[@cocreate/server-mesh] Socket transport failure: ${err.message}`);
|
|
720
|
+
});
|
|
721
|
+
};
|
|
722
|
+
|
|
723
|
+
connectToPeer();
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
// Attach init directly to the instance to allow standard default-import invocation
|
|
728
|
+
serverMeshInstance.init = init;
|
|
729
|
+
|
|
730
|
+
// Default export matches standard functional entry contract
|
|
731
|
+
export default serverMeshInstance;
|