@morojs/moro 1.8.9 → 1.8.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/core/cluster/thread-cluster.d.ts +178 -0
- package/dist/core/cluster/thread-cluster.js +389 -0
- package/dist/core/cluster/thread-cluster.js.map +1 -0
- package/dist/core/framework.js.map +1 -1
- package/dist/core/http/moro-engine-server.d.ts +48 -1
- package/dist/core/http/moro-engine-server.js +386 -37
- package/dist/core/http/moro-engine-server.js.map +1 -1
- package/dist/core/http/utils/compression.js +12 -6
- package/dist/core/http/utils/compression.js.map +1 -1
- package/dist/core/http/utils/uws-worker-clustering.js +10 -6
- package/dist/core/http/utils/uws-worker-clustering.js.map +1 -1
- package/dist/core/http/uws-http-server.js +3 -2
- package/dist/core/http/uws-http-server.js.map +1 -1
- package/dist/core/jobs/leader-election.js +16 -5
- package/dist/core/jobs/leader-election.js.map +1 -1
- package/dist/core/networking/websocket-manager.js +4 -2
- package/dist/core/networking/websocket-manager.js.map +1 -1
- package/dist/core/routing/index.d.ts +9 -2
- package/dist/core/routing/index.js +12 -7
- package/dist/core/routing/index.js.map +1 -1
- package/dist/core/routing/unified-router.d.ts +25 -1
- package/dist/core/routing/unified-router.js +69 -6
- package/dist/core/routing/unified-router.js.map +1 -1
- package/dist/core/runtime/node-adapter.d.ts +1 -1
- package/dist/core/runtime/node-adapter.js.map +1 -1
- package/dist/core/utilities/builtin.d.ts +1 -0
- package/dist/core/utilities/builtin.js +25 -0
- package/dist/core/utilities/builtin.js.map +1 -0
- package/dist/core/utilities/package-utils.d.ts +29 -3
- package/dist/core/utilities/package-utils.js +24 -12
- package/dist/core/utilities/package-utils.js.map +1 -1
- package/dist/core/workers/worker-manager.js +4 -3
- package/dist/core/workers/worker-manager.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js.map +1 -1
- package/dist/moro.d.ts +5 -0
- package/dist/moro.js +401 -211
- package/dist/moro.js.map +1 -1
- package/dist/types/http.d.ts +1 -1
- package/dist/types/http.js.map +1 -1
- package/package.json +1 -1
package/dist/moro.js
CHANGED
|
@@ -9,11 +9,16 @@ import { UnifiedRouter, } from './core/routing/unified-router.js';
|
|
|
9
9
|
import { PathMatcher } from './core/routing/path-matcher.js';
|
|
10
10
|
import { AppDocumentationManager } from './core/docs/index.js';
|
|
11
11
|
import { EventEmitter } from 'events';
|
|
12
|
-
import
|
|
13
|
-
|
|
12
|
+
import { requireBuiltin } from './core/utilities/builtin.js';
|
|
13
|
+
// node:cluster (it drags in child_process, dgram and net) is loaded only when
|
|
14
|
+
// process clustering actually runs; worker_threads only on the clustering
|
|
15
|
+
// path. See core/utilities/builtin.ts.
|
|
16
|
+
const nodeCluster = () => requireBuiltin('cluster');
|
|
17
|
+
const wt = () => requireBuiltin('worker_threads');
|
|
18
|
+
import { ThreadClusterPrimary, ThreadClusterWorker, computeWorkerCount, isUserTunedYoungGen, resolveClusterTransport, threadWorkerInfo, YOUNG_GEN_MB_FOR_SEMI_SPACE_4, } from './core/cluster/thread-cluster.js';
|
|
14
19
|
import { normalizeValidationError } from './core/validation/schema-interface.js';
|
|
15
20
|
import { buildModuleBasePath } from './core/utilities/module-path.js';
|
|
16
|
-
import { filePathToImportURL } from './core/utilities/package-utils.js';
|
|
21
|
+
import { filePathToImportURL, loadNativeEngine } from './core/utilities/package-utils.js';
|
|
17
22
|
// Configuration System Integration
|
|
18
23
|
import { initializeConfig, initializeConfigAsync } from './core/config/index.js';
|
|
19
24
|
// Runtime System Integration
|
|
@@ -339,7 +344,7 @@ export class Moro extends EventEmitter {
|
|
|
339
344
|
};
|
|
340
345
|
}
|
|
341
346
|
get(path, handler, options) {
|
|
342
|
-
if (handler) {
|
|
347
|
+
if (handler !== undefined) {
|
|
343
348
|
// Direct route registration
|
|
344
349
|
return this.addRoute('GET', path, handler, options);
|
|
345
350
|
}
|
|
@@ -347,7 +352,7 @@ export class Moro extends EventEmitter {
|
|
|
347
352
|
return this.unifiedRouter.get(path);
|
|
348
353
|
}
|
|
349
354
|
post(path, handler, options) {
|
|
350
|
-
if (handler) {
|
|
355
|
+
if (handler !== undefined) {
|
|
351
356
|
// Direct route registration
|
|
352
357
|
return this.addRoute('POST', path, handler, options);
|
|
353
358
|
}
|
|
@@ -355,7 +360,7 @@ export class Moro extends EventEmitter {
|
|
|
355
360
|
return this.unifiedRouter.post(path);
|
|
356
361
|
}
|
|
357
362
|
put(path, handler, options) {
|
|
358
|
-
if (handler) {
|
|
363
|
+
if (handler !== undefined) {
|
|
359
364
|
// Direct route registration
|
|
360
365
|
return this.addRoute('PUT', path, handler, options);
|
|
361
366
|
}
|
|
@@ -363,7 +368,7 @@ export class Moro extends EventEmitter {
|
|
|
363
368
|
return this.unifiedRouter.put(path);
|
|
364
369
|
}
|
|
365
370
|
delete(path, handler, options) {
|
|
366
|
-
if (handler) {
|
|
371
|
+
if (handler !== undefined) {
|
|
367
372
|
// Direct route registration
|
|
368
373
|
return this.addRoute('DELETE', path, handler, options);
|
|
369
374
|
}
|
|
@@ -371,7 +376,7 @@ export class Moro extends EventEmitter {
|
|
|
371
376
|
return this.unifiedRouter.delete(path);
|
|
372
377
|
}
|
|
373
378
|
patch(path, handler, options) {
|
|
374
|
-
if (handler) {
|
|
379
|
+
if (handler !== undefined) {
|
|
375
380
|
// Direct route registration
|
|
376
381
|
return this.addRoute('PATCH', path, handler, options);
|
|
377
382
|
}
|
|
@@ -379,13 +384,13 @@ export class Moro extends EventEmitter {
|
|
|
379
384
|
return this.unifiedRouter.patch(path);
|
|
380
385
|
}
|
|
381
386
|
head(path, handler, options) {
|
|
382
|
-
if (handler) {
|
|
387
|
+
if (handler !== undefined) {
|
|
383
388
|
return this.addRoute('HEAD', path, handler, options);
|
|
384
389
|
}
|
|
385
390
|
return this.unifiedRouter.head(path);
|
|
386
391
|
}
|
|
387
392
|
options(path, handler, options) {
|
|
388
|
-
if (handler) {
|
|
393
|
+
if (handler !== undefined) {
|
|
389
394
|
return this.addRoute('OPTIONS', path, handler, options);
|
|
390
395
|
}
|
|
391
396
|
return this.unifiedRouter.options(path);
|
|
@@ -778,7 +783,8 @@ export class Moro extends EventEmitter {
|
|
|
778
783
|
}
|
|
779
784
|
// Check if clustering is enabled for massive performance gains
|
|
780
785
|
if (this.config.performance?.clustering?.enabled) {
|
|
781
|
-
|
|
786
|
+
// Worker threads (engine backend, POSIX, engine >= 1.1.6) or node:cluster
|
|
787
|
+
// processes - startWithClustering picks and logs the transport.
|
|
782
788
|
this.startWithClustering(port, host, callback);
|
|
783
789
|
return;
|
|
784
790
|
}
|
|
@@ -1231,6 +1237,23 @@ export class Moro extends EventEmitter {
|
|
|
1231
1237
|
}
|
|
1232
1238
|
// Private methods
|
|
1233
1239
|
addRoute(method, path, handler, options = {}) {
|
|
1240
|
+
// A literal body - app.get('/', '') - is the same route the builder's
|
|
1241
|
+
// .handler('') registers: the unified router turns it into a handler and,
|
|
1242
|
+
// with nothing else configured, a reply the native engine answers itself.
|
|
1243
|
+
// It never enters the legacy direct-route table, whose wrapper would
|
|
1244
|
+
// call it as a function.
|
|
1245
|
+
if (typeof handler !== 'function') {
|
|
1246
|
+
this.unifiedRouter.registerRoute({
|
|
1247
|
+
method: method,
|
|
1248
|
+
path,
|
|
1249
|
+
handler: handler,
|
|
1250
|
+
validation: options.validation,
|
|
1251
|
+
rateLimit: options.rateLimit,
|
|
1252
|
+
cache: options.cache,
|
|
1253
|
+
middleware: options.middleware,
|
|
1254
|
+
});
|
|
1255
|
+
return this;
|
|
1256
|
+
}
|
|
1234
1257
|
// Register with unified router (primary routing system)
|
|
1235
1258
|
this.unifiedRouter.addRoute(method, path, handler, options.middleware || []);
|
|
1236
1259
|
// Also store in legacy routes array for backward compatibility
|
|
@@ -1276,6 +1299,19 @@ export class Moro extends EventEmitter {
|
|
|
1276
1299
|
if (httpServer && typeof httpServer.setRouterHandler === 'function') {
|
|
1277
1300
|
const router = this.unifiedRouter;
|
|
1278
1301
|
httpServer.setRouterHandler((req, res) => router.handleRequest(req, res));
|
|
1302
|
+
// A server that answers fixed replies itself (MoroEngineServer on an
|
|
1303
|
+
// engine with capabilities.staticRoutes) gets every route whose handler
|
|
1304
|
+
// is a literal body, so those requests never enter JS. The route's
|
|
1305
|
+
// handler stays registered: it answers HEAD, and everything on servers
|
|
1306
|
+
// without the capability.
|
|
1307
|
+
if (typeof httpServer.setStaticRoute === 'function') {
|
|
1308
|
+
for (const schema of router.getAllRoutes()) {
|
|
1309
|
+
const fixed = schema.static;
|
|
1310
|
+
if (fixed) {
|
|
1311
|
+
httpServer.setStaticRoute(schema.method, schema.path, fixed.status, fixed.headers, fixed.body);
|
|
1312
|
+
}
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1279
1315
|
// Servers with a native router (uWS) additionally get fast-path routes
|
|
1280
1316
|
// registered directly on it - evaluated at listen so late-loaded routes
|
|
1281
1317
|
// are included; error handling shares the unified router's semantics
|
|
@@ -1652,150 +1688,275 @@ export class Moro extends EventEmitter {
|
|
|
1652
1688
|
* - BetterStack Node.js Clustering: https://betterstack.com/community/guides/scaling-nodejs/node-clustering/
|
|
1653
1689
|
*/
|
|
1654
1690
|
clusterWorkers = new Map();
|
|
1691
|
+
threadCluster = undefined;
|
|
1692
|
+
// Entry point for performance.clustering: decides whether THIS instance is
|
|
1693
|
+
// a worker (thread or process) or the primary, and which transport the
|
|
1694
|
+
// primary uses. Config keys are unchanged; the transport is automatic.
|
|
1655
1695
|
startWithClustering(port, host, callback) {
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
// x10 microbenchmark (~560k both ways), while sustained-load RSS drops
|
|
1696
|
-
// ~30 MB per worker (115 -> 81 MB on the 25-route POST soak).
|
|
1697
|
-
const userTunedGC = process.execArgv.some(a => a.includes('--max-semi-space-size')) ||
|
|
1698
|
-
(process.env.NODE_OPTIONS || '').includes('--max-semi-space-size');
|
|
1699
|
-
// Set cluster settings for better performance
|
|
1700
|
-
cluster.setupMaster({
|
|
1701
|
-
exec: process.argv[1] || process.execPath,
|
|
1702
|
-
args: process.argv.slice(2),
|
|
1703
|
-
silent: false,
|
|
1704
|
-
...(userTunedGC ? {} : { execArgv: [...process.execArgv, '--max-semi-space-size=4'] }),
|
|
1696
|
+
const { count: workerCount, detail } = computeWorkerCount(this.config.performance?.clustering);
|
|
1697
|
+
const tinfo = threadWorkerInfo();
|
|
1698
|
+
if (tinfo) {
|
|
1699
|
+
this.startClusterWorker({ kind: 'thread', info: tinfo }, port, host);
|
|
1700
|
+
return;
|
|
1701
|
+
}
|
|
1702
|
+
if (nodeCluster().isWorker) {
|
|
1703
|
+
this.startClusterWorker({ kind: 'process' }, port, host);
|
|
1704
|
+
return;
|
|
1705
|
+
}
|
|
1706
|
+
this.logger.info(detail, 'Cluster');
|
|
1707
|
+
const engineKind = this.engine;
|
|
1708
|
+
const transport = resolveClusterTransport({
|
|
1709
|
+
platform: process.platform,
|
|
1710
|
+
isMainThread: wt().isMainThread,
|
|
1711
|
+
argv1: process.argv[1],
|
|
1712
|
+
engineServer: engineKind.server,
|
|
1713
|
+
enginePackage: engineKind.enginePackage,
|
|
1714
|
+
workerThreadsCapable: loadNativeEngine()?.capabilities?.workerThreads === true,
|
|
1715
|
+
});
|
|
1716
|
+
if (transport.kind === 'threads') {
|
|
1717
|
+
this.startThreadPrimary(transport.entry, workerCount, port, host, callback);
|
|
1718
|
+
return;
|
|
1719
|
+
}
|
|
1720
|
+
this.logger.info(`Clustering: ${workerCount} worker processes (node:cluster; ${transport.reason})`, 'Cluster');
|
|
1721
|
+
this.startProcessPrimary(workerCount, port, host, callback);
|
|
1722
|
+
}
|
|
1723
|
+
// Services that must run exactly ONCE per cluster, in the primary: the job
|
|
1724
|
+
// scheduler (leader election already treats the main thread of a
|
|
1725
|
+
// non-cluster-worker process as a participant) and the gRPC server (its own
|
|
1726
|
+
// port). GraphQL registers HTTP/WS routes, so it initialises in every
|
|
1727
|
+
// worker instead. Before 1.9.0 none of these started under clustering.
|
|
1728
|
+
startPrimaryServices() {
|
|
1729
|
+
this.startJobScheduler().catch(err => {
|
|
1730
|
+
this.logger.error(`Failed to start job scheduler: ${String(err)}`);
|
|
1731
|
+
});
|
|
1732
|
+
if (this.grpcManager && !this.grpcStarted) {
|
|
1733
|
+
this.startGrpc().catch(error => {
|
|
1734
|
+
this.logger.error(`Failed to start gRPC server: ${error}`, 'GRPC');
|
|
1705
1735
|
});
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1736
|
+
}
|
|
1737
|
+
}
|
|
1738
|
+
startThreadPrimary(entry, workerCount, port, host, callback) {
|
|
1739
|
+
// The libuv threadpool is process-wide and sized once at first use;
|
|
1740
|
+
// the process path set this per worker process. Only if the operator
|
|
1741
|
+
// did not choose a size.
|
|
1742
|
+
if (!process.env.UV_THREADPOOL_SIZE)
|
|
1743
|
+
process.env.UV_THREADPOOL_SIZE = '64';
|
|
1744
|
+
// Same young-generation bound the process path applies through execArgv
|
|
1745
|
+
// (see YOUNG_GEN_MB_FOR_SEMI_SPACE_4), unless the operator tuned GC.
|
|
1746
|
+
const resourceLimits = isUserTunedYoungGen()
|
|
1747
|
+
? undefined
|
|
1748
|
+
: { maxYoungGenerationSizeMb: YOUNG_GEN_MB_FOR_SEMI_SPACE_4 };
|
|
1749
|
+
const engineKind = this.engine;
|
|
1750
|
+
this.logger.info(`Clustering: ${workerCount} worker threads (worker_threads + SO_REUSEPORT, ` +
|
|
1751
|
+
`${engineKind.enginePackage}${engineKind.engineVersion ? ` v${engineKind.engineVersion}` : ''})`, 'Cluster');
|
|
1752
|
+
let fellBack = false;
|
|
1753
|
+
const fallBackToProcesses = (why) => {
|
|
1754
|
+
if (fellBack)
|
|
1755
|
+
return;
|
|
1756
|
+
fellBack = true;
|
|
1757
|
+
this.logger.warn(`Worker threads unavailable (${why}); falling back to worker processes`, 'Cluster');
|
|
1758
|
+
const primary = this.threadCluster;
|
|
1759
|
+
this.threadCluster = undefined;
|
|
1760
|
+
void primary?.shutdown(1000).finally(() => {
|
|
1761
|
+
this.startProcessPrimary(workerCount, port, host, callback);
|
|
1762
|
+
});
|
|
1763
|
+
};
|
|
1764
|
+
const primary = new ThreadClusterPrimary({
|
|
1765
|
+
entry,
|
|
1766
|
+
argv: process.argv.slice(2),
|
|
1767
|
+
workerCount,
|
|
1768
|
+
logger: this.logger,
|
|
1769
|
+
...(resourceLimits ? { resourceLimits } : {}),
|
|
1770
|
+
onReady: () => {
|
|
1771
|
+
this.startPrimaryServices();
|
|
1772
|
+
if (callback)
|
|
1773
|
+
callback();
|
|
1774
|
+
},
|
|
1775
|
+
onFatal: (error, kind) => {
|
|
1776
|
+
if (kind === 'bind') {
|
|
1777
|
+
// Processes would fail identically: surface it like the single-process
|
|
1778
|
+
// path does (Node's server convention; rethrows without a listener).
|
|
1779
|
+
this.emit('error', error);
|
|
1780
|
+
return;
|
|
1719
1781
|
}
|
|
1720
|
-
|
|
1721
|
-
|
|
1782
|
+
fallBackToProcesses(error.message);
|
|
1783
|
+
},
|
|
1784
|
+
});
|
|
1785
|
+
this.threadCluster = primary;
|
|
1786
|
+
try {
|
|
1787
|
+
primary.start();
|
|
1788
|
+
}
|
|
1789
|
+
catch (error) {
|
|
1790
|
+
fallBackToProcesses(error instanceof Error ? error.message : String(error));
|
|
1791
|
+
return;
|
|
1792
|
+
}
|
|
1793
|
+
// Graceful shutdown: drain every worker thread (each closes its server),
|
|
1794
|
+
// then the primary's own services; a second signal exits at once.
|
|
1795
|
+
let shuttingDown = false;
|
|
1796
|
+
const gracefulShutdown = (code) => {
|
|
1797
|
+
if (shuttingDown) {
|
|
1798
|
+
process.exit(code);
|
|
1799
|
+
return;
|
|
1800
|
+
}
|
|
1801
|
+
shuttingDown = true;
|
|
1802
|
+
this.logger.info('Gracefully shutting down cluster...', 'Cluster');
|
|
1803
|
+
this.close()
|
|
1804
|
+
.then(() => process.exit(code))
|
|
1805
|
+
.catch(() => process.exit(1));
|
|
1806
|
+
};
|
|
1807
|
+
process.on('SIGINT', () => gracefulShutdown(0));
|
|
1808
|
+
process.on('SIGTERM', () => gracefulShutdown(0));
|
|
1809
|
+
}
|
|
1810
|
+
startProcessPrimary(workerCount, port, host, callback) {
|
|
1811
|
+
void port;
|
|
1812
|
+
void host;
|
|
1813
|
+
this.logger.info(`Starting ${workerCount} workers`, 'Cluster');
|
|
1814
|
+
// Optimize cluster scheduling for high concurrency
|
|
1815
|
+
// Round-robin is the default on all platforms except Windows (Node.js docs)
|
|
1816
|
+
// Provides better load distribution than shared socket approach
|
|
1817
|
+
nodeCluster().schedulingPolicy = nodeCluster().SCHED_RR;
|
|
1818
|
+
// Workers inherit the primary's execArgv; additionally bound each
|
|
1819
|
+
// worker's V8 young generation unless the user tuned it themselves.
|
|
1820
|
+
// On big machines V8 sizes the nursery from total RAM and grows it
|
|
1821
|
+
// under sustained load (~10-34 MB extra RSS per process, measured on a
|
|
1822
|
+
// 64 GB box) - and in cluster mode that cost multiplies by the worker
|
|
1823
|
+
// count. A/B-benched 2026-08-01: identical throughput at 4 MB semi-space
|
|
1824
|
+
// in both the realistic profile (~92k vs ~90k req/s) and the pipelined
|
|
1825
|
+
// x10 microbenchmark (~560k both ways), while sustained-load RSS drops
|
|
1826
|
+
// ~30 MB per worker (115 -> 81 MB on the 25-route POST soak).
|
|
1827
|
+
const userTunedGC = isUserTunedYoungGen();
|
|
1828
|
+
// Set cluster settings for better performance
|
|
1829
|
+
nodeCluster().setupMaster({
|
|
1830
|
+
exec: process.argv[1] || process.execPath,
|
|
1831
|
+
args: process.argv.slice(2),
|
|
1832
|
+
silent: false,
|
|
1833
|
+
...(userTunedGC ? {} : { execArgv: [...process.execArgv, '--max-semi-space-size=4'] }),
|
|
1834
|
+
});
|
|
1835
|
+
// IPC Optimization: Reduce communication overhead between master and workers
|
|
1836
|
+
// Research shows excessive IPC can create bottlenecks in clustered applications
|
|
1837
|
+
// (Source: BetterStack - Node.js Clustering Guide)
|
|
1838
|
+
process.env.NODE_CLUSTER_SCHED_POLICY = 'rr'; // Ensure round-robin
|
|
1839
|
+
process.env.NODE_DISABLE_COLORS = '1'; // Reduce IPC message size by disabling color codes
|
|
1840
|
+
// Graceful shutdown: SIGTERM every worker (each stops accepting, drains
|
|
1841
|
+
// its in-flight responses, then exits), wait for their exits up to a
|
|
1842
|
+
// bounded deadline, and only then exit. Exiting right after the kill
|
|
1843
|
+
// used to close the cluster IPC channel, and a node:cluster worker whose
|
|
1844
|
+
// primary disappears exits immediately - cutting the very responses the
|
|
1845
|
+
// worker was draining. A second signal while draining exits at once.
|
|
1846
|
+
let shuttingDown = false;
|
|
1847
|
+
const gracefulShutdown = () => {
|
|
1848
|
+
if (shuttingDown)
|
|
1849
|
+
process.exit(0);
|
|
1850
|
+
shuttingDown = true;
|
|
1851
|
+
this.logger.info('Gracefully shutting down cluster...', 'Cluster');
|
|
1852
|
+
if (this.clusterWorkers.size === 0) {
|
|
1853
|
+
nodeCluster().removeAllListeners();
|
|
1854
|
+
process.exit(0);
|
|
1855
|
+
}
|
|
1856
|
+
const pending = new Set(this.clusterWorkers.keys());
|
|
1857
|
+
const timer = setTimeout(() => {
|
|
1858
|
+
this.logger.warn(`Shutdown deadline reached; terminating ${pending.size} worker process(es)`, 'Cluster');
|
|
1859
|
+
for (const pid of pending)
|
|
1860
|
+
this.clusterWorkers.get(pid)?.kill('SIGKILL');
|
|
1861
|
+
finish();
|
|
1862
|
+
}, 3000);
|
|
1863
|
+
const finish = () => {
|
|
1864
|
+
clearTimeout(timer);
|
|
1865
|
+
nodeCluster().removeAllListeners();
|
|
1722
1866
|
process.exit(0);
|
|
1723
1867
|
};
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1868
|
+
for (const [pid, worker] of this.clusterWorkers) {
|
|
1869
|
+
worker.removeAllListeners(); // no restart on these exits
|
|
1870
|
+
worker.once('exit', () => {
|
|
1871
|
+
pending.delete(pid);
|
|
1872
|
+
if (pending.size === 0)
|
|
1873
|
+
finish();
|
|
1874
|
+
});
|
|
1875
|
+
worker.kill('SIGTERM');
|
|
1876
|
+
}
|
|
1877
|
+
};
|
|
1878
|
+
// Handle process signals for graceful shutdown
|
|
1879
|
+
process.on('SIGINT', gracefulShutdown);
|
|
1880
|
+
process.on('SIGTERM', gracefulShutdown);
|
|
1881
|
+
// Fork workers with basic tracking
|
|
1882
|
+
for (let i = 0; i < workerCount; i++) {
|
|
1883
|
+
const worker = nodeCluster().fork();
|
|
1884
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
1885
|
+
this.clusterWorkers.set(worker.process.pid, worker);
|
|
1886
|
+
this.logger.info(`Worker ${worker.process.pid} started`, 'Cluster');
|
|
1887
|
+
// Handle individual worker messages
|
|
1888
|
+
worker.on('message', this.handleWorkerMessage.bind(this));
|
|
1889
|
+
}
|
|
1890
|
+
// Primary-only services start once the first worker is serving.
|
|
1891
|
+
let servicesStarted = false;
|
|
1892
|
+
nodeCluster().on('listening', () => {
|
|
1893
|
+
if (servicesStarted)
|
|
1894
|
+
return;
|
|
1895
|
+
servicesStarted = true;
|
|
1896
|
+
this.startPrimaryServices();
|
|
1897
|
+
});
|
|
1898
|
+
// Simple worker exit handling
|
|
1899
|
+
nodeCluster().on('exit', (worker, code, signal) => {
|
|
1900
|
+
const pid = worker.process.pid;
|
|
1901
|
+
this.clusterWorkers.delete(pid);
|
|
1902
|
+
if (code !== 0 && !worker.exitedAfterDisconnect) {
|
|
1903
|
+
this.logger.warn(`Worker ${pid} died unexpectedly (${signal || code}). Restarting...`, 'Cluster');
|
|
1904
|
+
// Simple restart
|
|
1905
|
+
const newWorker = nodeCluster().fork();
|
|
1730
1906
|
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
1731
|
-
this.clusterWorkers.set(
|
|
1732
|
-
this.logger.info(`Worker ${
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1907
|
+
this.clusterWorkers.set(newWorker.process.pid, newWorker);
|
|
1908
|
+
this.logger.info(`Worker ${newWorker.process.pid} restarted`, 'Cluster');
|
|
1909
|
+
}
|
|
1910
|
+
});
|
|
1911
|
+
// Master process callback
|
|
1912
|
+
if (callback)
|
|
1913
|
+
callback();
|
|
1914
|
+
}
|
|
1915
|
+
// One worker, either a worker_threads thread or a node:cluster process:
|
|
1916
|
+
// boot the app exactly like the single-process path (auto-discovery,
|
|
1917
|
+
// file-based routes, WebSocket registrations, GraphQL) and listen with
|
|
1918
|
+
// SO_REUSEPORT. Signals/exit are the differences: a thread gets 'shutdown'
|
|
1919
|
+
// from the primary over parentPort and never calls process.exit().
|
|
1920
|
+
startClusterWorker(mode, port, host) {
|
|
1921
|
+
const label = mode.kind === 'thread' ? `Worker thread ${wt().threadId}` : `Worker ${process.pid}`;
|
|
1922
|
+
const workerRef = mode.kind === 'thread' ? wt().threadId : process.pid;
|
|
1923
|
+
this.logger.info(`${label} initializing`, 'Worker');
|
|
1924
|
+
if (mode.kind === 'process') {
|
|
1925
|
+
// Worker-specific optimizations for high concurrency (a thread shares
|
|
1926
|
+
// the primary's pool, sized before it was spawned)
|
|
1927
|
+
process.env.UV_THREADPOOL_SIZE = '64';
|
|
1928
|
+
}
|
|
1929
|
+
// Reduce logging contention in workers (major bottleneck)
|
|
1930
|
+
// Multiple workers writing to same log files creates I/O contention
|
|
1931
|
+
// ONLY reduce log level if user didn't explicitly set one
|
|
1932
|
+
if (!this.userSetLogger) {
|
|
1933
|
+
// Workers log less frequently to reduce I/O contention (only if not explicitly configured)
|
|
1934
|
+
applyLoggingConfiguration(undefined, { level: 'warn' }); // Only warnings and errors
|
|
1935
|
+
}
|
|
1936
|
+
// Optimize garbage collection for workers
|
|
1937
|
+
// eslint-disable-next-line no-undef
|
|
1938
|
+
if (global.gc) {
|
|
1939
|
+
setInterval(() => {
|
|
1940
|
+
// eslint-disable-next-line no-undef
|
|
1941
|
+
if (global.gc)
|
|
1942
|
+
global.gc();
|
|
1943
|
+
}, 60000); // GC every 60 seconds (less frequent)
|
|
1944
|
+
}
|
|
1945
|
+
let tcw = null;
|
|
1946
|
+
if (mode.kind === 'thread') {
|
|
1947
|
+
tcw = new ThreadClusterWorker(this.logger);
|
|
1948
|
+
// 'shutdown' from the primary: drain and close; the thread then exits
|
|
1949
|
+
// on its own once the engine's handles are gone.
|
|
1950
|
+
tcw.onShutdown(async () => {
|
|
1951
|
+
this.eventBus.removeAllListeners();
|
|
1952
|
+
this.removeAllListeners();
|
|
1953
|
+
await this.close();
|
|
1748
1954
|
});
|
|
1749
|
-
// Master process callback
|
|
1750
|
-
if (callback)
|
|
1751
|
-
callback();
|
|
1752
1955
|
}
|
|
1753
1956
|
else {
|
|
1754
|
-
// Worker process - start the actual server with proper cleanup
|
|
1755
|
-
this.logger.info(`Worker ${process.pid} initializing`, 'Worker');
|
|
1756
|
-
// Worker-specific optimizations for high concurrency
|
|
1757
|
-
process.env.UV_THREADPOOL_SIZE = '64';
|
|
1758
|
-
// Reduce logging contention in workers (major bottleneck)
|
|
1759
|
-
// Multiple workers writing to same log files creates I/O contention
|
|
1760
|
-
// ONLY reduce log level if user didn't explicitly set one
|
|
1761
|
-
if (!this.userSetLogger) {
|
|
1762
|
-
// Workers log less frequently to reduce I/O contention (only if not explicitly configured)
|
|
1763
|
-
applyLoggingConfiguration(undefined, { level: 'warn' }); // Only warnings and errors
|
|
1764
|
-
}
|
|
1765
|
-
// Research-based memory optimization for workers
|
|
1766
|
-
const totalMemoryGB = os.totalmem() / (1024 * 1024 * 1024);
|
|
1767
|
-
const workerCount = Object.keys(cluster.workers || {}).length || 1;
|
|
1768
|
-
// Conservative memory allocation
|
|
1769
|
-
const heapSizePerWorkerMB = Math.min(Math.floor(((totalMemoryGB * 1024) / workerCount) * 0.8), // 80% of available memory
|
|
1770
|
-
1536 // Cap at 1.5GB (GC efficiency threshold from research)
|
|
1771
|
-
);
|
|
1772
|
-
process.env.NODE_OPTIONS = `--max-old-space-size=${heapSizePerWorkerMB}`;
|
|
1773
|
-
this.logger.debug(`Worker memory allocated: ${heapSizePerWorkerMB}MB heap (${workerCount} workers, ${totalMemoryGB.toFixed(1)}GB total)`, 'Worker');
|
|
1774
|
-
// Optimize V8 flags for better performance
|
|
1775
|
-
if (process.env.NODE_ENV === 'production') {
|
|
1776
|
-
// Aggressive V8 optimizations for maximum performance
|
|
1777
|
-
const v8Flags = [
|
|
1778
|
-
'--optimize-for-size', // Trade memory for speed
|
|
1779
|
-
'--always-opt', // Always optimize functions
|
|
1780
|
-
'--turbo-fast-api-calls', // Optimize API calls
|
|
1781
|
-
'--turbo-escape-analysis', // Escape analysis optimization
|
|
1782
|
-
'--turbo-inline-api-calls', // Inline API calls
|
|
1783
|
-
`--max-old-space-size=${heapSizePerWorkerMB}`, // Limit memory to prevent GC pressure
|
|
1784
|
-
];
|
|
1785
|
-
process.env.NODE_OPTIONS = (process.env.NODE_OPTIONS || '') + ' ' + v8Flags.join(' ');
|
|
1786
|
-
}
|
|
1787
|
-
// Optimize garbage collection for workers
|
|
1788
|
-
// eslint-disable-next-line no-undef
|
|
1789
|
-
if (global.gc) {
|
|
1790
|
-
setInterval(() => {
|
|
1791
|
-
// eslint-disable-next-line no-undef
|
|
1792
|
-
if (global.gc)
|
|
1793
|
-
global.gc();
|
|
1794
|
-
}, 60000); // GC every 60 seconds (less frequent)
|
|
1795
|
-
}
|
|
1796
1957
|
// Graceful shutdown for worker
|
|
1797
1958
|
const workerShutdown = () => {
|
|
1798
|
-
this.logger.info(
|
|
1959
|
+
this.logger.info(`${label} shutting down gracefully...`, 'Worker');
|
|
1799
1960
|
// Clean up event listeners
|
|
1800
1961
|
this.eventBus.removeAllListeners();
|
|
1801
1962
|
this.removeAllListeners();
|
|
@@ -1812,79 +1973,96 @@ export class Moro extends EventEmitter {
|
|
|
1812
1973
|
// Handle worker shutdown signals
|
|
1813
1974
|
process.on('SIGTERM', workerShutdown);
|
|
1814
1975
|
process.on('SIGINT', workerShutdown);
|
|
1815
|
-
|
|
1816
|
-
|
|
1976
|
+
}
|
|
1977
|
+
const fatal = (code, message) => {
|
|
1978
|
+
this.logger.error(`${label} ${message}`, 'Worker');
|
|
1979
|
+
if (tcw) {
|
|
1980
|
+
tcw.fatal(code, message);
|
|
1981
|
+
void this.close().finally(() => tcw?.close());
|
|
1982
|
+
}
|
|
1983
|
+
else {
|
|
1984
|
+
process.exit(1);
|
|
1985
|
+
}
|
|
1986
|
+
};
|
|
1987
|
+
// Continue with normal server startup for this worker
|
|
1988
|
+
void this.eventBus.emit('server:starting', {
|
|
1989
|
+
port,
|
|
1990
|
+
runtime: this.runtimeType,
|
|
1991
|
+
worker: workerRef,
|
|
1992
|
+
});
|
|
1993
|
+
// Add documentation middleware first (if enabled)
|
|
1994
|
+
try {
|
|
1995
|
+
const docsMiddleware = this.documentation.getDocsMiddleware();
|
|
1996
|
+
this.coreFramework.addMiddleware(docsMiddleware);
|
|
1997
|
+
}
|
|
1998
|
+
catch {
|
|
1999
|
+
// Documentation not enabled, that's fine
|
|
2000
|
+
}
|
|
2001
|
+
// Attach the unified router (handles both chainable and direct routes)
|
|
2002
|
+
this.attachUnifiedRouter();
|
|
2003
|
+
// Register legacy direct routes with the HTTP server (for backward compatibility)
|
|
2004
|
+
if (this.routes.length > 0) {
|
|
2005
|
+
this.registerDirectRoutes();
|
|
2006
|
+
}
|
|
2007
|
+
const workerCallback = () => {
|
|
2008
|
+
const displayHost = host || 'localhost';
|
|
2009
|
+
this.logger.info(`${label} ready on ${displayHost}:${port}`, 'Worker');
|
|
2010
|
+
void this.eventBus.emit('server:started', {
|
|
1817
2011
|
port,
|
|
1818
2012
|
runtime: this.runtimeType,
|
|
1819
|
-
worker:
|
|
2013
|
+
worker: workerRef,
|
|
2014
|
+
engine: this.engine,
|
|
1820
2015
|
});
|
|
1821
|
-
//
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
2016
|
+
// GraphQL registers HTTP/WS routes on this app: every worker.
|
|
2017
|
+
this.ensureGraphQLInitialized().catch(error => {
|
|
2018
|
+
this.logger.error(`Failed to initialize GraphQL: ${error}`, 'GraphQL');
|
|
2019
|
+
});
|
|
2020
|
+
tcw?.ready(port);
|
|
2021
|
+
};
|
|
2022
|
+
const startWorkerListening = () => {
|
|
2023
|
+
if (host) {
|
|
2024
|
+
this.coreFramework.listen(port, host, workerCallback);
|
|
1825
2025
|
}
|
|
1826
|
-
|
|
1827
|
-
|
|
2026
|
+
else {
|
|
2027
|
+
this.coreFramework.listen(port, workerCallback);
|
|
1828
2028
|
}
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
2029
|
+
};
|
|
2030
|
+
// Same init as the single-process path: auto-discovery, file-based
|
|
2031
|
+
// routes, queued WebSocket registrations - before 1.9.0 workers skipped
|
|
2032
|
+
// discovery/route loading entirely.
|
|
2033
|
+
Promise.all([
|
|
2034
|
+
this.ensureAutoDiscoveryComplete().then(() => this.ensureRoutesLoaded()),
|
|
2035
|
+
this.processQueuedWebSocketRegistrations(),
|
|
2036
|
+
])
|
|
2037
|
+
.then(startWorkerListening)
|
|
2038
|
+
.catch(error => {
|
|
2039
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2040
|
+
const code = error?.code;
|
|
2041
|
+
// listen() itself failed (EADDRINUSE etc.): retrying is guaranteed to
|
|
2042
|
+
// fail again and would surface as an unhandled rejection crash-loop.
|
|
2043
|
+
// Report fatally so the primary sees a clean death.
|
|
2044
|
+
if (code === 'EADDRINUSE' ||
|
|
2045
|
+
code === 'EACCES' ||
|
|
2046
|
+
/EADDRINUSE|failed to bind/i.test(message)) {
|
|
2047
|
+
fatal(code === 'EACCES' ? 'EACCES' : 'EADDRINUSE', `could not bind port ${port}: ${message}`);
|
|
2048
|
+
return;
|
|
1834
2049
|
}
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
this.coreFramework.listen(port, workerCallback);
|
|
1852
|
-
}
|
|
1853
|
-
};
|
|
1854
|
-
this.processQueuedWebSocketRegistrations()
|
|
1855
|
-
.then(startWorkerListening)
|
|
1856
|
-
.catch(error => {
|
|
1857
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
1858
|
-
const code = error?.code;
|
|
1859
|
-
// listen() itself failed (EADDRINUSE etc.): retrying is guaranteed to
|
|
1860
|
-
// fail again and would surface as an unhandled rejection crash-loop.
|
|
1861
|
-
// Log fatally and exit so the cluster primary sees a clean death.
|
|
1862
|
-
// Prefer the code the native/Node servers set; fall back to message.
|
|
1863
|
-
if (code === 'EADDRINUSE' ||
|
|
1864
|
-
code === 'EACCES' ||
|
|
1865
|
-
/EADDRINUSE|failed to bind/i.test(message)) {
|
|
1866
|
-
this.logger.error(`Worker ${process.pid} could not bind port ${port}: ${message}`, 'Worker');
|
|
1867
|
-
process.exit(1);
|
|
1868
|
-
}
|
|
1869
|
-
this.logger.error('WebSocket initialization failed in worker', 'Worker', {
|
|
1870
|
-
error: message,
|
|
1871
|
-
});
|
|
1872
|
-
// For WebSocket failures with queued registrations, error will propagate
|
|
1873
|
-
if (error instanceof Error &&
|
|
1874
|
-
error.message.includes('WebSocket features require a WebSocket adapter')) {
|
|
1875
|
-
throw error;
|
|
1876
|
-
}
|
|
1877
|
-
// WebSocket setup failed but the HTTP server can still serve: start
|
|
1878
|
-
// anyway, and surface (not swallow) a listen failure from this path.
|
|
1879
|
-
try {
|
|
1880
|
-
startWorkerListening();
|
|
1881
|
-
}
|
|
1882
|
-
catch (listenError) {
|
|
1883
|
-
this.logger.error(`Worker ${process.pid} failed to start: ${listenError instanceof Error ? listenError.message : String(listenError)}`, 'Worker');
|
|
1884
|
-
process.exit(1);
|
|
1885
|
-
}
|
|
1886
|
-
});
|
|
1887
|
-
}
|
|
2050
|
+
this.logger.error('Initialization failed in worker', 'Worker', { error: message });
|
|
2051
|
+
// For WebSocket failures with queued registrations, error will propagate
|
|
2052
|
+
if (error instanceof Error &&
|
|
2053
|
+
error.message.includes('WebSocket features require a WebSocket adapter')) {
|
|
2054
|
+
throw error;
|
|
2055
|
+
}
|
|
2056
|
+
// Init failed but the HTTP server can still serve: start anyway, and
|
|
2057
|
+
// surface (not swallow) a listen failure from this path.
|
|
2058
|
+
try {
|
|
2059
|
+
startWorkerListening();
|
|
2060
|
+
}
|
|
2061
|
+
catch (listenError) {
|
|
2062
|
+
const lm = listenError instanceof Error ? listenError.message : String(listenError);
|
|
2063
|
+
fatal(listenError?.code === 'EACCES' ? 'EACCES' : 'EADDRINUSE', `failed to start: ${lm}`);
|
|
2064
|
+
}
|
|
2065
|
+
});
|
|
1888
2066
|
}
|
|
1889
2067
|
// Simple worker message handler
|
|
1890
2068
|
handleWorkerMessage(message) {
|
|
@@ -1999,6 +2177,18 @@ export class Moro extends EventEmitter {
|
|
|
1999
2177
|
}
|
|
2000
2178
|
}
|
|
2001
2179
|
}
|
|
2180
|
+
// Cluster worker threads first: each drains its server via its own
|
|
2181
|
+
// close(); the primary's services below stop after the traffic did.
|
|
2182
|
+
if (this.threadCluster) {
|
|
2183
|
+
const primary = this.threadCluster;
|
|
2184
|
+
this.threadCluster = undefined;
|
|
2185
|
+
try {
|
|
2186
|
+
await primary.shutdown();
|
|
2187
|
+
}
|
|
2188
|
+
catch (err) {
|
|
2189
|
+
this.logger.error(`Error shutting down cluster worker threads: ${String(err)}`);
|
|
2190
|
+
}
|
|
2191
|
+
}
|
|
2002
2192
|
// Shutdown worker threads first (fastest)
|
|
2003
2193
|
try {
|
|
2004
2194
|
this.logger.info('Shutting down worker threads...');
|