@omnitron-dev/titan-pm 0.1.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backoff.d.ts +12 -0
- package/dist/backoff.d.ts.map +1 -0
- package/dist/backoff.js +44 -0
- package/dist/backoff.js.map +1 -0
- package/dist/child-contract.d.ts +58 -0
- package/dist/child-contract.d.ts.map +1 -0
- package/dist/child-contract.js +27 -0
- package/dist/child-contract.js.map +1 -0
- package/dist/confirm-death.d.ts +18 -0
- package/dist/confirm-death.d.ts.map +1 -0
- package/dist/confirm-death.js +49 -0
- package/dist/confirm-death.js.map +1 -0
- package/dist/decorators.d.ts +1 -10
- package/dist/decorators.d.ts.map +1 -1
- package/dist/decorators.js +43 -99
- package/dist/decorators.js.map +1 -1
- package/dist/discovery.d.ts +24 -0
- package/dist/discovery.d.ts.map +1 -0
- package/dist/discovery.js +162 -0
- package/dist/discovery.js.map +1 -0
- package/dist/fork-worker.js +64 -27
- package/dist/fork-worker.js.map +1 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/last-resort-handlers.d.ts +5 -0
- package/dist/last-resort-handlers.d.ts.map +1 -0
- package/dist/last-resort-handlers.js +40 -0
- package/dist/last-resort-handlers.js.map +1 -0
- package/dist/lifecycle-queue.d.ts +20 -0
- package/dist/lifecycle-queue.d.ts.map +1 -0
- package/dist/lifecycle-queue.js +110 -0
- package/dist/lifecycle-queue.js.map +1 -0
- package/dist/liveness.d.ts +5 -0
- package/dist/liveness.d.ts.map +1 -0
- package/dist/liveness.js +34 -0
- package/dist/liveness.js.map +1 -0
- package/dist/netron-client.d.ts +10 -9
- package/dist/netron-client.d.ts.map +1 -1
- package/dist/netron-client.js +32 -21
- package/dist/netron-client.js.map +1 -1
- package/dist/parent-death-watchdog.d.ts +13 -0
- package/dist/parent-death-watchdog.d.ts.map +1 -0
- package/dist/parent-death-watchdog.js +58 -0
- package/dist/parent-death-watchdog.js.map +1 -0
- package/dist/pm.module.d.ts.map +1 -1
- package/dist/pm.module.js.map +1 -1
- package/dist/process-health.d.ts +2 -0
- package/dist/process-health.d.ts.map +1 -1
- package/dist/process-health.js +49 -13
- package/dist/process-health.js.map +1 -1
- package/dist/process-manager.d.ts +6 -0
- package/dist/process-manager.d.ts.map +1 -1
- package/dist/process-manager.js +111 -4
- package/dist/process-manager.js.map +1 -1
- package/dist/process-metrics.d.ts +1 -0
- package/dist/process-metrics.d.ts.map +1 -1
- package/dist/process-metrics.js +18 -19
- package/dist/process-metrics.js.map +1 -1
- package/dist/process-pool.d.ts +12 -0
- package/dist/process-pool.d.ts.map +1 -1
- package/dist/process-pool.js +340 -147
- package/dist/process-pool.js.map +1 -1
- package/dist/process-registry.d.ts.map +1 -1
- package/dist/process-spawner.d.ts +27 -5
- package/dist/process-spawner.d.ts.map +1 -1
- package/dist/process-spawner.js +397 -70
- package/dist/process-spawner.js.map +1 -1
- package/dist/process-supervisor.d.ts +7 -0
- package/dist/process-supervisor.d.ts.map +1 -1
- package/dist/process-supervisor.js +141 -17
- package/dist/process-supervisor.js.map +1 -1
- package/dist/process-workflow.d.ts.map +1 -1
- package/dist/process-workflow.js +11 -4
- package/dist/process-workflow.js.map +1 -1
- package/dist/service-proxy.d.ts.map +1 -1
- package/dist/service-proxy.js +33 -3
- package/dist/service-proxy.js.map +1 -1
- package/dist/shutdown-windows.d.ts +19 -0
- package/dist/shutdown-windows.d.ts.map +1 -0
- package/dist/shutdown-windows.js +35 -0
- package/dist/shutdown-windows.js.map +1 -0
- package/dist/thread-pool.d.ts +7 -0
- package/dist/thread-pool.d.ts.map +1 -0
- package/dist/thread-pool.js +11 -0
- package/dist/thread-pool.js.map +1 -0
- package/dist/types.d.ts +91 -37
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +35 -39
- package/dist/types.js.map +1 -1
- package/dist/utils/deque.d.ts.map +1 -1
- package/dist/worker-health.d.ts +8 -0
- package/dist/worker-health.d.ts.map +1 -0
- package/dist/worker-health.js +63 -0
- package/dist/worker-health.js.map +1 -0
- package/dist/worker-runtime.js +25 -52
- package/dist/worker-runtime.js.map +1 -1
- package/dist/worker-traffic.d.ts +8 -0
- package/dist/worker-traffic.d.ts.map +1 -0
- package/dist/worker-traffic.js +56 -0
- package/dist/worker-traffic.js.map +1 -0
- package/package.json +11 -10
package/dist/process-pool.js
CHANGED
|
@@ -2,6 +2,7 @@ import { cpus } from 'node:os';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { EventEmitter } from '@omnitron-dev/eventemitter';
|
|
4
4
|
import { Errors } from '@omnitron-dev/titan/errors';
|
|
5
|
+
import { FailureTracker, computeBackoff } from '@omnitron-dev/titan/utils';
|
|
5
6
|
import { PoolStrategy } from './types.js';
|
|
6
7
|
import { Deque } from './utils/index.js';
|
|
7
8
|
export class PoolBackpressureError extends Error {
|
|
@@ -31,6 +32,8 @@ export class ProcessPool {
|
|
|
31
32
|
this.lastScaleCheck = Date.now();
|
|
32
33
|
this.scaleHistory = [];
|
|
33
34
|
this.unhealthyWorkers = new Set();
|
|
35
|
+
this.healthFailures = new FailureTracker();
|
|
36
|
+
this.cpuSamples = new Map();
|
|
34
37
|
this.queueBatchSize = 10;
|
|
35
38
|
this.queueProcessingScheduled = false;
|
|
36
39
|
this.circuitBreaker = {
|
|
@@ -42,6 +45,13 @@ export class ProcessPool {
|
|
|
42
45
|
this.emitter = new EventEmitter();
|
|
43
46
|
this.healthyWorkersCache = [];
|
|
44
47
|
this.healthyWorkersCacheValid = false;
|
|
48
|
+
this.healthCheckRunning = false;
|
|
49
|
+
this.autoScaleRunning = false;
|
|
50
|
+
this.autoScaleConsecutiveFailures = 0;
|
|
51
|
+
this.autoScaleNextRetryAt = 0;
|
|
52
|
+
this.recyclingRunning = false;
|
|
53
|
+
this.memoryMonitorRunning = false;
|
|
54
|
+
this.heartbeatRunning = false;
|
|
45
55
|
this.poolOptions = this.normalizeOptions(options);
|
|
46
56
|
this.memoryLimitBytes = this.parseMemoryLimit(options.memoryLimit || '512MB');
|
|
47
57
|
this.memoryWarningThreshold = options.memoryWarningThreshold || 0.8;
|
|
@@ -157,8 +167,8 @@ export class ProcessPool {
|
|
|
157
167
|
const pmProcessId = proxy?.__processId;
|
|
158
168
|
if (!pmProcessId)
|
|
159
169
|
return null;
|
|
160
|
-
const handle = this.manager.
|
|
161
|
-
return handle ?? null;
|
|
170
|
+
const handle = this.manager.getWorkerHandle?.(pmProcessId);
|
|
171
|
+
return (handle ?? null);
|
|
162
172
|
}
|
|
163
173
|
get metrics() {
|
|
164
174
|
const avgResponseTime = this.totalRequests > 0 ? this.totalResponseTime / this.totalRequests : 0;
|
|
@@ -166,19 +176,35 @@ export class ProcessPool {
|
|
|
166
176
|
let totalCpu = 0;
|
|
167
177
|
let totalMemory = 0;
|
|
168
178
|
let healthyWorkers = 0;
|
|
179
|
+
let totalCpuPercent = 0;
|
|
180
|
+
let cpuPercentSamples = 0;
|
|
181
|
+
let totalMemoryPercent = 0;
|
|
182
|
+
let memoryPercentSamples = 0;
|
|
169
183
|
this.workers.forEach((worker) => {
|
|
170
184
|
if (worker.metrics) {
|
|
171
185
|
totalCpu += worker.metrics.cpu;
|
|
172
186
|
totalMemory += worker.metrics.memory;
|
|
187
|
+
if (typeof worker.metrics.cpuPercent === 'number') {
|
|
188
|
+
totalCpuPercent += worker.metrics.cpuPercent;
|
|
189
|
+
cpuPercentSamples++;
|
|
190
|
+
}
|
|
191
|
+
if (typeof worker.metrics.memoryPercent === 'number') {
|
|
192
|
+
totalMemoryPercent += worker.metrics.memoryPercent;
|
|
193
|
+
memoryPercentSamples++;
|
|
194
|
+
}
|
|
173
195
|
}
|
|
174
196
|
if (worker.health === 'healthy')
|
|
175
197
|
healthyWorkers++;
|
|
176
198
|
});
|
|
177
199
|
const avgCpu = this.workers.size > 0 ? totalCpu / this.workers.size : 0;
|
|
178
200
|
const avgMemory = this.workers.size > 0 ? totalMemory / this.workers.size : 0;
|
|
201
|
+
const avgCpuPercent = cpuPercentSamples > 0 ? totalCpuPercent / cpuPercentSamples : undefined;
|
|
202
|
+
const avgMemoryPercent = memoryPercentSamples > 0 ? totalMemoryPercent / memoryPercentSamples : undefined;
|
|
179
203
|
return {
|
|
180
204
|
cpu: avgCpu,
|
|
181
205
|
memory: avgMemory,
|
|
206
|
+
...(avgCpuPercent !== undefined ? { cpuPercent: avgCpuPercent } : {}),
|
|
207
|
+
...(avgMemoryPercent !== undefined ? { memoryPercent: avgMemoryPercent } : {}),
|
|
182
208
|
queueSize: this.queue.length,
|
|
183
209
|
activeWorkers: this.active,
|
|
184
210
|
totalWorkers: this.workers.size,
|
|
@@ -278,6 +304,10 @@ export class ProcessPool {
|
|
|
278
304
|
clearInterval(this.memoryMonitorTimer);
|
|
279
305
|
this.memoryMonitorTimer = undefined;
|
|
280
306
|
}
|
|
307
|
+
if (this.circuitBreakerHalfOpenTimer) {
|
|
308
|
+
clearTimeout(this.circuitBreakerHalfOpenTimer);
|
|
309
|
+
this.circuitBreakerHalfOpenTimer = undefined;
|
|
310
|
+
}
|
|
281
311
|
await this.drain();
|
|
282
312
|
const shutdownPromises = Array.from(this.workers.values()).map((worker) => this.shutdownWorker(worker.id));
|
|
283
313
|
await Promise.all(shutdownPromises);
|
|
@@ -285,8 +315,15 @@ export class ProcessPool {
|
|
|
285
315
|
this.isInitialized = false;
|
|
286
316
|
this.isShuttingDown = false;
|
|
287
317
|
this.isDraining = false;
|
|
288
|
-
|
|
289
|
-
|
|
318
|
+
try {
|
|
319
|
+
this.emit('pool:destroyed', { class: this.processName });
|
|
320
|
+
}
|
|
321
|
+
catch (err) {
|
|
322
|
+
this.logger.warn({ err }, 'pool:destroyed subscriber threw — continuing teardown');
|
|
323
|
+
}
|
|
324
|
+
finally {
|
|
325
|
+
this.emitter.removeAllListeners();
|
|
326
|
+
}
|
|
290
327
|
}
|
|
291
328
|
async execute(method, ...args) {
|
|
292
329
|
if (this.poolOptions.circuitBreaker?.enabled && this.circuitBreaker.isOpen) {
|
|
@@ -297,8 +334,7 @@ export class ProcessPool {
|
|
|
297
334
|
if (this.isDraining) {
|
|
298
335
|
throw Errors.conflict('Pool is draining, not accepting new requests');
|
|
299
336
|
}
|
|
300
|
-
if (this.workers.size === 0 ||
|
|
301
|
-
(this.active >= this.workers.size && this.queue.length < (this.poolOptions.maxQueueSize || 100))) {
|
|
337
|
+
if (this.workers.size === 0 || this.active >= this.workers.size) {
|
|
302
338
|
return this.queueRequest(method, args);
|
|
303
339
|
}
|
|
304
340
|
const worker = await this.selectWorker();
|
|
@@ -319,6 +355,9 @@ export class ProcessPool {
|
|
|
319
355
|
: {};
|
|
320
356
|
const proxy = await this.manager.spawn(this.processPathOrClass, {
|
|
321
357
|
name: `${this.processName}-pool-${workerIndex}`,
|
|
358
|
+
...(this.poolOptions.requestTimeout !== undefined && {
|
|
359
|
+
requestTimeout: this.poolOptions.requestTimeout,
|
|
360
|
+
}),
|
|
322
361
|
...baseOptions,
|
|
323
362
|
...factoryOptions,
|
|
324
363
|
dependencies: {
|
|
@@ -365,11 +404,14 @@ export class ProcessPool {
|
|
|
365
404
|
return;
|
|
366
405
|
try {
|
|
367
406
|
if (worker.processing.size > 0) {
|
|
368
|
-
this.logger.debug({
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
407
|
+
this.logger.debug({ workerId, activeRequests: worker.processing.size }, 'Waiting for worker to finish active requests');
|
|
408
|
+
const deadline = Date.now() + 5_000;
|
|
409
|
+
while (worker.processing.size > 0 && Date.now() < deadline) {
|
|
410
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
411
|
+
}
|
|
412
|
+
if (worker.processing.size > 0) {
|
|
413
|
+
this.logger.warn({ workerId, activeRequests: worker.processing.size }, 'Worker still has active requests after grace period; killing anyway');
|
|
414
|
+
}
|
|
373
415
|
}
|
|
374
416
|
await this.manager.kill(workerId);
|
|
375
417
|
this.workers.delete(workerId);
|
|
@@ -566,13 +608,17 @@ export class ProcessPool {
|
|
|
566
608
|
worker.processing.add(requestId);
|
|
567
609
|
worker.currentLoad++;
|
|
568
610
|
this.totalRequests++;
|
|
611
|
+
let timeoutHandle = null;
|
|
569
612
|
try {
|
|
570
613
|
const timeout = this.poolOptions.requestTimeout;
|
|
571
614
|
let result;
|
|
572
615
|
if (timeout) {
|
|
616
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
617
|
+
timeoutHandle = setTimeout(() => reject(Errors.timeout('Request', timeout)), timeout);
|
|
618
|
+
});
|
|
573
619
|
result = await Promise.race([
|
|
574
620
|
worker.proxy[method](...args),
|
|
575
|
-
|
|
621
|
+
timeoutPromise,
|
|
576
622
|
]);
|
|
577
623
|
}
|
|
578
624
|
else {
|
|
@@ -597,7 +643,7 @@ export class ProcessPool {
|
|
|
597
643
|
if (this.poolOptions.circuitBreaker?.enabled) {
|
|
598
644
|
this.handleCircuitBreakerFailure();
|
|
599
645
|
}
|
|
600
|
-
if (worker.errors > (this.poolOptions.healthCheck?.unhealthyThreshold
|
|
646
|
+
if (worker.errors > (this.poolOptions.healthCheck?.unhealthyThreshold ?? 3)) {
|
|
601
647
|
worker.health = 'unhealthy';
|
|
602
648
|
this.unhealthyWorkers.add(worker.id);
|
|
603
649
|
this.invalidateHealthyWorkersCache();
|
|
@@ -606,13 +652,15 @@ export class ProcessPool {
|
|
|
606
652
|
throw error;
|
|
607
653
|
}
|
|
608
654
|
finally {
|
|
655
|
+
if (timeoutHandle)
|
|
656
|
+
clearTimeout(timeoutHandle);
|
|
609
657
|
worker.processing.delete(requestId);
|
|
610
|
-
worker.currentLoad
|
|
658
|
+
worker.currentLoad = Math.max(0, worker.currentLoad - 1);
|
|
611
659
|
worker.lastUsed = Date.now();
|
|
612
660
|
}
|
|
613
661
|
}
|
|
614
662
|
async queueRequest(method, args) {
|
|
615
|
-
if (this.queue.length >= (this.poolOptions.maxQueueSize
|
|
663
|
+
if (this.queue.length >= (this.poolOptions.maxQueueSize ?? 100)) {
|
|
616
664
|
throw new PoolBackpressureError(this.queue.length);
|
|
617
665
|
}
|
|
618
666
|
return new Promise((resolve, reject) => {
|
|
@@ -639,7 +687,9 @@ export class ProcessPool {
|
|
|
639
687
|
queueSize: this.queue.length,
|
|
640
688
|
method,
|
|
641
689
|
});
|
|
642
|
-
this.processQueue()
|
|
690
|
+
void this.processQueue().catch((err) => {
|
|
691
|
+
this.logger.error({ err, class: this.processName }, 'Process pool queue processing failed');
|
|
692
|
+
});
|
|
643
693
|
});
|
|
644
694
|
}
|
|
645
695
|
async processQueue() {
|
|
@@ -679,7 +729,9 @@ export class ProcessPool {
|
|
|
679
729
|
processed++;
|
|
680
730
|
}
|
|
681
731
|
if (this.queue.length > 0) {
|
|
682
|
-
this.processQueue()
|
|
732
|
+
void this.processQueue().catch((err) => {
|
|
733
|
+
this.logger.error({ err, class: this.processName }, 'Process pool queue processing failed');
|
|
734
|
+
});
|
|
683
735
|
}
|
|
684
736
|
}
|
|
685
737
|
selectBestFromTwo(workers, idx1, idx2) {
|
|
@@ -689,68 +741,116 @@ export class ProcessPool {
|
|
|
689
741
|
const score2 = worker2.currentLoad * 10 + (worker2.memoryUsage / this.memoryLimitBytes) * 5;
|
|
690
742
|
return score1 <= score2 ? idx1 : idx2;
|
|
691
743
|
}
|
|
744
|
+
emitHealthDecision(decision, workerId, error) {
|
|
745
|
+
if (!decision)
|
|
746
|
+
return;
|
|
747
|
+
const payload = {
|
|
748
|
+
workerId,
|
|
749
|
+
consecutiveFailures: decision.count,
|
|
750
|
+
elapsedMs: decision.elapsedMs,
|
|
751
|
+
...(error !== undefined && { error }),
|
|
752
|
+
};
|
|
753
|
+
switch (decision.level) {
|
|
754
|
+
case 'first':
|
|
755
|
+
this.logger.warn(payload, 'Worker health check started failing');
|
|
756
|
+
break;
|
|
757
|
+
case 'continuing':
|
|
758
|
+
this.logger.debug(payload, 'Worker health check still failing');
|
|
759
|
+
break;
|
|
760
|
+
case 'persistent':
|
|
761
|
+
this.logger.error(payload, 'Worker health check persistent failure — worker will be replaced when threshold is reached');
|
|
762
|
+
break;
|
|
763
|
+
case 'suppress':
|
|
764
|
+
break;
|
|
765
|
+
case 'recovery':
|
|
766
|
+
this.logger.info(payload, 'Worker health check recovered');
|
|
767
|
+
break;
|
|
768
|
+
default: {
|
|
769
|
+
const unexpected = decision.level;
|
|
770
|
+
this.logger.warn({ ...payload, level: unexpected }, 'Unhandled failure-tracker level');
|
|
771
|
+
break;
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
}
|
|
692
775
|
setupHealthMonitoring() {
|
|
693
776
|
const baseInterval = this.poolOptions.healthCheck?.interval || 30000;
|
|
694
777
|
const HEALTH_CHECK_CONCURRENCY = 10;
|
|
695
778
|
const MAX_BACKOFF_MULTIPLIER = 8;
|
|
696
779
|
this.healthCheckTimer = setInterval(async () => {
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
780
|
+
if (this.healthCheckRunning)
|
|
781
|
+
return;
|
|
782
|
+
this.healthCheckRunning = true;
|
|
783
|
+
try {
|
|
784
|
+
let healthChanged = false;
|
|
785
|
+
const workers = Array.from(this.workers.values());
|
|
786
|
+
const now = Date.now();
|
|
787
|
+
const checkWorkerHealth = async (worker) => {
|
|
788
|
+
const previousHealth = worker.health;
|
|
789
|
+
if (worker.consecutiveFailures > 0) {
|
|
790
|
+
const backoffMultiplier = computeBackoff({
|
|
791
|
+
attempt: worker.consecutiveFailures - 1,
|
|
792
|
+
baseMs: 1,
|
|
793
|
+
maxMs: MAX_BACKOFF_MULTIPLIER,
|
|
794
|
+
});
|
|
795
|
+
const nextCheckTime = worker.lastHeartbeat + baseInterval * backoffMultiplier;
|
|
796
|
+
if (now < nextCheckTime) {
|
|
797
|
+
return false;
|
|
798
|
+
}
|
|
707
799
|
}
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
health
|
|
714
|
-
|
|
715
|
-
|
|
800
|
+
try {
|
|
801
|
+
if ('__getHealth' in worker.proxy) {
|
|
802
|
+
const health = await worker.proxy.__getHealth();
|
|
803
|
+
worker.health =
|
|
804
|
+
health.status === 'healthy' ? 'healthy' : health.status === 'degraded' ? 'degraded' : 'unhealthy';
|
|
805
|
+
if (worker.health === 'healthy') {
|
|
806
|
+
worker.consecutiveFailures = 0;
|
|
807
|
+
this.emitHealthDecision(this.healthFailures.recordSuccess(worker.id), worker.id, undefined);
|
|
808
|
+
}
|
|
716
809
|
}
|
|
717
810
|
}
|
|
718
|
-
|
|
719
|
-
|
|
811
|
+
catch (error) {
|
|
812
|
+
const decision = this.healthFailures.recordFailure(worker.id);
|
|
813
|
+
this.emitHealthDecision(decision, worker.id, error);
|
|
814
|
+
worker.health = 'unhealthy';
|
|
815
|
+
worker.consecutiveFailures++;
|
|
816
|
+
}
|
|
817
|
+
try {
|
|
818
|
+
const raw = (await worker.proxy
|
|
819
|
+
.__getMetrics?.());
|
|
820
|
+
if (raw) {
|
|
821
|
+
worker.metrics = this.withDerivedPercentages(worker.id, raw);
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
catch (error) {
|
|
825
|
+
this.logger.debug({ workerId: worker.id, error }, 'Worker metrics unavailable this tick');
|
|
826
|
+
}
|
|
827
|
+
return previousHealth !== worker.health;
|
|
828
|
+
};
|
|
829
|
+
for (let i = 0; i < workers.length; i += HEALTH_CHECK_CONCURRENCY) {
|
|
830
|
+
const batch = workers.slice(i, i + HEALTH_CHECK_CONCURRENCY);
|
|
831
|
+
const results = await Promise.all(batch.map(checkWorkerHealth));
|
|
832
|
+
if (results.some((changed) => changed)) {
|
|
833
|
+
healthChanged = true;
|
|
720
834
|
}
|
|
721
835
|
}
|
|
722
|
-
|
|
723
|
-
this.
|
|
724
|
-
error,
|
|
725
|
-
workerId: worker.id,
|
|
726
|
-
consecutiveFailures: worker.consecutiveFailures + 1,
|
|
727
|
-
}, 'Health check failed');
|
|
728
|
-
worker.health = 'unhealthy';
|
|
729
|
-
worker.consecutiveFailures++;
|
|
836
|
+
if (healthChanged) {
|
|
837
|
+
this.invalidateHealthyWorkersCache();
|
|
730
838
|
}
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
839
|
+
const unhealthyThreshold = this.poolOptions.healthCheck?.unhealthyThreshold ?? 3;
|
|
840
|
+
for (const worker of workers) {
|
|
841
|
+
if (worker.health === 'unhealthy' && worker.consecutiveFailures >= unhealthyThreshold && !this.isShuttingDown) {
|
|
842
|
+
this.logger.info({
|
|
843
|
+
workerId: worker.id,
|
|
844
|
+
consecutiveFailures: worker.consecutiveFailures,
|
|
845
|
+
restarts: worker.restarts,
|
|
846
|
+
}, 'Replacing unhealthy worker');
|
|
847
|
+
worker.restarts++;
|
|
848
|
+
await this.replaceWorker(worker.id);
|
|
849
|
+
}
|
|
738
850
|
}
|
|
739
851
|
}
|
|
740
|
-
|
|
741
|
-
this.
|
|
742
|
-
}
|
|
743
|
-
const unhealthyThreshold = this.poolOptions.healthCheck?.unhealthyThreshold || 3;
|
|
744
|
-
for (const worker of workers) {
|
|
745
|
-
if (worker.health === 'unhealthy' && worker.consecutiveFailures >= unhealthyThreshold && !this.isShuttingDown) {
|
|
746
|
-
this.logger.info({
|
|
747
|
-
workerId: worker.id,
|
|
748
|
-
consecutiveFailures: worker.consecutiveFailures,
|
|
749
|
-
restarts: worker.restarts,
|
|
750
|
-
}, 'Replacing unhealthy worker');
|
|
751
|
-
worker.restarts++;
|
|
752
|
-
await this.replaceWorker(worker.id);
|
|
753
|
-
}
|
|
852
|
+
finally {
|
|
853
|
+
this.healthCheckRunning = false;
|
|
754
854
|
}
|
|
755
855
|
}, baseInterval);
|
|
756
856
|
}
|
|
@@ -758,42 +858,71 @@ export class ProcessPool {
|
|
|
758
858
|
if (!this.poolOptions.autoScale?.enabled)
|
|
759
859
|
return;
|
|
760
860
|
const checkInterval = this.poolOptions.autoScale?.checkInterval ?? 10_000;
|
|
861
|
+
const FAILURE_BACKOFF_MAX_MS = 5 * 60_000;
|
|
761
862
|
this.autoScaleTimer = setInterval(async () => {
|
|
762
863
|
if (this.isShuttingDown || this.isDraining)
|
|
763
864
|
return;
|
|
865
|
+
if (this.autoScaleRunning)
|
|
866
|
+
return;
|
|
867
|
+
const now = Date.now();
|
|
868
|
+
if (now < this.autoScaleNextRetryAt)
|
|
869
|
+
return;
|
|
870
|
+
this.autoScaleRunning = true;
|
|
764
871
|
try {
|
|
765
|
-
const
|
|
766
|
-
const cooldownPeriod = this.poolOptions.autoScale?.cooldownPeriod || 60000;
|
|
872
|
+
const cooldownPeriod = this.poolOptions.autoScale?.cooldownPeriod ?? 60000;
|
|
767
873
|
if (now - this.lastScaleCheck < cooldownPeriod)
|
|
768
874
|
return;
|
|
769
875
|
const metrics = this.metrics;
|
|
770
876
|
const config = this.poolOptions.autoScale;
|
|
771
877
|
const queueThreshold = config.queueThreshold ?? 50;
|
|
772
|
-
const
|
|
773
|
-
|
|
774
|
-
|
|
878
|
+
const targetCpu = config.targetCPU ?? 70;
|
|
879
|
+
const targetMemory = config.targetMemory ?? 80;
|
|
880
|
+
const cpuOverTarget = metrics.cpuPercent !== undefined && metrics.cpuPercent > targetCpu;
|
|
881
|
+
const memoryOverTarget = metrics.memoryPercent !== undefined && metrics.memoryPercent > targetMemory;
|
|
882
|
+
const shouldScaleUp = cpuOverTarget ||
|
|
883
|
+
memoryOverTarget ||
|
|
884
|
+
(metrics.saturation ?? 0) > (config.scaleUpThreshold ?? 0.8) ||
|
|
775
885
|
this.queue.length > queueThreshold;
|
|
776
|
-
const scaleDownCpu =
|
|
777
|
-
const scaleDownMemory =
|
|
778
|
-
const
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
886
|
+
const scaleDownCpu = targetCpu * 0.4;
|
|
887
|
+
const scaleDownMemory = targetMemory * 0.5;
|
|
888
|
+
const cpuIdle = metrics.cpuPercent !== undefined && metrics.cpuPercent < scaleDownCpu;
|
|
889
|
+
const memoryIdle = metrics.memoryPercent !== undefined && metrics.memoryPercent < scaleDownMemory;
|
|
890
|
+
const shouldScaleDown = cpuIdle &&
|
|
891
|
+
memoryIdle &&
|
|
892
|
+
(metrics.saturation ?? 0) < (config.scaleDownThreshold ?? 0.3);
|
|
893
|
+
if (shouldScaleUp && this.workers.size < (config.max ?? 10)) {
|
|
782
894
|
const queuePressure = this.queue.length / queueThreshold;
|
|
783
895
|
const scaleStep = Math.max(1, Math.ceil(queuePressure));
|
|
784
896
|
const maxWorkers = config.max ?? 10;
|
|
785
897
|
const newSize = Math.min(this.workers.size + scaleStep, maxWorkers);
|
|
786
898
|
await this.scale(newSize);
|
|
787
899
|
this.lastScaleCheck = now;
|
|
900
|
+
this.autoScaleConsecutiveFailures = 0;
|
|
788
901
|
}
|
|
789
|
-
else if (shouldScaleDown && this.workers.size > (config.min
|
|
790
|
-
const newSize = Math.max(this.workers.size - 1, config.min
|
|
902
|
+
else if (shouldScaleDown && this.workers.size > (config.min ?? 1)) {
|
|
903
|
+
const newSize = Math.max(this.workers.size - 1, config.min ?? 1);
|
|
791
904
|
await this.scale(newSize);
|
|
792
905
|
this.lastScaleCheck = now;
|
|
906
|
+
this.autoScaleConsecutiveFailures = 0;
|
|
907
|
+
}
|
|
908
|
+
else {
|
|
909
|
+
this.autoScaleConsecutiveFailures = 0;
|
|
793
910
|
}
|
|
794
911
|
}
|
|
795
912
|
catch (error) {
|
|
796
|
-
this.
|
|
913
|
+
this.autoScaleConsecutiveFailures += 1;
|
|
914
|
+
const failures = this.autoScaleConsecutiveFailures;
|
|
915
|
+
const backoffMultiplier = Math.min(2 ** (failures - 1), Math.ceil(FAILURE_BACKOFF_MAX_MS / checkInterval));
|
|
916
|
+
const backoffMs = Math.min(checkInterval * backoffMultiplier, FAILURE_BACKOFF_MAX_MS);
|
|
917
|
+
this.autoScaleNextRetryAt = Date.now() + backoffMs;
|
|
918
|
+
this.logger.error({
|
|
919
|
+
err: error instanceof Error ? error : new Error(String(error)),
|
|
920
|
+
consecutiveFailures: failures,
|
|
921
|
+
nextRetryInMs: backoffMs,
|
|
922
|
+
}, 'Error during auto-scaling check');
|
|
923
|
+
}
|
|
924
|
+
finally {
|
|
925
|
+
this.autoScaleRunning = false;
|
|
797
926
|
}
|
|
798
927
|
}, checkInterval);
|
|
799
928
|
}
|
|
@@ -801,6 +930,9 @@ export class ProcessPool {
|
|
|
801
930
|
this.recycleTimer = setInterval(async () => {
|
|
802
931
|
if (this.isShuttingDown)
|
|
803
932
|
return;
|
|
933
|
+
if (this.recyclingRunning)
|
|
934
|
+
return;
|
|
935
|
+
this.recyclingRunning = true;
|
|
804
936
|
try {
|
|
805
937
|
const now = Date.now();
|
|
806
938
|
const maxLifetime = this.poolOptions.maxLifetime || 3600000;
|
|
@@ -834,6 +966,9 @@ export class ProcessPool {
|
|
|
834
966
|
catch (error) {
|
|
835
967
|
this.logger.error({ error }, 'Error during worker recycling');
|
|
836
968
|
}
|
|
969
|
+
finally {
|
|
970
|
+
this.recyclingRunning = false;
|
|
971
|
+
}
|
|
837
972
|
}, 60000);
|
|
838
973
|
}
|
|
839
974
|
setupMemoryMonitoring() {
|
|
@@ -841,42 +976,51 @@ export class ProcessPool {
|
|
|
841
976
|
this.memoryMonitorTimer = setInterval(async () => {
|
|
842
977
|
if (this.isShuttingDown)
|
|
843
978
|
return;
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
const
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
979
|
+
if (this.memoryMonitorRunning)
|
|
980
|
+
return;
|
|
981
|
+
this.memoryMonitorRunning = true;
|
|
982
|
+
try {
|
|
983
|
+
const workers = Array.from(this.workers.values());
|
|
984
|
+
await Promise.all(workers.map(async (worker) => {
|
|
985
|
+
try {
|
|
986
|
+
if ('__getMetrics' in worker.proxy) {
|
|
987
|
+
const metrics = await worker.proxy.__getMetrics();
|
|
988
|
+
const footprint = typeof metrics?.memoryRss === 'number' ? metrics.memoryRss : metrics?.memory;
|
|
989
|
+
if (typeof footprint === 'number') {
|
|
990
|
+
worker.memoryUsage = footprint;
|
|
991
|
+
worker.peakMemoryUsage = Math.max(worker.peakMemoryUsage, footprint);
|
|
992
|
+
const memoryRatio = footprint / this.memoryLimitBytes;
|
|
993
|
+
if (memoryRatio > 0.9) {
|
|
994
|
+
worker.weight = 10;
|
|
995
|
+
}
|
|
996
|
+
else if (memoryRatio > 0.7) {
|
|
997
|
+
worker.weight = 50;
|
|
998
|
+
}
|
|
999
|
+
else {
|
|
1000
|
+
worker.weight = 100;
|
|
1001
|
+
}
|
|
861
1002
|
}
|
|
862
1003
|
}
|
|
863
1004
|
}
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
}
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
})
|
|
879
|
-
}
|
|
1005
|
+
catch (error) {
|
|
1006
|
+
this.logger.debug({ error, workerId: worker.id }, 'Failed to collect memory metrics');
|
|
1007
|
+
}
|
|
1008
|
+
}));
|
|
1009
|
+
const totalMemory = workers.reduce((sum, w) => sum + w.memoryUsage, 0);
|
|
1010
|
+
const avgMemory = workers.length > 0 ? totalMemory / workers.length : 0;
|
|
1011
|
+
this.emit('pool:memory', {
|
|
1012
|
+
totalMemory,
|
|
1013
|
+
avgMemory,
|
|
1014
|
+
workers: workers.map((w) => ({
|
|
1015
|
+
id: w.id,
|
|
1016
|
+
memory: w.memoryUsage,
|
|
1017
|
+
peak: w.peakMemoryUsage,
|
|
1018
|
+
})),
|
|
1019
|
+
});
|
|
1020
|
+
}
|
|
1021
|
+
finally {
|
|
1022
|
+
this.memoryMonitorRunning = false;
|
|
1023
|
+
}
|
|
880
1024
|
}, MEMORY_CHECK_INTERVAL);
|
|
881
1025
|
}
|
|
882
1026
|
setupHeartbeatMonitoring() {
|
|
@@ -886,47 +1030,67 @@ export class ProcessPool {
|
|
|
886
1030
|
this.heartbeatTimer = setInterval(async () => {
|
|
887
1031
|
if (this.isShuttingDown)
|
|
888
1032
|
return;
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
1033
|
+
if (this.heartbeatRunning)
|
|
1034
|
+
return;
|
|
1035
|
+
this.heartbeatRunning = true;
|
|
1036
|
+
try {
|
|
1037
|
+
const now = Date.now();
|
|
1038
|
+
const workers = Array.from(this.workers.values());
|
|
1039
|
+
await Promise.all(workers.map(async (worker) => {
|
|
1040
|
+
let timeoutHandle = null;
|
|
1041
|
+
try {
|
|
1042
|
+
const heartbeatPromise = this.sendHeartbeat(worker);
|
|
1043
|
+
const timeoutPromise = new Promise((resolve) => {
|
|
1044
|
+
timeoutHandle = setTimeout(() => resolve(false), HEARTBEAT_TIMEOUT);
|
|
1045
|
+
});
|
|
1046
|
+
const responded = await Promise.race([heartbeatPromise, timeoutPromise]);
|
|
1047
|
+
if (timeoutHandle) {
|
|
1048
|
+
clearTimeout(timeoutHandle);
|
|
1049
|
+
timeoutHandle = null;
|
|
1050
|
+
}
|
|
1051
|
+
if (responded) {
|
|
1052
|
+
worker.lastHeartbeat = now;
|
|
1053
|
+
worker.consecutiveFailures = 0;
|
|
1054
|
+
}
|
|
1055
|
+
else {
|
|
1056
|
+
worker.consecutiveFailures++;
|
|
1057
|
+
if (worker.consecutiveFailures >= MAX_MISSED_HEARTBEATS) {
|
|
1058
|
+
this.logger.warn({
|
|
1059
|
+
workerId: worker.id,
|
|
1060
|
+
missedHeartbeats: worker.consecutiveFailures,
|
|
1061
|
+
}, 'Worker unresponsive - marking unhealthy');
|
|
1062
|
+
worker.health = 'unhealthy';
|
|
1063
|
+
this.unhealthyWorkers.add(worker.id);
|
|
1064
|
+
this.invalidateHealthyWorkersCache();
|
|
1065
|
+
this.emit('worker:unresponsive', { workerId: worker.id });
|
|
1066
|
+
if (!this.isShuttingDown) {
|
|
1067
|
+
await this.replaceWorker(worker.id);
|
|
1068
|
+
}
|
|
913
1069
|
}
|
|
914
1070
|
}
|
|
915
1071
|
}
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
1072
|
+
catch (error) {
|
|
1073
|
+
worker.consecutiveFailures++;
|
|
1074
|
+
this.logger.debug({ error, workerId: worker.id }, 'Heartbeat failed');
|
|
1075
|
+
}
|
|
1076
|
+
finally {
|
|
1077
|
+
if (timeoutHandle)
|
|
1078
|
+
clearTimeout(timeoutHandle);
|
|
1079
|
+
}
|
|
1080
|
+
}));
|
|
1081
|
+
}
|
|
1082
|
+
finally {
|
|
1083
|
+
this.heartbeatRunning = false;
|
|
1084
|
+
}
|
|
922
1085
|
}, HEARTBEAT_INTERVAL);
|
|
923
1086
|
}
|
|
924
1087
|
async sendHeartbeat(worker) {
|
|
925
1088
|
try {
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
return
|
|
1089
|
+
const getHealth = worker.proxy.__getHealth;
|
|
1090
|
+
if (typeof getHealth !== 'function') {
|
|
1091
|
+
return false;
|
|
929
1092
|
}
|
|
1093
|
+
await getHealth.call(worker.proxy);
|
|
930
1094
|
return true;
|
|
931
1095
|
}
|
|
932
1096
|
catch {
|
|
@@ -985,6 +1149,26 @@ export class ProcessPool {
|
|
|
985
1149
|
const duration = (Date.now() - firstEntry.timestamp) / 1000;
|
|
986
1150
|
return duration > 0 ? this.successfulRequests / duration : 0;
|
|
987
1151
|
}
|
|
1152
|
+
withDerivedPercentages(workerId, raw) {
|
|
1153
|
+
const now = Date.now();
|
|
1154
|
+
const derived = { ...raw };
|
|
1155
|
+
if (typeof raw.cpu === 'number') {
|
|
1156
|
+
const previous = this.cpuSamples.get(workerId);
|
|
1157
|
+
if (previous) {
|
|
1158
|
+
const elapsedSeconds = (now - previous.at) / 1000;
|
|
1159
|
+
const cpuSeconds = raw.cpu - previous.seconds;
|
|
1160
|
+
if (elapsedSeconds > 0 && cpuSeconds >= 0) {
|
|
1161
|
+
derived.cpuPercent = (cpuSeconds / elapsedSeconds) * 100;
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
this.cpuSamples.set(workerId, { seconds: raw.cpu, at: now });
|
|
1165
|
+
}
|
|
1166
|
+
const footprint = typeof raw.memoryRss === 'number' ? raw.memoryRss : raw.memory;
|
|
1167
|
+
if (typeof footprint === 'number' && this.memoryLimitBytes > 0) {
|
|
1168
|
+
derived.memoryPercent = (footprint / this.memoryLimitBytes) * 100;
|
|
1169
|
+
}
|
|
1170
|
+
return derived;
|
|
1171
|
+
}
|
|
988
1172
|
calculateSaturation() {
|
|
989
1173
|
const totalCapacity = this.workers.size * 2;
|
|
990
1174
|
const currentLoad = this.active + this.queue.length;
|
|
@@ -1003,8 +1187,12 @@ export class ProcessPool {
|
|
|
1003
1187
|
this.circuitBreaker.successAfterOpen = 0;
|
|
1004
1188
|
this.logger.warn('Circuit breaker opened');
|
|
1005
1189
|
this.emit('circuitbreaker:open');
|
|
1190
|
+
if (this.circuitBreakerHalfOpenTimer) {
|
|
1191
|
+
clearTimeout(this.circuitBreakerHalfOpenTimer);
|
|
1192
|
+
}
|
|
1006
1193
|
const timeout = this.poolOptions.circuitBreaker?.timeout || 60000;
|
|
1007
|
-
setTimeout(() => {
|
|
1194
|
+
this.circuitBreakerHalfOpenTimer = setTimeout(() => {
|
|
1195
|
+
this.circuitBreakerHalfOpenTimer = undefined;
|
|
1008
1196
|
if (this.circuitBreaker.isOpen) {
|
|
1009
1197
|
this.logger.info('Circuit breaker entering half-open state');
|
|
1010
1198
|
this.emit('circuitbreaker:halfopen');
|
|
@@ -1015,6 +1203,11 @@ export class ProcessPool {
|
|
|
1015
1203
|
this.circuitBreaker.isOpen = false;
|
|
1016
1204
|
this.circuitBreaker.failures = 0;
|
|
1017
1205
|
this.circuitBreaker.successAfterOpen = 0;
|
|
1206
|
+
this.circuitBreaker.lastFailTime = 0;
|
|
1207
|
+
if (this.circuitBreakerHalfOpenTimer) {
|
|
1208
|
+
clearTimeout(this.circuitBreakerHalfOpenTimer);
|
|
1209
|
+
this.circuitBreakerHalfOpenTimer = undefined;
|
|
1210
|
+
}
|
|
1018
1211
|
this.logger.info('Circuit breaker closed');
|
|
1019
1212
|
this.emit('circuitbreaker:close');
|
|
1020
1213
|
}
|