@omnitron-dev/titan-pm 0.1.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/backoff.d.ts +12 -0
  2. package/dist/backoff.d.ts.map +1 -0
  3. package/dist/backoff.js +44 -0
  4. package/dist/backoff.js.map +1 -0
  5. package/dist/child-contract.d.ts +58 -0
  6. package/dist/child-contract.d.ts.map +1 -0
  7. package/dist/child-contract.js +27 -0
  8. package/dist/child-contract.js.map +1 -0
  9. package/dist/confirm-death.d.ts +18 -0
  10. package/dist/confirm-death.d.ts.map +1 -0
  11. package/dist/confirm-death.js +49 -0
  12. package/dist/confirm-death.js.map +1 -0
  13. package/dist/decorators.d.ts +1 -10
  14. package/dist/decorators.d.ts.map +1 -1
  15. package/dist/decorators.js +43 -99
  16. package/dist/decorators.js.map +1 -1
  17. package/dist/discovery.d.ts +24 -0
  18. package/dist/discovery.d.ts.map +1 -0
  19. package/dist/discovery.js +162 -0
  20. package/dist/discovery.js.map +1 -0
  21. package/dist/fork-worker.js +64 -27
  22. package/dist/fork-worker.js.map +1 -1
  23. package/dist/index.d.ts +6 -1
  24. package/dist/index.d.ts.map +1 -1
  25. package/dist/index.js +5 -0
  26. package/dist/index.js.map +1 -1
  27. package/dist/last-resort-handlers.d.ts +5 -0
  28. package/dist/last-resort-handlers.d.ts.map +1 -0
  29. package/dist/last-resort-handlers.js +40 -0
  30. package/dist/last-resort-handlers.js.map +1 -0
  31. package/dist/lifecycle-queue.d.ts +20 -0
  32. package/dist/lifecycle-queue.d.ts.map +1 -0
  33. package/dist/lifecycle-queue.js +110 -0
  34. package/dist/lifecycle-queue.js.map +1 -0
  35. package/dist/liveness.d.ts +5 -0
  36. package/dist/liveness.d.ts.map +1 -0
  37. package/dist/liveness.js +34 -0
  38. package/dist/liveness.js.map +1 -0
  39. package/dist/netron-client.d.ts +10 -9
  40. package/dist/netron-client.d.ts.map +1 -1
  41. package/dist/netron-client.js +32 -21
  42. package/dist/netron-client.js.map +1 -1
  43. package/dist/parent-death-watchdog.d.ts +13 -0
  44. package/dist/parent-death-watchdog.d.ts.map +1 -0
  45. package/dist/parent-death-watchdog.js +58 -0
  46. package/dist/parent-death-watchdog.js.map +1 -0
  47. package/dist/pm.module.d.ts.map +1 -1
  48. package/dist/pm.module.js.map +1 -1
  49. package/dist/process-health.d.ts +2 -0
  50. package/dist/process-health.d.ts.map +1 -1
  51. package/dist/process-health.js +49 -13
  52. package/dist/process-health.js.map +1 -1
  53. package/dist/process-manager.d.ts +6 -0
  54. package/dist/process-manager.d.ts.map +1 -1
  55. package/dist/process-manager.js +111 -4
  56. package/dist/process-manager.js.map +1 -1
  57. package/dist/process-metrics.d.ts +1 -0
  58. package/dist/process-metrics.d.ts.map +1 -1
  59. package/dist/process-metrics.js +18 -19
  60. package/dist/process-metrics.js.map +1 -1
  61. package/dist/process-pool.d.ts +12 -0
  62. package/dist/process-pool.d.ts.map +1 -1
  63. package/dist/process-pool.js +340 -147
  64. package/dist/process-pool.js.map +1 -1
  65. package/dist/process-registry.d.ts.map +1 -1
  66. package/dist/process-spawner.d.ts +27 -5
  67. package/dist/process-spawner.d.ts.map +1 -1
  68. package/dist/process-spawner.js +397 -70
  69. package/dist/process-spawner.js.map +1 -1
  70. package/dist/process-supervisor.d.ts +7 -0
  71. package/dist/process-supervisor.d.ts.map +1 -1
  72. package/dist/process-supervisor.js +141 -17
  73. package/dist/process-supervisor.js.map +1 -1
  74. package/dist/process-workflow.d.ts.map +1 -1
  75. package/dist/process-workflow.js +11 -4
  76. package/dist/process-workflow.js.map +1 -1
  77. package/dist/service-proxy.d.ts.map +1 -1
  78. package/dist/service-proxy.js +33 -3
  79. package/dist/service-proxy.js.map +1 -1
  80. package/dist/shutdown-windows.d.ts +19 -0
  81. package/dist/shutdown-windows.d.ts.map +1 -0
  82. package/dist/shutdown-windows.js +35 -0
  83. package/dist/shutdown-windows.js.map +1 -0
  84. package/dist/thread-pool.d.ts +7 -0
  85. package/dist/thread-pool.d.ts.map +1 -0
  86. package/dist/thread-pool.js +11 -0
  87. package/dist/thread-pool.js.map +1 -0
  88. package/dist/types.d.ts +91 -37
  89. package/dist/types.d.ts.map +1 -1
  90. package/dist/types.js +35 -39
  91. package/dist/types.js.map +1 -1
  92. package/dist/utils/deque.d.ts.map +1 -1
  93. package/dist/worker-health.d.ts +8 -0
  94. package/dist/worker-health.d.ts.map +1 -0
  95. package/dist/worker-health.js +63 -0
  96. package/dist/worker-health.js.map +1 -0
  97. package/dist/worker-runtime.js +25 -52
  98. package/dist/worker-runtime.js.map +1 -1
  99. package/dist/worker-traffic.d.ts +8 -0
  100. package/dist/worker-traffic.d.ts.map +1 -0
  101. package/dist/worker-traffic.js +56 -0
  102. package/dist/worker-traffic.js.map +1 -0
  103. package/package.json +11 -10
@@ -2,6 +2,7 @@ import { cpus } from 'node:os';
2
2
  import path from 'node:path';
3
3
  import { EventEmitter } from '@omnitron-dev/eventemitter';
4
4
  import { Errors } from '@omnitron-dev/titan/errors';
5
+ import { FailureTracker, computeBackoff } from '@omnitron-dev/titan/utils';
5
6
  import { PoolStrategy } from './types.js';
6
7
  import { Deque } from './utils/index.js';
7
8
  export class PoolBackpressureError extends Error {
@@ -31,6 +32,8 @@ export class ProcessPool {
31
32
  this.lastScaleCheck = Date.now();
32
33
  this.scaleHistory = [];
33
34
  this.unhealthyWorkers = new Set();
35
+ this.healthFailures = new FailureTracker();
36
+ this.cpuSamples = new Map();
34
37
  this.queueBatchSize = 10;
35
38
  this.queueProcessingScheduled = false;
36
39
  this.circuitBreaker = {
@@ -42,6 +45,13 @@ export class ProcessPool {
42
45
  this.emitter = new EventEmitter();
43
46
  this.healthyWorkersCache = [];
44
47
  this.healthyWorkersCacheValid = false;
48
+ this.healthCheckRunning = false;
49
+ this.autoScaleRunning = false;
50
+ this.autoScaleConsecutiveFailures = 0;
51
+ this.autoScaleNextRetryAt = 0;
52
+ this.recyclingRunning = false;
53
+ this.memoryMonitorRunning = false;
54
+ this.heartbeatRunning = false;
45
55
  this.poolOptions = this.normalizeOptions(options);
46
56
  this.memoryLimitBytes = this.parseMemoryLimit(options.memoryLimit || '512MB');
47
57
  this.memoryWarningThreshold = options.memoryWarningThreshold || 0.8;
@@ -157,8 +167,8 @@ export class ProcessPool {
157
167
  const pmProcessId = proxy?.__processId;
158
168
  if (!pmProcessId)
159
169
  return null;
160
- const handle = this.manager.workers?.get(pmProcessId);
161
- return handle ?? null;
170
+ const handle = this.manager.getWorkerHandle?.(pmProcessId);
171
+ return (handle ?? null);
162
172
  }
163
173
  get metrics() {
164
174
  const avgResponseTime = this.totalRequests > 0 ? this.totalResponseTime / this.totalRequests : 0;
@@ -166,19 +176,35 @@ export class ProcessPool {
166
176
  let totalCpu = 0;
167
177
  let totalMemory = 0;
168
178
  let healthyWorkers = 0;
179
+ let totalCpuPercent = 0;
180
+ let cpuPercentSamples = 0;
181
+ let totalMemoryPercent = 0;
182
+ let memoryPercentSamples = 0;
169
183
  this.workers.forEach((worker) => {
170
184
  if (worker.metrics) {
171
185
  totalCpu += worker.metrics.cpu;
172
186
  totalMemory += worker.metrics.memory;
187
+ if (typeof worker.metrics.cpuPercent === 'number') {
188
+ totalCpuPercent += worker.metrics.cpuPercent;
189
+ cpuPercentSamples++;
190
+ }
191
+ if (typeof worker.metrics.memoryPercent === 'number') {
192
+ totalMemoryPercent += worker.metrics.memoryPercent;
193
+ memoryPercentSamples++;
194
+ }
173
195
  }
174
196
  if (worker.health === 'healthy')
175
197
  healthyWorkers++;
176
198
  });
177
199
  const avgCpu = this.workers.size > 0 ? totalCpu / this.workers.size : 0;
178
200
  const avgMemory = this.workers.size > 0 ? totalMemory / this.workers.size : 0;
201
+ const avgCpuPercent = cpuPercentSamples > 0 ? totalCpuPercent / cpuPercentSamples : undefined;
202
+ const avgMemoryPercent = memoryPercentSamples > 0 ? totalMemoryPercent / memoryPercentSamples : undefined;
179
203
  return {
180
204
  cpu: avgCpu,
181
205
  memory: avgMemory,
206
+ ...(avgCpuPercent !== undefined ? { cpuPercent: avgCpuPercent } : {}),
207
+ ...(avgMemoryPercent !== undefined ? { memoryPercent: avgMemoryPercent } : {}),
182
208
  queueSize: this.queue.length,
183
209
  activeWorkers: this.active,
184
210
  totalWorkers: this.workers.size,
@@ -278,6 +304,10 @@ export class ProcessPool {
278
304
  clearInterval(this.memoryMonitorTimer);
279
305
  this.memoryMonitorTimer = undefined;
280
306
  }
307
+ if (this.circuitBreakerHalfOpenTimer) {
308
+ clearTimeout(this.circuitBreakerHalfOpenTimer);
309
+ this.circuitBreakerHalfOpenTimer = undefined;
310
+ }
281
311
  await this.drain();
282
312
  const shutdownPromises = Array.from(this.workers.values()).map((worker) => this.shutdownWorker(worker.id));
283
313
  await Promise.all(shutdownPromises);
@@ -285,8 +315,15 @@ export class ProcessPool {
285
315
  this.isInitialized = false;
286
316
  this.isShuttingDown = false;
287
317
  this.isDraining = false;
288
- this.emit('pool:destroyed', { class: this.processName });
289
- this.emitter.removeAllListeners();
318
+ try {
319
+ this.emit('pool:destroyed', { class: this.processName });
320
+ }
321
+ catch (err) {
322
+ this.logger.warn({ err }, 'pool:destroyed subscriber threw — continuing teardown');
323
+ }
324
+ finally {
325
+ this.emitter.removeAllListeners();
326
+ }
290
327
  }
291
328
  async execute(method, ...args) {
292
329
  if (this.poolOptions.circuitBreaker?.enabled && this.circuitBreaker.isOpen) {
@@ -297,8 +334,7 @@ export class ProcessPool {
297
334
  if (this.isDraining) {
298
335
  throw Errors.conflict('Pool is draining, not accepting new requests');
299
336
  }
300
- if (this.workers.size === 0 ||
301
- (this.active >= this.workers.size && this.queue.length < (this.poolOptions.maxQueueSize || 100))) {
337
+ if (this.workers.size === 0 || this.active >= this.workers.size) {
302
338
  return this.queueRequest(method, args);
303
339
  }
304
340
  const worker = await this.selectWorker();
@@ -319,6 +355,9 @@ export class ProcessPool {
319
355
  : {};
320
356
  const proxy = await this.manager.spawn(this.processPathOrClass, {
321
357
  name: `${this.processName}-pool-${workerIndex}`,
358
+ ...(this.poolOptions.requestTimeout !== undefined && {
359
+ requestTimeout: this.poolOptions.requestTimeout,
360
+ }),
322
361
  ...baseOptions,
323
362
  ...factoryOptions,
324
363
  dependencies: {
@@ -365,11 +404,14 @@ export class ProcessPool {
365
404
  return;
366
405
  try {
367
406
  if (worker.processing.size > 0) {
368
- this.logger.debug({
369
- workerId,
370
- activeRequests: worker.processing.size,
371
- }, 'Waiting for worker to finish active requests');
372
- await new Promise((resolve) => setTimeout(resolve, 5000));
407
+ this.logger.debug({ workerId, activeRequests: worker.processing.size }, 'Waiting for worker to finish active requests');
408
+ const deadline = Date.now() + 5_000;
409
+ while (worker.processing.size > 0 && Date.now() < deadline) {
410
+ await new Promise((resolve) => setTimeout(resolve, 50));
411
+ }
412
+ if (worker.processing.size > 0) {
413
+ this.logger.warn({ workerId, activeRequests: worker.processing.size }, 'Worker still has active requests after grace period; killing anyway');
414
+ }
373
415
  }
374
416
  await this.manager.kill(workerId);
375
417
  this.workers.delete(workerId);
@@ -566,13 +608,17 @@ export class ProcessPool {
566
608
  worker.processing.add(requestId);
567
609
  worker.currentLoad++;
568
610
  this.totalRequests++;
611
+ let timeoutHandle = null;
569
612
  try {
570
613
  const timeout = this.poolOptions.requestTimeout;
571
614
  let result;
572
615
  if (timeout) {
616
+ const timeoutPromise = new Promise((_, reject) => {
617
+ timeoutHandle = setTimeout(() => reject(Errors.timeout('Request', timeout)), timeout);
618
+ });
573
619
  result = await Promise.race([
574
620
  worker.proxy[method](...args),
575
- new Promise((_, reject) => setTimeout(() => reject(Errors.timeout('Request', timeout)), timeout)),
621
+ timeoutPromise,
576
622
  ]);
577
623
  }
578
624
  else {
@@ -597,7 +643,7 @@ export class ProcessPool {
597
643
  if (this.poolOptions.circuitBreaker?.enabled) {
598
644
  this.handleCircuitBreakerFailure();
599
645
  }
600
- if (worker.errors > (this.poolOptions.healthCheck?.unhealthyThreshold || 3)) {
646
+ if (worker.errors > (this.poolOptions.healthCheck?.unhealthyThreshold ?? 3)) {
601
647
  worker.health = 'unhealthy';
602
648
  this.unhealthyWorkers.add(worker.id);
603
649
  this.invalidateHealthyWorkersCache();
@@ -606,13 +652,15 @@ export class ProcessPool {
606
652
  throw error;
607
653
  }
608
654
  finally {
655
+ if (timeoutHandle)
656
+ clearTimeout(timeoutHandle);
609
657
  worker.processing.delete(requestId);
610
- worker.currentLoad--;
658
+ worker.currentLoad = Math.max(0, worker.currentLoad - 1);
611
659
  worker.lastUsed = Date.now();
612
660
  }
613
661
  }
614
662
  async queueRequest(method, args) {
615
- if (this.queue.length >= (this.poolOptions.maxQueueSize || 100)) {
663
+ if (this.queue.length >= (this.poolOptions.maxQueueSize ?? 100)) {
616
664
  throw new PoolBackpressureError(this.queue.length);
617
665
  }
618
666
  return new Promise((resolve, reject) => {
@@ -639,7 +687,9 @@ export class ProcessPool {
639
687
  queueSize: this.queue.length,
640
688
  method,
641
689
  });
642
- this.processQueue();
690
+ void this.processQueue().catch((err) => {
691
+ this.logger.error({ err, class: this.processName }, 'Process pool queue processing failed');
692
+ });
643
693
  });
644
694
  }
645
695
  async processQueue() {
@@ -679,7 +729,9 @@ export class ProcessPool {
679
729
  processed++;
680
730
  }
681
731
  if (this.queue.length > 0) {
682
- this.processQueue();
732
+ void this.processQueue().catch((err) => {
733
+ this.logger.error({ err, class: this.processName }, 'Process pool queue processing failed');
734
+ });
683
735
  }
684
736
  }
685
737
  selectBestFromTwo(workers, idx1, idx2) {
@@ -689,68 +741,116 @@ export class ProcessPool {
689
741
  const score2 = worker2.currentLoad * 10 + (worker2.memoryUsage / this.memoryLimitBytes) * 5;
690
742
  return score1 <= score2 ? idx1 : idx2;
691
743
  }
744
+ emitHealthDecision(decision, workerId, error) {
745
+ if (!decision)
746
+ return;
747
+ const payload = {
748
+ workerId,
749
+ consecutiveFailures: decision.count,
750
+ elapsedMs: decision.elapsedMs,
751
+ ...(error !== undefined && { error }),
752
+ };
753
+ switch (decision.level) {
754
+ case 'first':
755
+ this.logger.warn(payload, 'Worker health check started failing');
756
+ break;
757
+ case 'continuing':
758
+ this.logger.debug(payload, 'Worker health check still failing');
759
+ break;
760
+ case 'persistent':
761
+ this.logger.error(payload, 'Worker health check persistent failure — worker will be replaced when threshold is reached');
762
+ break;
763
+ case 'suppress':
764
+ break;
765
+ case 'recovery':
766
+ this.logger.info(payload, 'Worker health check recovered');
767
+ break;
768
+ default: {
769
+ const unexpected = decision.level;
770
+ this.logger.warn({ ...payload, level: unexpected }, 'Unhandled failure-tracker level');
771
+ break;
772
+ }
773
+ }
774
+ }
692
775
  setupHealthMonitoring() {
693
776
  const baseInterval = this.poolOptions.healthCheck?.interval || 30000;
694
777
  const HEALTH_CHECK_CONCURRENCY = 10;
695
778
  const MAX_BACKOFF_MULTIPLIER = 8;
696
779
  this.healthCheckTimer = setInterval(async () => {
697
- let healthChanged = false;
698
- const workers = Array.from(this.workers.values());
699
- const now = Date.now();
700
- const checkWorkerHealth = async (worker) => {
701
- const previousHealth = worker.health;
702
- if (worker.consecutiveFailures > 0) {
703
- const backoffMultiplier = Math.min(Math.pow(2, worker.consecutiveFailures - 1), MAX_BACKOFF_MULTIPLIER);
704
- const nextCheckTime = worker.lastHeartbeat + baseInterval * backoffMultiplier;
705
- if (now < nextCheckTime) {
706
- return false;
780
+ if (this.healthCheckRunning)
781
+ return;
782
+ this.healthCheckRunning = true;
783
+ try {
784
+ let healthChanged = false;
785
+ const workers = Array.from(this.workers.values());
786
+ const now = Date.now();
787
+ const checkWorkerHealth = async (worker) => {
788
+ const previousHealth = worker.health;
789
+ if (worker.consecutiveFailures > 0) {
790
+ const backoffMultiplier = computeBackoff({
791
+ attempt: worker.consecutiveFailures - 1,
792
+ baseMs: 1,
793
+ maxMs: MAX_BACKOFF_MULTIPLIER,
794
+ });
795
+ const nextCheckTime = worker.lastHeartbeat + baseInterval * backoffMultiplier;
796
+ if (now < nextCheckTime) {
797
+ return false;
798
+ }
707
799
  }
708
- }
709
- try {
710
- if ('__getHealth' in worker.proxy) {
711
- const health = await worker.proxy.__getHealth();
712
- worker.health =
713
- health.status === 'healthy' ? 'healthy' : health.status === 'degraded' ? 'degraded' : 'unhealthy';
714
- if (worker.health === 'healthy') {
715
- worker.consecutiveFailures = 0;
800
+ try {
801
+ if ('__getHealth' in worker.proxy) {
802
+ const health = await worker.proxy.__getHealth();
803
+ worker.health =
804
+ health.status === 'healthy' ? 'healthy' : health.status === 'degraded' ? 'degraded' : 'unhealthy';
805
+ if (worker.health === 'healthy') {
806
+ worker.consecutiveFailures = 0;
807
+ this.emitHealthDecision(this.healthFailures.recordSuccess(worker.id), worker.id, undefined);
808
+ }
716
809
  }
717
810
  }
718
- if ('__getMetrics' in worker.proxy) {
719
- worker.metrics = await worker.proxy.__getMetrics();
811
+ catch (error) {
812
+ const decision = this.healthFailures.recordFailure(worker.id);
813
+ this.emitHealthDecision(decision, worker.id, error);
814
+ worker.health = 'unhealthy';
815
+ worker.consecutiveFailures++;
816
+ }
817
+ try {
818
+ const raw = (await worker.proxy
819
+ .__getMetrics?.());
820
+ if (raw) {
821
+ worker.metrics = this.withDerivedPercentages(worker.id, raw);
822
+ }
823
+ }
824
+ catch (error) {
825
+ this.logger.debug({ workerId: worker.id, error }, 'Worker metrics unavailable this tick');
826
+ }
827
+ return previousHealth !== worker.health;
828
+ };
829
+ for (let i = 0; i < workers.length; i += HEALTH_CHECK_CONCURRENCY) {
830
+ const batch = workers.slice(i, i + HEALTH_CHECK_CONCURRENCY);
831
+ const results = await Promise.all(batch.map(checkWorkerHealth));
832
+ if (results.some((changed) => changed)) {
833
+ healthChanged = true;
720
834
  }
721
835
  }
722
- catch (error) {
723
- this.logger.warn({
724
- error,
725
- workerId: worker.id,
726
- consecutiveFailures: worker.consecutiveFailures + 1,
727
- }, 'Health check failed');
728
- worker.health = 'unhealthy';
729
- worker.consecutiveFailures++;
836
+ if (healthChanged) {
837
+ this.invalidateHealthyWorkersCache();
730
838
  }
731
- return previousHealth !== worker.health;
732
- };
733
- for (let i = 0; i < workers.length; i += HEALTH_CHECK_CONCURRENCY) {
734
- const batch = workers.slice(i, i + HEALTH_CHECK_CONCURRENCY);
735
- const results = await Promise.all(batch.map(checkWorkerHealth));
736
- if (results.some((changed) => changed)) {
737
- healthChanged = true;
839
+ const unhealthyThreshold = this.poolOptions.healthCheck?.unhealthyThreshold ?? 3;
840
+ for (const worker of workers) {
841
+ if (worker.health === 'unhealthy' && worker.consecutiveFailures >= unhealthyThreshold && !this.isShuttingDown) {
842
+ this.logger.info({
843
+ workerId: worker.id,
844
+ consecutiveFailures: worker.consecutiveFailures,
845
+ restarts: worker.restarts,
846
+ }, 'Replacing unhealthy worker');
847
+ worker.restarts++;
848
+ await this.replaceWorker(worker.id);
849
+ }
738
850
  }
739
851
  }
740
- if (healthChanged) {
741
- this.invalidateHealthyWorkersCache();
742
- }
743
- const unhealthyThreshold = this.poolOptions.healthCheck?.unhealthyThreshold || 3;
744
- for (const worker of workers) {
745
- if (worker.health === 'unhealthy' && worker.consecutiveFailures >= unhealthyThreshold && !this.isShuttingDown) {
746
- this.logger.info({
747
- workerId: worker.id,
748
- consecutiveFailures: worker.consecutiveFailures,
749
- restarts: worker.restarts,
750
- }, 'Replacing unhealthy worker');
751
- worker.restarts++;
752
- await this.replaceWorker(worker.id);
753
- }
852
+ finally {
853
+ this.healthCheckRunning = false;
754
854
  }
755
855
  }, baseInterval);
756
856
  }
@@ -758,42 +858,71 @@ export class ProcessPool {
758
858
  if (!this.poolOptions.autoScale?.enabled)
759
859
  return;
760
860
  const checkInterval = this.poolOptions.autoScale?.checkInterval ?? 10_000;
861
+ const FAILURE_BACKOFF_MAX_MS = 5 * 60_000;
761
862
  this.autoScaleTimer = setInterval(async () => {
762
863
  if (this.isShuttingDown || this.isDraining)
763
864
  return;
865
+ if (this.autoScaleRunning)
866
+ return;
867
+ const now = Date.now();
868
+ if (now < this.autoScaleNextRetryAt)
869
+ return;
870
+ this.autoScaleRunning = true;
764
871
  try {
765
- const now = Date.now();
766
- const cooldownPeriod = this.poolOptions.autoScale?.cooldownPeriod || 60000;
872
+ const cooldownPeriod = this.poolOptions.autoScale?.cooldownPeriod ?? 60000;
767
873
  if (now - this.lastScaleCheck < cooldownPeriod)
768
874
  return;
769
875
  const metrics = this.metrics;
770
876
  const config = this.poolOptions.autoScale;
771
877
  const queueThreshold = config.queueThreshold ?? 50;
772
- const shouldScaleUp = metrics.cpu > (config.targetCPU || 70) ||
773
- metrics.memory > (config.targetMemory || 80) ||
774
- (metrics.saturation || 0) > (config.scaleUpThreshold || 0.8) ||
878
+ const targetCpu = config.targetCPU ?? 70;
879
+ const targetMemory = config.targetMemory ?? 80;
880
+ const cpuOverTarget = metrics.cpuPercent !== undefined && metrics.cpuPercent > targetCpu;
881
+ const memoryOverTarget = metrics.memoryPercent !== undefined && metrics.memoryPercent > targetMemory;
882
+ const shouldScaleUp = cpuOverTarget ||
883
+ memoryOverTarget ||
884
+ (metrics.saturation ?? 0) > (config.scaleUpThreshold ?? 0.8) ||
775
885
  this.queue.length > queueThreshold;
776
- const scaleDownCpu = (config.targetCPU ?? 70) * 0.4;
777
- const scaleDownMemory = (config.targetMemory ?? 80) * 0.5;
778
- const shouldScaleDown = metrics.cpu < scaleDownCpu &&
779
- metrics.memory < scaleDownMemory &&
780
- (metrics.saturation || 0) < (config.scaleDownThreshold || 0.3);
781
- if (shouldScaleUp && this.workers.size < (config.max || 10)) {
886
+ const scaleDownCpu = targetCpu * 0.4;
887
+ const scaleDownMemory = targetMemory * 0.5;
888
+ const cpuIdle = metrics.cpuPercent !== undefined && metrics.cpuPercent < scaleDownCpu;
889
+ const memoryIdle = metrics.memoryPercent !== undefined && metrics.memoryPercent < scaleDownMemory;
890
+ const shouldScaleDown = cpuIdle &&
891
+ memoryIdle &&
892
+ (metrics.saturation ?? 0) < (config.scaleDownThreshold ?? 0.3);
893
+ if (shouldScaleUp && this.workers.size < (config.max ?? 10)) {
782
894
  const queuePressure = this.queue.length / queueThreshold;
783
895
  const scaleStep = Math.max(1, Math.ceil(queuePressure));
784
896
  const maxWorkers = config.max ?? 10;
785
897
  const newSize = Math.min(this.workers.size + scaleStep, maxWorkers);
786
898
  await this.scale(newSize);
787
899
  this.lastScaleCheck = now;
900
+ this.autoScaleConsecutiveFailures = 0;
788
901
  }
789
- else if (shouldScaleDown && this.workers.size > (config.min || 1)) {
790
- const newSize = Math.max(this.workers.size - 1, config.min || 1);
902
+ else if (shouldScaleDown && this.workers.size > (config.min ?? 1)) {
903
+ const newSize = Math.max(this.workers.size - 1, config.min ?? 1);
791
904
  await this.scale(newSize);
792
905
  this.lastScaleCheck = now;
906
+ this.autoScaleConsecutiveFailures = 0;
907
+ }
908
+ else {
909
+ this.autoScaleConsecutiveFailures = 0;
793
910
  }
794
911
  }
795
912
  catch (error) {
796
- this.logger.error({ err: error instanceof Error ? error : new Error(String(error)) }, 'Error during auto-scaling check');
913
+ this.autoScaleConsecutiveFailures += 1;
914
+ const failures = this.autoScaleConsecutiveFailures;
915
+ const backoffMultiplier = Math.min(2 ** (failures - 1), Math.ceil(FAILURE_BACKOFF_MAX_MS / checkInterval));
916
+ const backoffMs = Math.min(checkInterval * backoffMultiplier, FAILURE_BACKOFF_MAX_MS);
917
+ this.autoScaleNextRetryAt = Date.now() + backoffMs;
918
+ this.logger.error({
919
+ err: error instanceof Error ? error : new Error(String(error)),
920
+ consecutiveFailures: failures,
921
+ nextRetryInMs: backoffMs,
922
+ }, 'Error during auto-scaling check');
923
+ }
924
+ finally {
925
+ this.autoScaleRunning = false;
797
926
  }
798
927
  }, checkInterval);
799
928
  }
@@ -801,6 +930,9 @@ export class ProcessPool {
801
930
  this.recycleTimer = setInterval(async () => {
802
931
  if (this.isShuttingDown)
803
932
  return;
933
+ if (this.recyclingRunning)
934
+ return;
935
+ this.recyclingRunning = true;
804
936
  try {
805
937
  const now = Date.now();
806
938
  const maxLifetime = this.poolOptions.maxLifetime || 3600000;
@@ -834,6 +966,9 @@ export class ProcessPool {
834
966
  catch (error) {
835
967
  this.logger.error({ error }, 'Error during worker recycling');
836
968
  }
969
+ finally {
970
+ this.recyclingRunning = false;
971
+ }
837
972
  }, 60000);
838
973
  }
839
974
  setupMemoryMonitoring() {
@@ -841,42 +976,51 @@ export class ProcessPool {
841
976
  this.memoryMonitorTimer = setInterval(async () => {
842
977
  if (this.isShuttingDown)
843
978
  return;
844
- const workers = Array.from(this.workers.values());
845
- await Promise.all(workers.map(async (worker) => {
846
- try {
847
- if ('__getMetrics' in worker.proxy) {
848
- const metrics = await worker.proxy.__getMetrics();
849
- if (metrics && typeof metrics.memory === 'number') {
850
- worker.memoryUsage = metrics.memory;
851
- worker.peakMemoryUsage = Math.max(worker.peakMemoryUsage, metrics.memory);
852
- const memoryRatio = metrics.memory / this.memoryLimitBytes;
853
- if (memoryRatio > 0.9) {
854
- worker.weight = 10;
855
- }
856
- else if (memoryRatio > 0.7) {
857
- worker.weight = 50;
858
- }
859
- else {
860
- worker.weight = 100;
979
+ if (this.memoryMonitorRunning)
980
+ return;
981
+ this.memoryMonitorRunning = true;
982
+ try {
983
+ const workers = Array.from(this.workers.values());
984
+ await Promise.all(workers.map(async (worker) => {
985
+ try {
986
+ if ('__getMetrics' in worker.proxy) {
987
+ const metrics = await worker.proxy.__getMetrics();
988
+ const footprint = typeof metrics?.memoryRss === 'number' ? metrics.memoryRss : metrics?.memory;
989
+ if (typeof footprint === 'number') {
990
+ worker.memoryUsage = footprint;
991
+ worker.peakMemoryUsage = Math.max(worker.peakMemoryUsage, footprint);
992
+ const memoryRatio = footprint / this.memoryLimitBytes;
993
+ if (memoryRatio > 0.9) {
994
+ worker.weight = 10;
995
+ }
996
+ else if (memoryRatio > 0.7) {
997
+ worker.weight = 50;
998
+ }
999
+ else {
1000
+ worker.weight = 100;
1001
+ }
861
1002
  }
862
1003
  }
863
1004
  }
864
- }
865
- catch (error) {
866
- this.logger.debug({ error, workerId: worker.id }, 'Failed to collect memory metrics');
867
- }
868
- }));
869
- const totalMemory = workers.reduce((sum, w) => sum + w.memoryUsage, 0);
870
- const avgMemory = workers.length > 0 ? totalMemory / workers.length : 0;
871
- this.emit('pool:memory', {
872
- totalMemory,
873
- avgMemory,
874
- workers: workers.map((w) => ({
875
- id: w.id,
876
- memory: w.memoryUsage,
877
- peak: w.peakMemoryUsage,
878
- })),
879
- });
1005
+ catch (error) {
1006
+ this.logger.debug({ error, workerId: worker.id }, 'Failed to collect memory metrics');
1007
+ }
1008
+ }));
1009
+ const totalMemory = workers.reduce((sum, w) => sum + w.memoryUsage, 0);
1010
+ const avgMemory = workers.length > 0 ? totalMemory / workers.length : 0;
1011
+ this.emit('pool:memory', {
1012
+ totalMemory,
1013
+ avgMemory,
1014
+ workers: workers.map((w) => ({
1015
+ id: w.id,
1016
+ memory: w.memoryUsage,
1017
+ peak: w.peakMemoryUsage,
1018
+ })),
1019
+ });
1020
+ }
1021
+ finally {
1022
+ this.memoryMonitorRunning = false;
1023
+ }
880
1024
  }, MEMORY_CHECK_INTERVAL);
881
1025
  }
882
1026
  setupHeartbeatMonitoring() {
@@ -886,47 +1030,67 @@ export class ProcessPool {
886
1030
  this.heartbeatTimer = setInterval(async () => {
887
1031
  if (this.isShuttingDown)
888
1032
  return;
889
- const now = Date.now();
890
- const workers = Array.from(this.workers.values());
891
- await Promise.all(workers.map(async (worker) => {
892
- try {
893
- const heartbeatPromise = this.sendHeartbeat(worker);
894
- const timeoutPromise = new Promise((resolve) => setTimeout(() => resolve(false), HEARTBEAT_TIMEOUT));
895
- const responded = await Promise.race([heartbeatPromise, timeoutPromise]);
896
- if (responded) {
897
- worker.lastHeartbeat = now;
898
- worker.consecutiveFailures = 0;
899
- }
900
- else {
901
- worker.consecutiveFailures++;
902
- if (worker.consecutiveFailures >= MAX_MISSED_HEARTBEATS) {
903
- this.logger.warn({
904
- workerId: worker.id,
905
- missedHeartbeats: worker.consecutiveFailures,
906
- }, 'Worker unresponsive - marking unhealthy');
907
- worker.health = 'unhealthy';
908
- this.unhealthyWorkers.add(worker.id);
909
- this.invalidateHealthyWorkersCache();
910
- this.emit('worker:unresponsive', { workerId: worker.id });
911
- if (!this.isShuttingDown) {
912
- await this.replaceWorker(worker.id);
1033
+ if (this.heartbeatRunning)
1034
+ return;
1035
+ this.heartbeatRunning = true;
1036
+ try {
1037
+ const now = Date.now();
1038
+ const workers = Array.from(this.workers.values());
1039
+ await Promise.all(workers.map(async (worker) => {
1040
+ let timeoutHandle = null;
1041
+ try {
1042
+ const heartbeatPromise = this.sendHeartbeat(worker);
1043
+ const timeoutPromise = new Promise((resolve) => {
1044
+ timeoutHandle = setTimeout(() => resolve(false), HEARTBEAT_TIMEOUT);
1045
+ });
1046
+ const responded = await Promise.race([heartbeatPromise, timeoutPromise]);
1047
+ if (timeoutHandle) {
1048
+ clearTimeout(timeoutHandle);
1049
+ timeoutHandle = null;
1050
+ }
1051
+ if (responded) {
1052
+ worker.lastHeartbeat = now;
1053
+ worker.consecutiveFailures = 0;
1054
+ }
1055
+ else {
1056
+ worker.consecutiveFailures++;
1057
+ if (worker.consecutiveFailures >= MAX_MISSED_HEARTBEATS) {
1058
+ this.logger.warn({
1059
+ workerId: worker.id,
1060
+ missedHeartbeats: worker.consecutiveFailures,
1061
+ }, 'Worker unresponsive - marking unhealthy');
1062
+ worker.health = 'unhealthy';
1063
+ this.unhealthyWorkers.add(worker.id);
1064
+ this.invalidateHealthyWorkersCache();
1065
+ this.emit('worker:unresponsive', { workerId: worker.id });
1066
+ if (!this.isShuttingDown) {
1067
+ await this.replaceWorker(worker.id);
1068
+ }
913
1069
  }
914
1070
  }
915
1071
  }
916
- }
917
- catch (error) {
918
- worker.consecutiveFailures++;
919
- this.logger.debug({ error, workerId: worker.id }, 'Heartbeat failed');
920
- }
921
- }));
1072
+ catch (error) {
1073
+ worker.consecutiveFailures++;
1074
+ this.logger.debug({ error, workerId: worker.id }, 'Heartbeat failed');
1075
+ }
1076
+ finally {
1077
+ if (timeoutHandle)
1078
+ clearTimeout(timeoutHandle);
1079
+ }
1080
+ }));
1081
+ }
1082
+ finally {
1083
+ this.heartbeatRunning = false;
1084
+ }
922
1085
  }, HEARTBEAT_INTERVAL);
923
1086
  }
924
1087
  async sendHeartbeat(worker) {
925
1088
  try {
926
- if ('__getHealth' in worker.proxy) {
927
- await worker.proxy.__getHealth();
928
- return true;
1089
+ const getHealth = worker.proxy.__getHealth;
1090
+ if (typeof getHealth !== 'function') {
1091
+ return false;
929
1092
  }
1093
+ await getHealth.call(worker.proxy);
930
1094
  return true;
931
1095
  }
932
1096
  catch {
@@ -985,6 +1149,26 @@ export class ProcessPool {
985
1149
  const duration = (Date.now() - firstEntry.timestamp) / 1000;
986
1150
  return duration > 0 ? this.successfulRequests / duration : 0;
987
1151
  }
1152
+ withDerivedPercentages(workerId, raw) {
1153
+ const now = Date.now();
1154
+ const derived = { ...raw };
1155
+ if (typeof raw.cpu === 'number') {
1156
+ const previous = this.cpuSamples.get(workerId);
1157
+ if (previous) {
1158
+ const elapsedSeconds = (now - previous.at) / 1000;
1159
+ const cpuSeconds = raw.cpu - previous.seconds;
1160
+ if (elapsedSeconds > 0 && cpuSeconds >= 0) {
1161
+ derived.cpuPercent = (cpuSeconds / elapsedSeconds) * 100;
1162
+ }
1163
+ }
1164
+ this.cpuSamples.set(workerId, { seconds: raw.cpu, at: now });
1165
+ }
1166
+ const footprint = typeof raw.memoryRss === 'number' ? raw.memoryRss : raw.memory;
1167
+ if (typeof footprint === 'number' && this.memoryLimitBytes > 0) {
1168
+ derived.memoryPercent = (footprint / this.memoryLimitBytes) * 100;
1169
+ }
1170
+ return derived;
1171
+ }
988
1172
  calculateSaturation() {
989
1173
  const totalCapacity = this.workers.size * 2;
990
1174
  const currentLoad = this.active + this.queue.length;
@@ -1003,8 +1187,12 @@ export class ProcessPool {
1003
1187
  this.circuitBreaker.successAfterOpen = 0;
1004
1188
  this.logger.warn('Circuit breaker opened');
1005
1189
  this.emit('circuitbreaker:open');
1190
+ if (this.circuitBreakerHalfOpenTimer) {
1191
+ clearTimeout(this.circuitBreakerHalfOpenTimer);
1192
+ }
1006
1193
  const timeout = this.poolOptions.circuitBreaker?.timeout || 60000;
1007
- setTimeout(() => {
1194
+ this.circuitBreakerHalfOpenTimer = setTimeout(() => {
1195
+ this.circuitBreakerHalfOpenTimer = undefined;
1008
1196
  if (this.circuitBreaker.isOpen) {
1009
1197
  this.logger.info('Circuit breaker entering half-open state');
1010
1198
  this.emit('circuitbreaker:halfopen');
@@ -1015,6 +1203,11 @@ export class ProcessPool {
1015
1203
  this.circuitBreaker.isOpen = false;
1016
1204
  this.circuitBreaker.failures = 0;
1017
1205
  this.circuitBreaker.successAfterOpen = 0;
1206
+ this.circuitBreaker.lastFailTime = 0;
1207
+ if (this.circuitBreakerHalfOpenTimer) {
1208
+ clearTimeout(this.circuitBreakerHalfOpenTimer);
1209
+ this.circuitBreakerHalfOpenTimer = undefined;
1210
+ }
1018
1211
  this.logger.info('Circuit breaker closed');
1019
1212
  this.emit('circuitbreaker:close');
1020
1213
  }