@omnitron-dev/titan-pm 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fork-worker.js +27 -6
- package/dist/fork-worker.js.map +1 -1
- package/dist/netron-client.d.ts +0 -1
- package/dist/netron-client.d.ts.map +1 -1
- package/dist/netron-client.js +16 -11
- package/dist/netron-client.js.map +1 -1
- package/dist/process-health.d.ts +2 -0
- package/dist/process-health.d.ts.map +1 -1
- package/dist/process-health.js +29 -9
- package/dist/process-health.js.map +1 -1
- package/dist/process-manager.d.ts +2 -0
- package/dist/process-manager.d.ts.map +1 -1
- package/dist/process-manager.js +60 -0
- package/dist/process-manager.js.map +1 -1
- package/dist/process-metrics.d.ts +1 -0
- package/dist/process-metrics.d.ts.map +1 -1
- package/dist/process-metrics.js +8 -0
- package/dist/process-metrics.js.map +1 -1
- package/dist/process-pool.d.ts +8 -0
- package/dist/process-pool.d.ts.map +1 -1
- package/dist/process-pool.js +236 -126
- package/dist/process-pool.js.map +1 -1
- package/dist/process-spawner.d.ts +9 -3
- package/dist/process-spawner.d.ts.map +1 -1
- package/dist/process-spawner.js +130 -30
- package/dist/process-spawner.js.map +1 -1
- package/dist/process-supervisor.d.ts +6 -0
- package/dist/process-supervisor.d.ts.map +1 -1
- package/dist/process-supervisor.js +123 -13
- package/dist/process-supervisor.js.map +1 -1
- package/dist/process-workflow.d.ts.map +1 -1
- package/dist/process-workflow.js +11 -4
- package/dist/process-workflow.js.map +1 -1
- package/dist/types.d.ts +12 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/worker-runtime.js +13 -4
- package/dist/worker-runtime.js.map +1 -1
- package/package.json +7 -7
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"process-metrics.js","sourceRoot":"","sources":["../src/process-metrics.ts"],"names":[],"mappings":"AAYA,MAAM,OAAO,uBAAuB;
|
|
1
|
+
{"version":3,"file":"process-metrics.js","sourceRoot":"","sources":["../src/process-metrics.ts"],"names":[],"mappings":"AAYA,MAAM,OAAO,uBAAuB;IAYlC,YAA6B,MAAe;QAAf,WAAM,GAAN,MAAM,CAAS;QAXpC,eAAU,GAAG,IAAI,GAAG,EAA0B,CAAC;QAC/C,mBAAc,GAAG,IAAI,GAAG,EAA6B,CAAC;QACtD,oBAAe,GAAG,IAAI,GAAG,EAAoB,CAAC;QAO9C,sBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;IAEC,CAAC;IAKhD,eAAe,CAAC,SAAiB,EAAE,KAAwB,EAAE,WAAmB,IAAI;QAClF,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACnC,OAAO;QACT,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,EAAE,6BAA6B,CAAC,CAAC;QAGhE,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACvC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAGxC,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YACvC,IAAI,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC;gBAAE,OAAO;YAClD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACtC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;gBACjD,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACxC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,2BAA2B,CAAC,CAAC;YACvE,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC,EAAE,QAAQ,CAAC,CAAC;QAEb,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IAC5C,CAAC;IAKD,cAAc,CAAC,SAAiB;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,SAAS,EAAE,CAAC;YACd,aAAa,CAAC,SAAS,CAAC,CAAC;YACzB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAClC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACtC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACvC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAEzC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,EAAE,4BAA4B,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAKD,UAAU,CAAC,SAAiB;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACnD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC;IAC7C,CAAC;IAKD,iBAAiB,CAAC,SAAiB;QACjC,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;IAClD,CAAC;IAKD,oBAAoB,CAAC,SAAiB,EAAE,aAAqB,KAAK;QAChE,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACnD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAS,CAAC,SAAS,IAAI,GAAG,GAAI,CAAS,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC;QAE7G,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;IAKD,YAAY,CAAC,SAAiB,EAAE,QAAgB;QAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QAC5D,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAGzB,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YAC5B,SAAS,CAAC,KAAK,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACjD,CAAC;IASO,KAAK,CAAC,cAAc,CAAC,KAAwB;QACnD,IAAI,CAAC;YAEH,IAAI,cAAc,IAAI,KAAK,EAAE,CAAC;gBAC5B,OAAO,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;YACpC,CAAC;YAGD,OAAO;gBACL,GAAG,EAAE,CAAC;gBACN,MAAM,EAAE,CAAC;gBACT,QAAQ,EAAE,CAAC;gBACX,MAAM,EAAE,CAAC;aACV,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,mCAAmC,CAAC,CAAC;YAElE,OAAO;gBACL,GAAG,EAAE,CAAC;gBACN,MAAM,EAAE,CAAC;gBACT,QAAQ,EAAE,CAAC;gBACX,MAAM,EAAE,CAAC;aACV,CAAC;QACJ,CAAC;IACH,CAAC;IAKO,YAAY,CAAC,SAAiB,EAAE,OAAwB;QAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QAGzD,MAAM,kBAAkB,GAAG;YACzB,GAAG,OAAO;YACV,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QAEF,OAAO,CAAC,IAAI,CAAC,kBAAqC,CAAC,CAAC;QAGpD,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAG5C,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACtD,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtC,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAKO,uBAAuB,CAAC,SAAmB;QACjD,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpD,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;QAE1B,OAAO;YACL,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAChC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAChC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAChC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAChC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAChC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG;SAC9C,CAAC;IACJ,CAAC;IAKO,UAAU,CAAC,MAAgB,EAAE,CAAS;QAC5C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAElC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACvD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC;IAKO,gBAAgB,CAAC,OAA0B;QACjD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO;gBACL,GAAG,EAAE,CAAC;gBACN,MAAM,EAAE,CAAC;gBACT,QAAQ,EAAE,CAAC;gBACX,MAAM,EAAE,CAAC;aACV,CAAC;QACJ,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CACxB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACX,GAAG,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;YACpB,MAAM,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM;YAC7B,QAAQ,EAAE,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC;YACjD,MAAM,EAAE,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;SAC5C,CAAC,EACF;YACE,GAAG,EAAE,CAAC;YACN,MAAM,EAAE,CAAC;YACT,QAAQ,EAAE,CAAC;YACX,MAAM,EAAE,CAAC;SACV,CACF,CAAC;QAEF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;QAG7B,IAAI,OAAoC,CAAC;QACzC,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAQ,CAAC,CAAC;QAE/E,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,OAAO,GAAG;gBACR,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACnD,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACnD,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACnD,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACnD,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACnD,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;aACtD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,GAAG,EAAE,GAAG,CAAC,GAAG,GAAG,KAAK;YACpB,MAAM,EAAE,GAAG,CAAC,MAAM,GAAG,KAAK;YAC1B,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,OAAO;SACR,CAAC;IACJ,CAAC;IAKO,OAAO,CAAC,OAAiB;QAC/B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACnC,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAC7D,CAAC;CACF"}
|
package/dist/process-pool.d.ts
CHANGED
|
@@ -28,6 +28,7 @@ export declare class ProcessPool<T> {
|
|
|
28
28
|
private heartbeatTimer?;
|
|
29
29
|
private memoryMonitorTimer?;
|
|
30
30
|
private unhealthyWorkers;
|
|
31
|
+
private readonly healthFailures;
|
|
31
32
|
private readonly memoryLimitBytes;
|
|
32
33
|
private readonly memoryWarningThreshold;
|
|
33
34
|
private readonly queueBatchSize;
|
|
@@ -83,10 +84,16 @@ export declare class ProcessPool<T> {
|
|
|
83
84
|
private processQueue;
|
|
84
85
|
private processQueueBatch;
|
|
85
86
|
private selectBestFromTwo;
|
|
87
|
+
private emitHealthDecision;
|
|
88
|
+
private healthCheckRunning;
|
|
86
89
|
private setupHealthMonitoring;
|
|
90
|
+
private autoScaleRunning;
|
|
87
91
|
private setupAutoScaling;
|
|
92
|
+
private recyclingRunning;
|
|
88
93
|
private setupRecycling;
|
|
94
|
+
private memoryMonitorRunning;
|
|
89
95
|
private setupMemoryMonitoring;
|
|
96
|
+
private heartbeatRunning;
|
|
90
97
|
private setupHeartbeatMonitoring;
|
|
91
98
|
private sendHeartbeat;
|
|
92
99
|
private replaceWorker;
|
|
@@ -97,6 +104,7 @@ export declare class ProcessPool<T> {
|
|
|
97
104
|
private calculateThroughput;
|
|
98
105
|
private calculateSaturation;
|
|
99
106
|
private handleCircuitBreakerFailure;
|
|
107
|
+
private circuitBreakerHalfOpenTimer?;
|
|
100
108
|
private openCircuitBreaker;
|
|
101
109
|
private closeCircuitBreaker;
|
|
102
110
|
private shouldTryHalfOpen;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"process-pool.d.ts","sourceRoot":"","sources":["../src/process-pool.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"process-pool.d.ts","sourceRoot":"","sources":["../src/process-pool.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AAEjE,OAAO,KAAK,EAAE,mBAAmB,EAAgB,YAAY,EAAE,eAAe,EAAmB,MAAM,YAAY,CAAC;AAOpH,qBAAa,qBAAsB,SAAQ,KAAK;IAC9C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAClB,SAAS,EAAE,MAAM,EAAE,YAAY,SAAO;CAKnD;AA8DD,qBAAa,WAAW,CAAC,CAAC;IAiEtB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IAEnC,OAAO,CAAC,QAAQ,CAAC,MAAM;IAnEzB,OAAO,CAAC,OAAO,CAAoC;IACnD,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAK;IACnC,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,kBAAkB,CAAK;IAG/B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,kBAAkB,CAAK;IAG/B,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,cAAc,CAAc;IACpC,OAAO,CAAC,YAAY,CAAqF;IAGzG,OAAO,CAAC,gBAAgB,CAAC,CAAiB;IAC1C,OAAO,CAAC,YAAY,CAAC,CAAiB;IACtC,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,kBAAkB,CAAC,CAAiB;IAC5C,OAAO,CAAC,gBAAgB,CAAqB;IAU7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAwB;IAGvD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAGhD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAc;IAC7C,OAAO,CAAC,wBAAwB,CAAS;IAGzC,OAAO,CAAC,cAAc,CAKpB;IAGF,OAAO,CAAC,OAAO,CAAsB;IAGrC,OAAO,CAAC,WAAW,CAAsB;IAGzC,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,wBAAwB,CAAS;gBAGtB,OAAO,EAAE,eAAe,EACxB,kBAAkB,EAAE,MAAM,GAAG,CAAC,KAAK,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,EACzE,OAAO,EAAE,mBAAmB,EACX,MAAM,EAAE,OAAO;IAalC,OAAO,CAAC,gBAAgB;IAsBxB,OAAO,KAAK,WAAW,GAItB;IAKD,OAAO,CAAC,gBAAgB;IAuClB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAmDjC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAKD,IAAI,MAAM,IAAI,MAAM,CAMnB;IAKD,IAAI,OAAO,IAAI,MAAM,CAEpB;IAKD,YAAY,IAAI,MAAM,EAAE;IAcxB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KAAE,GAAG,IAAI;IAavF,IAAI,OAAO,IAAI,YAAY,CAsC1B;IAKD,IAAI,mBAAmB,IAAI;QACzB,KAAK,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;QACvC,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,OAAO,CAAC;KACpB,CAcA;IAKK,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgDrC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBtB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAkFxB,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC;IAyD3D,OAAO,CAAC,WAAW;YAUL,WAAW;YAgEX,cAAc;IA8C5B,OAAO,CAAC,6BAA6B;IAOrC,OAAO,CAAC,iBAAiB;YAWX,YAAY;IA8D1B,OAAO,CAAC,gBAAgB;IAYxB,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,cAAc;IAyBtB,OAAO,CAAC,kBAAkB;IAU1B,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,eAAe;IAavB,OAAO,CAAC,sBAAsB;IAsB9B,OAAO,CAAC,wBAAwB;IAoChC,OAAO,CAAC,uBAAuB;IA8B/B,OAAO,CAAC,eAAe;YA2BT,eAAe;YAiFf,YAAY;YA2CZ,YAAY;IAkB1B,OAAO,CAAC,iBAAiB;IAoDzB,OAAO,CAAC,iBAAiB;IAiBzB,OAAO,CAAC,kBAAkB;IA6C1B,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,qBAAqB;IAoH7B,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,gBAAgB;IAiExB,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,cAAc;IAkEtB,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,qBAAqB;IAmE7B,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,wBAAwB;YAsElB,aAAa;YAgBb,aAAa;YAeb,UAAU;YAoBV,OAAO;YAWP,SAAS;YAaT,qBAAqB;IAgBnC,OAAO,CAAC,mBAAmB;IAgB3B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,2BAA2B;IAoBnC,OAAO,CAAC,2BAA2B,CAAC,CAAiB;IAKrD,OAAO,CAAC,kBAAkB;IA8B1B,OAAO,CAAC,mBAAmB;IAkB3B,OAAO,CAAC,iBAAiB;IAQzB,OAAO,CAAC,IAAI;IAOZ,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,GAAG,IAAI;IAO3D,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,GAAG,IAAI;CAG7D"}
|
package/dist/process-pool.js
CHANGED
|
@@ -2,6 +2,7 @@ import { cpus } from 'node:os';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { EventEmitter } from '@omnitron-dev/eventemitter';
|
|
4
4
|
import { Errors } from '@omnitron-dev/titan/errors';
|
|
5
|
+
import { FailureTracker, computeBackoff } from '@omnitron-dev/titan/utils';
|
|
5
6
|
import { PoolStrategy } from './types.js';
|
|
6
7
|
import { Deque } from './utils/index.js';
|
|
7
8
|
export class PoolBackpressureError extends Error {
|
|
@@ -31,6 +32,7 @@ export class ProcessPool {
|
|
|
31
32
|
this.lastScaleCheck = Date.now();
|
|
32
33
|
this.scaleHistory = [];
|
|
33
34
|
this.unhealthyWorkers = new Set();
|
|
35
|
+
this.healthFailures = new FailureTracker();
|
|
34
36
|
this.queueBatchSize = 10;
|
|
35
37
|
this.queueProcessingScheduled = false;
|
|
36
38
|
this.circuitBreaker = {
|
|
@@ -42,6 +44,11 @@ export class ProcessPool {
|
|
|
42
44
|
this.emitter = new EventEmitter();
|
|
43
45
|
this.healthyWorkersCache = [];
|
|
44
46
|
this.healthyWorkersCacheValid = false;
|
|
47
|
+
this.healthCheckRunning = false;
|
|
48
|
+
this.autoScaleRunning = false;
|
|
49
|
+
this.recyclingRunning = false;
|
|
50
|
+
this.memoryMonitorRunning = false;
|
|
51
|
+
this.heartbeatRunning = false;
|
|
45
52
|
this.poolOptions = this.normalizeOptions(options);
|
|
46
53
|
this.memoryLimitBytes = this.parseMemoryLimit(options.memoryLimit || '512MB');
|
|
47
54
|
this.memoryWarningThreshold = options.memoryWarningThreshold || 0.8;
|
|
@@ -157,8 +164,8 @@ export class ProcessPool {
|
|
|
157
164
|
const pmProcessId = proxy?.__processId;
|
|
158
165
|
if (!pmProcessId)
|
|
159
166
|
return null;
|
|
160
|
-
const handle = this.manager.
|
|
161
|
-
return handle ?? null;
|
|
167
|
+
const handle = this.manager.getWorkerHandle?.(pmProcessId);
|
|
168
|
+
return (handle ?? null);
|
|
162
169
|
}
|
|
163
170
|
get metrics() {
|
|
164
171
|
const avgResponseTime = this.totalRequests > 0 ? this.totalResponseTime / this.totalRequests : 0;
|
|
@@ -278,6 +285,10 @@ export class ProcessPool {
|
|
|
278
285
|
clearInterval(this.memoryMonitorTimer);
|
|
279
286
|
this.memoryMonitorTimer = undefined;
|
|
280
287
|
}
|
|
288
|
+
if (this.circuitBreakerHalfOpenTimer) {
|
|
289
|
+
clearTimeout(this.circuitBreakerHalfOpenTimer);
|
|
290
|
+
this.circuitBreakerHalfOpenTimer = undefined;
|
|
291
|
+
}
|
|
281
292
|
await this.drain();
|
|
282
293
|
const shutdownPromises = Array.from(this.workers.values()).map((worker) => this.shutdownWorker(worker.id));
|
|
283
294
|
await Promise.all(shutdownPromises);
|
|
@@ -285,8 +296,15 @@ export class ProcessPool {
|
|
|
285
296
|
this.isInitialized = false;
|
|
286
297
|
this.isShuttingDown = false;
|
|
287
298
|
this.isDraining = false;
|
|
288
|
-
|
|
289
|
-
|
|
299
|
+
try {
|
|
300
|
+
this.emit('pool:destroyed', { class: this.processName });
|
|
301
|
+
}
|
|
302
|
+
catch (err) {
|
|
303
|
+
this.logger.warn({ err }, 'pool:destroyed subscriber threw — continuing teardown');
|
|
304
|
+
}
|
|
305
|
+
finally {
|
|
306
|
+
this.emitter.removeAllListeners();
|
|
307
|
+
}
|
|
290
308
|
}
|
|
291
309
|
async execute(method, ...args) {
|
|
292
310
|
if (this.poolOptions.circuitBreaker?.enabled && this.circuitBreaker.isOpen) {
|
|
@@ -297,8 +315,7 @@ export class ProcessPool {
|
|
|
297
315
|
if (this.isDraining) {
|
|
298
316
|
throw Errors.conflict('Pool is draining, not accepting new requests');
|
|
299
317
|
}
|
|
300
|
-
if (this.workers.size === 0 ||
|
|
301
|
-
(this.active >= this.workers.size && this.queue.length < (this.poolOptions.maxQueueSize || 100))) {
|
|
318
|
+
if (this.workers.size === 0 || this.active >= this.workers.size) {
|
|
302
319
|
return this.queueRequest(method, args);
|
|
303
320
|
}
|
|
304
321
|
const worker = await this.selectWorker();
|
|
@@ -365,11 +382,14 @@ export class ProcessPool {
|
|
|
365
382
|
return;
|
|
366
383
|
try {
|
|
367
384
|
if (worker.processing.size > 0) {
|
|
368
|
-
this.logger.debug({
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
385
|
+
this.logger.debug({ workerId, activeRequests: worker.processing.size }, 'Waiting for worker to finish active requests');
|
|
386
|
+
const deadline = Date.now() + 5_000;
|
|
387
|
+
while (worker.processing.size > 0 && Date.now() < deadline) {
|
|
388
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
389
|
+
}
|
|
390
|
+
if (worker.processing.size > 0) {
|
|
391
|
+
this.logger.warn({ workerId, activeRequests: worker.processing.size }, 'Worker still has active requests after grace period; killing anyway');
|
|
392
|
+
}
|
|
373
393
|
}
|
|
374
394
|
await this.manager.kill(workerId);
|
|
375
395
|
this.workers.delete(workerId);
|
|
@@ -566,13 +586,17 @@ export class ProcessPool {
|
|
|
566
586
|
worker.processing.add(requestId);
|
|
567
587
|
worker.currentLoad++;
|
|
568
588
|
this.totalRequests++;
|
|
589
|
+
let timeoutHandle = null;
|
|
569
590
|
try {
|
|
570
591
|
const timeout = this.poolOptions.requestTimeout;
|
|
571
592
|
let result;
|
|
572
593
|
if (timeout) {
|
|
594
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
595
|
+
timeoutHandle = setTimeout(() => reject(Errors.timeout('Request', timeout)), timeout);
|
|
596
|
+
});
|
|
573
597
|
result = await Promise.race([
|
|
574
598
|
worker.proxy[method](...args),
|
|
575
|
-
|
|
599
|
+
timeoutPromise,
|
|
576
600
|
]);
|
|
577
601
|
}
|
|
578
602
|
else {
|
|
@@ -606,8 +630,10 @@ export class ProcessPool {
|
|
|
606
630
|
throw error;
|
|
607
631
|
}
|
|
608
632
|
finally {
|
|
633
|
+
if (timeoutHandle)
|
|
634
|
+
clearTimeout(timeoutHandle);
|
|
609
635
|
worker.processing.delete(requestId);
|
|
610
|
-
worker.currentLoad
|
|
636
|
+
worker.currentLoad = Math.max(0, worker.currentLoad - 1);
|
|
611
637
|
worker.lastUsed = Date.now();
|
|
612
638
|
}
|
|
613
639
|
}
|
|
@@ -689,68 +715,104 @@ export class ProcessPool {
|
|
|
689
715
|
const score2 = worker2.currentLoad * 10 + (worker2.memoryUsage / this.memoryLimitBytes) * 5;
|
|
690
716
|
return score1 <= score2 ? idx1 : idx2;
|
|
691
717
|
}
|
|
718
|
+
emitHealthDecision(decision, workerId, error) {
|
|
719
|
+
if (!decision)
|
|
720
|
+
return;
|
|
721
|
+
const payload = {
|
|
722
|
+
workerId,
|
|
723
|
+
consecutiveFailures: decision.count,
|
|
724
|
+
elapsedMs: decision.elapsedMs,
|
|
725
|
+
...(error !== undefined && { error }),
|
|
726
|
+
};
|
|
727
|
+
switch (decision.level) {
|
|
728
|
+
case 'first':
|
|
729
|
+
this.logger.warn(payload, 'Worker health check started failing');
|
|
730
|
+
break;
|
|
731
|
+
case 'continuing':
|
|
732
|
+
this.logger.debug(payload, 'Worker health check still failing');
|
|
733
|
+
break;
|
|
734
|
+
case 'persistent':
|
|
735
|
+
this.logger.error(payload, 'Worker health check persistent failure — worker will be replaced when threshold is reached');
|
|
736
|
+
break;
|
|
737
|
+
case 'suppress':
|
|
738
|
+
break;
|
|
739
|
+
case 'recovery':
|
|
740
|
+
this.logger.info(payload, 'Worker health check recovered');
|
|
741
|
+
break;
|
|
742
|
+
}
|
|
743
|
+
}
|
|
692
744
|
setupHealthMonitoring() {
|
|
693
745
|
const baseInterval = this.poolOptions.healthCheck?.interval || 30000;
|
|
694
746
|
const HEALTH_CHECK_CONCURRENCY = 10;
|
|
695
747
|
const MAX_BACKOFF_MULTIPLIER = 8;
|
|
696
748
|
this.healthCheckTimer = setInterval(async () => {
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
749
|
+
if (this.healthCheckRunning)
|
|
750
|
+
return;
|
|
751
|
+
this.healthCheckRunning = true;
|
|
752
|
+
try {
|
|
753
|
+
let healthChanged = false;
|
|
754
|
+
const workers = Array.from(this.workers.values());
|
|
755
|
+
const now = Date.now();
|
|
756
|
+
const checkWorkerHealth = async (worker) => {
|
|
757
|
+
const previousHealth = worker.health;
|
|
758
|
+
if (worker.consecutiveFailures > 0) {
|
|
759
|
+
const backoffMultiplier = computeBackoff({
|
|
760
|
+
attempt: worker.consecutiveFailures - 1,
|
|
761
|
+
baseMs: 1,
|
|
762
|
+
maxMs: MAX_BACKOFF_MULTIPLIER,
|
|
763
|
+
});
|
|
764
|
+
const nextCheckTime = worker.lastHeartbeat + baseInterval * backoffMultiplier;
|
|
765
|
+
if (now < nextCheckTime) {
|
|
766
|
+
return false;
|
|
767
|
+
}
|
|
707
768
|
}
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
health
|
|
714
|
-
|
|
715
|
-
|
|
769
|
+
try {
|
|
770
|
+
if ('__getHealth' in worker.proxy) {
|
|
771
|
+
const health = await worker.proxy.__getHealth();
|
|
772
|
+
worker.health =
|
|
773
|
+
health.status === 'healthy' ? 'healthy' : health.status === 'degraded' ? 'degraded' : 'unhealthy';
|
|
774
|
+
if (worker.health === 'healthy') {
|
|
775
|
+
worker.consecutiveFailures = 0;
|
|
776
|
+
this.emitHealthDecision(this.healthFailures.recordSuccess(worker.id), worker.id, undefined);
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
if ('__getMetrics' in worker.proxy) {
|
|
780
|
+
worker.metrics = await worker.proxy.__getMetrics();
|
|
716
781
|
}
|
|
717
782
|
}
|
|
718
|
-
|
|
719
|
-
|
|
783
|
+
catch (error) {
|
|
784
|
+
const decision = this.healthFailures.recordFailure(worker.id);
|
|
785
|
+
this.emitHealthDecision(decision, worker.id, error);
|
|
786
|
+
worker.health = 'unhealthy';
|
|
787
|
+
worker.consecutiveFailures++;
|
|
788
|
+
}
|
|
789
|
+
return previousHealth !== worker.health;
|
|
790
|
+
};
|
|
791
|
+
for (let i = 0; i < workers.length; i += HEALTH_CHECK_CONCURRENCY) {
|
|
792
|
+
const batch = workers.slice(i, i + HEALTH_CHECK_CONCURRENCY);
|
|
793
|
+
const results = await Promise.all(batch.map(checkWorkerHealth));
|
|
794
|
+
if (results.some((changed) => changed)) {
|
|
795
|
+
healthChanged = true;
|
|
720
796
|
}
|
|
721
797
|
}
|
|
722
|
-
|
|
723
|
-
this.
|
|
724
|
-
error,
|
|
725
|
-
workerId: worker.id,
|
|
726
|
-
consecutiveFailures: worker.consecutiveFailures + 1,
|
|
727
|
-
}, 'Health check failed');
|
|
728
|
-
worker.health = 'unhealthy';
|
|
729
|
-
worker.consecutiveFailures++;
|
|
798
|
+
if (healthChanged) {
|
|
799
|
+
this.invalidateHealthyWorkersCache();
|
|
730
800
|
}
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
801
|
+
const unhealthyThreshold = this.poolOptions.healthCheck?.unhealthyThreshold || 3;
|
|
802
|
+
for (const worker of workers) {
|
|
803
|
+
if (worker.health === 'unhealthy' && worker.consecutiveFailures >= unhealthyThreshold && !this.isShuttingDown) {
|
|
804
|
+
this.logger.info({
|
|
805
|
+
workerId: worker.id,
|
|
806
|
+
consecutiveFailures: worker.consecutiveFailures,
|
|
807
|
+
restarts: worker.restarts,
|
|
808
|
+
}, 'Replacing unhealthy worker');
|
|
809
|
+
worker.restarts++;
|
|
810
|
+
await this.replaceWorker(worker.id);
|
|
811
|
+
}
|
|
738
812
|
}
|
|
739
813
|
}
|
|
740
|
-
|
|
741
|
-
this.
|
|
742
|
-
}
|
|
743
|
-
const unhealthyThreshold = this.poolOptions.healthCheck?.unhealthyThreshold || 3;
|
|
744
|
-
for (const worker of workers) {
|
|
745
|
-
if (worker.health === 'unhealthy' && worker.consecutiveFailures >= unhealthyThreshold && !this.isShuttingDown) {
|
|
746
|
-
this.logger.info({
|
|
747
|
-
workerId: worker.id,
|
|
748
|
-
consecutiveFailures: worker.consecutiveFailures,
|
|
749
|
-
restarts: worker.restarts,
|
|
750
|
-
}, 'Replacing unhealthy worker');
|
|
751
|
-
worker.restarts++;
|
|
752
|
-
await this.replaceWorker(worker.id);
|
|
753
|
-
}
|
|
814
|
+
finally {
|
|
815
|
+
this.healthCheckRunning = false;
|
|
754
816
|
}
|
|
755
817
|
}, baseInterval);
|
|
756
818
|
}
|
|
@@ -761,6 +823,9 @@ export class ProcessPool {
|
|
|
761
823
|
this.autoScaleTimer = setInterval(async () => {
|
|
762
824
|
if (this.isShuttingDown || this.isDraining)
|
|
763
825
|
return;
|
|
826
|
+
if (this.autoScaleRunning)
|
|
827
|
+
return;
|
|
828
|
+
this.autoScaleRunning = true;
|
|
764
829
|
try {
|
|
765
830
|
const now = Date.now();
|
|
766
831
|
const cooldownPeriod = this.poolOptions.autoScale?.cooldownPeriod || 60000;
|
|
@@ -795,12 +860,18 @@ export class ProcessPool {
|
|
|
795
860
|
catch (error) {
|
|
796
861
|
this.logger.error({ err: error instanceof Error ? error : new Error(String(error)) }, 'Error during auto-scaling check');
|
|
797
862
|
}
|
|
863
|
+
finally {
|
|
864
|
+
this.autoScaleRunning = false;
|
|
865
|
+
}
|
|
798
866
|
}, checkInterval);
|
|
799
867
|
}
|
|
800
868
|
setupRecycling() {
|
|
801
869
|
this.recycleTimer = setInterval(async () => {
|
|
802
870
|
if (this.isShuttingDown)
|
|
803
871
|
return;
|
|
872
|
+
if (this.recyclingRunning)
|
|
873
|
+
return;
|
|
874
|
+
this.recyclingRunning = true;
|
|
804
875
|
try {
|
|
805
876
|
const now = Date.now();
|
|
806
877
|
const maxLifetime = this.poolOptions.maxLifetime || 3600000;
|
|
@@ -834,6 +905,9 @@ export class ProcessPool {
|
|
|
834
905
|
catch (error) {
|
|
835
906
|
this.logger.error({ error }, 'Error during worker recycling');
|
|
836
907
|
}
|
|
908
|
+
finally {
|
|
909
|
+
this.recyclingRunning = false;
|
|
910
|
+
}
|
|
837
911
|
}, 60000);
|
|
838
912
|
}
|
|
839
913
|
setupMemoryMonitoring() {
|
|
@@ -841,42 +915,50 @@ export class ProcessPool {
|
|
|
841
915
|
this.memoryMonitorTimer = setInterval(async () => {
|
|
842
916
|
if (this.isShuttingDown)
|
|
843
917
|
return;
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
const
|
|
853
|
-
if (
|
|
854
|
-
worker.
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
918
|
+
if (this.memoryMonitorRunning)
|
|
919
|
+
return;
|
|
920
|
+
this.memoryMonitorRunning = true;
|
|
921
|
+
try {
|
|
922
|
+
const workers = Array.from(this.workers.values());
|
|
923
|
+
await Promise.all(workers.map(async (worker) => {
|
|
924
|
+
try {
|
|
925
|
+
if ('__getMetrics' in worker.proxy) {
|
|
926
|
+
const metrics = await worker.proxy.__getMetrics();
|
|
927
|
+
if (metrics && typeof metrics.memory === 'number') {
|
|
928
|
+
worker.memoryUsage = metrics.memory;
|
|
929
|
+
worker.peakMemoryUsage = Math.max(worker.peakMemoryUsage, metrics.memory);
|
|
930
|
+
const memoryRatio = metrics.memory / this.memoryLimitBytes;
|
|
931
|
+
if (memoryRatio > 0.9) {
|
|
932
|
+
worker.weight = 10;
|
|
933
|
+
}
|
|
934
|
+
else if (memoryRatio > 0.7) {
|
|
935
|
+
worker.weight = 50;
|
|
936
|
+
}
|
|
937
|
+
else {
|
|
938
|
+
worker.weight = 100;
|
|
939
|
+
}
|
|
861
940
|
}
|
|
862
941
|
}
|
|
863
942
|
}
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
}
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
})
|
|
879
|
-
}
|
|
943
|
+
catch (error) {
|
|
944
|
+
this.logger.debug({ error, workerId: worker.id }, 'Failed to collect memory metrics');
|
|
945
|
+
}
|
|
946
|
+
}));
|
|
947
|
+
const totalMemory = workers.reduce((sum, w) => sum + w.memoryUsage, 0);
|
|
948
|
+
const avgMemory = workers.length > 0 ? totalMemory / workers.length : 0;
|
|
949
|
+
this.emit('pool:memory', {
|
|
950
|
+
totalMemory,
|
|
951
|
+
avgMemory,
|
|
952
|
+
workers: workers.map((w) => ({
|
|
953
|
+
id: w.id,
|
|
954
|
+
memory: w.memoryUsage,
|
|
955
|
+
peak: w.peakMemoryUsage,
|
|
956
|
+
})),
|
|
957
|
+
});
|
|
958
|
+
}
|
|
959
|
+
finally {
|
|
960
|
+
this.memoryMonitorRunning = false;
|
|
961
|
+
}
|
|
880
962
|
}, MEMORY_CHECK_INTERVAL);
|
|
881
963
|
}
|
|
882
964
|
setupHeartbeatMonitoring() {
|
|
@@ -886,39 +968,58 @@ export class ProcessPool {
|
|
|
886
968
|
this.heartbeatTimer = setInterval(async () => {
|
|
887
969
|
if (this.isShuttingDown)
|
|
888
970
|
return;
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
971
|
+
if (this.heartbeatRunning)
|
|
972
|
+
return;
|
|
973
|
+
this.heartbeatRunning = true;
|
|
974
|
+
try {
|
|
975
|
+
const now = Date.now();
|
|
976
|
+
const workers = Array.from(this.workers.values());
|
|
977
|
+
await Promise.all(workers.map(async (worker) => {
|
|
978
|
+
let timeoutHandle = null;
|
|
979
|
+
try {
|
|
980
|
+
const heartbeatPromise = this.sendHeartbeat(worker);
|
|
981
|
+
const timeoutPromise = new Promise((resolve) => {
|
|
982
|
+
timeoutHandle = setTimeout(() => resolve(false), HEARTBEAT_TIMEOUT);
|
|
983
|
+
});
|
|
984
|
+
const responded = await Promise.race([heartbeatPromise, timeoutPromise]);
|
|
985
|
+
if (timeoutHandle) {
|
|
986
|
+
clearTimeout(timeoutHandle);
|
|
987
|
+
timeoutHandle = null;
|
|
988
|
+
}
|
|
989
|
+
if (responded) {
|
|
990
|
+
worker.lastHeartbeat = now;
|
|
991
|
+
worker.consecutiveFailures = 0;
|
|
992
|
+
}
|
|
993
|
+
else {
|
|
994
|
+
worker.consecutiveFailures++;
|
|
995
|
+
if (worker.consecutiveFailures >= MAX_MISSED_HEARTBEATS) {
|
|
996
|
+
this.logger.warn({
|
|
997
|
+
workerId: worker.id,
|
|
998
|
+
missedHeartbeats: worker.consecutiveFailures,
|
|
999
|
+
}, 'Worker unresponsive - marking unhealthy');
|
|
1000
|
+
worker.health = 'unhealthy';
|
|
1001
|
+
this.unhealthyWorkers.add(worker.id);
|
|
1002
|
+
this.invalidateHealthyWorkersCache();
|
|
1003
|
+
this.emit('worker:unresponsive', { workerId: worker.id });
|
|
1004
|
+
if (!this.isShuttingDown) {
|
|
1005
|
+
await this.replaceWorker(worker.id);
|
|
1006
|
+
}
|
|
913
1007
|
}
|
|
914
1008
|
}
|
|
915
1009
|
}
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
1010
|
+
catch (error) {
|
|
1011
|
+
worker.consecutiveFailures++;
|
|
1012
|
+
this.logger.debug({ error, workerId: worker.id }, 'Heartbeat failed');
|
|
1013
|
+
}
|
|
1014
|
+
finally {
|
|
1015
|
+
if (timeoutHandle)
|
|
1016
|
+
clearTimeout(timeoutHandle);
|
|
1017
|
+
}
|
|
1018
|
+
}));
|
|
1019
|
+
}
|
|
1020
|
+
finally {
|
|
1021
|
+
this.heartbeatRunning = false;
|
|
1022
|
+
}
|
|
922
1023
|
}, HEARTBEAT_INTERVAL);
|
|
923
1024
|
}
|
|
924
1025
|
async sendHeartbeat(worker) {
|
|
@@ -1003,8 +1104,12 @@ export class ProcessPool {
|
|
|
1003
1104
|
this.circuitBreaker.successAfterOpen = 0;
|
|
1004
1105
|
this.logger.warn('Circuit breaker opened');
|
|
1005
1106
|
this.emit('circuitbreaker:open');
|
|
1107
|
+
if (this.circuitBreakerHalfOpenTimer) {
|
|
1108
|
+
clearTimeout(this.circuitBreakerHalfOpenTimer);
|
|
1109
|
+
}
|
|
1006
1110
|
const timeout = this.poolOptions.circuitBreaker?.timeout || 60000;
|
|
1007
|
-
setTimeout(() => {
|
|
1111
|
+
this.circuitBreakerHalfOpenTimer = setTimeout(() => {
|
|
1112
|
+
this.circuitBreakerHalfOpenTimer = undefined;
|
|
1008
1113
|
if (this.circuitBreaker.isOpen) {
|
|
1009
1114
|
this.logger.info('Circuit breaker entering half-open state');
|
|
1010
1115
|
this.emit('circuitbreaker:halfopen');
|
|
@@ -1015,6 +1120,11 @@ export class ProcessPool {
|
|
|
1015
1120
|
this.circuitBreaker.isOpen = false;
|
|
1016
1121
|
this.circuitBreaker.failures = 0;
|
|
1017
1122
|
this.circuitBreaker.successAfterOpen = 0;
|
|
1123
|
+
this.circuitBreaker.lastFailTime = 0;
|
|
1124
|
+
if (this.circuitBreakerHalfOpenTimer) {
|
|
1125
|
+
clearTimeout(this.circuitBreakerHalfOpenTimer);
|
|
1126
|
+
this.circuitBreakerHalfOpenTimer = undefined;
|
|
1127
|
+
}
|
|
1018
1128
|
this.logger.info('Circuit breaker closed');
|
|
1019
1129
|
this.emit('circuitbreaker:close');
|
|
1020
1130
|
}
|