@omnitron-dev/titan-pm 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backoff.d.ts +12 -0
- package/dist/backoff.d.ts.map +1 -0
- package/dist/backoff.js +44 -0
- package/dist/backoff.js.map +1 -0
- package/dist/child-contract.d.ts +58 -0
- package/dist/child-contract.d.ts.map +1 -0
- package/dist/child-contract.js +27 -0
- package/dist/child-contract.js.map +1 -0
- package/dist/confirm-death.d.ts +18 -0
- package/dist/confirm-death.d.ts.map +1 -0
- package/dist/confirm-death.js +49 -0
- package/dist/confirm-death.js.map +1 -0
- package/dist/decorators.d.ts +1 -10
- package/dist/decorators.d.ts.map +1 -1
- package/dist/decorators.js +43 -99
- package/dist/decorators.js.map +1 -1
- package/dist/discovery.d.ts +24 -0
- package/dist/discovery.d.ts.map +1 -0
- package/dist/discovery.js +162 -0
- package/dist/discovery.js.map +1 -0
- package/dist/fork-worker.js +61 -45
- package/dist/fork-worker.js.map +1 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/last-resort-handlers.d.ts +5 -0
- package/dist/last-resort-handlers.d.ts.map +1 -0
- package/dist/last-resort-handlers.js +40 -0
- package/dist/last-resort-handlers.js.map +1 -0
- package/dist/lifecycle-queue.d.ts +20 -0
- package/dist/lifecycle-queue.d.ts.map +1 -0
- package/dist/lifecycle-queue.js +110 -0
- package/dist/lifecycle-queue.js.map +1 -0
- package/dist/liveness.d.ts +5 -0
- package/dist/liveness.d.ts.map +1 -0
- package/dist/liveness.js +34 -0
- package/dist/liveness.js.map +1 -0
- package/dist/netron-client.d.ts +10 -8
- package/dist/netron-client.d.ts.map +1 -1
- package/dist/netron-client.js +16 -10
- package/dist/netron-client.js.map +1 -1
- package/dist/parent-death-watchdog.d.ts +13 -0
- package/dist/parent-death-watchdog.d.ts.map +1 -0
- package/dist/parent-death-watchdog.js +58 -0
- package/dist/parent-death-watchdog.js.map +1 -0
- package/dist/pm.module.d.ts.map +1 -1
- package/dist/pm.module.js.map +1 -1
- package/dist/process-health.d.ts.map +1 -1
- package/dist/process-health.js +20 -4
- package/dist/process-health.js.map +1 -1
- package/dist/process-manager.d.ts +4 -0
- package/dist/process-manager.d.ts.map +1 -1
- package/dist/process-manager.js +51 -4
- package/dist/process-manager.js.map +1 -1
- package/dist/process-metrics.d.ts.map +1 -1
- package/dist/process-metrics.js +10 -19
- package/dist/process-metrics.js.map +1 -1
- package/dist/process-pool.d.ts +4 -0
- package/dist/process-pool.d.ts.map +1 -1
- package/dist/process-pool.js +112 -29
- package/dist/process-pool.js.map +1 -1
- package/dist/process-registry.d.ts.map +1 -1
- package/dist/process-spawner.d.ts +18 -2
- package/dist/process-spawner.d.ts.map +1 -1
- package/dist/process-spawner.js +289 -62
- package/dist/process-spawner.js.map +1 -1
- package/dist/process-supervisor.d.ts +1 -0
- package/dist/process-supervisor.d.ts.map +1 -1
- package/dist/process-supervisor.js +59 -45
- package/dist/process-supervisor.js.map +1 -1
- package/dist/process-workflow.d.ts.map +1 -1
- package/dist/service-proxy.d.ts.map +1 -1
- package/dist/service-proxy.js +33 -3
- package/dist/service-proxy.js.map +1 -1
- package/dist/shutdown-windows.d.ts +19 -0
- package/dist/shutdown-windows.d.ts.map +1 -0
- package/dist/shutdown-windows.js +35 -0
- package/dist/shutdown-windows.js.map +1 -0
- package/dist/thread-pool.d.ts +7 -0
- package/dist/thread-pool.d.ts.map +1 -0
- package/dist/thread-pool.js +11 -0
- package/dist/thread-pool.js.map +1 -0
- package/dist/types.d.ts +79 -37
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +35 -39
- package/dist/types.js.map +1 -1
- package/dist/utils/deque.d.ts.map +1 -1
- package/dist/worker-health.d.ts +8 -0
- package/dist/worker-health.d.ts.map +1 -0
- package/dist/worker-health.js +63 -0
- package/dist/worker-health.js.map +1 -0
- package/dist/worker-runtime.js +17 -53
- package/dist/worker-runtime.js.map +1 -1
- package/dist/worker-traffic.d.ts +8 -0
- package/dist/worker-traffic.d.ts.map +1 -0
- package/dist/worker-traffic.js +56 -0
- package/dist/worker-traffic.js.map +1 -0
- package/package.json +9 -8
package/dist/process-metrics.js
CHANGED
|
@@ -19,7 +19,9 @@ export class ProcessMetricsCollector {
|
|
|
19
19
|
this.collectInProgress.add(processId);
|
|
20
20
|
try {
|
|
21
21
|
const metrics = await this.collectMetrics(proxy);
|
|
22
|
-
|
|
22
|
+
if (metrics) {
|
|
23
|
+
this.storeMetrics(processId, metrics);
|
|
24
|
+
}
|
|
23
25
|
}
|
|
24
26
|
catch (error) {
|
|
25
27
|
this.logger.error({ error, processId }, 'Failed to collect metrics');
|
|
@@ -73,30 +75,23 @@ export class ProcessMetricsCollector {
|
|
|
73
75
|
}
|
|
74
76
|
async collectMetrics(proxy) {
|
|
75
77
|
try {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
+
const getMetrics = proxy.__getMetrics;
|
|
79
|
+
if (typeof getMetrics !== 'function') {
|
|
80
|
+
return null;
|
|
78
81
|
}
|
|
79
|
-
return
|
|
80
|
-
cpu: 0,
|
|
81
|
-
memory: 0,
|
|
82
|
-
requests: 0,
|
|
83
|
-
errors: 0,
|
|
84
|
-
};
|
|
82
|
+
return await getMetrics.call(proxy);
|
|
85
83
|
}
|
|
86
84
|
catch (error) {
|
|
87
85
|
this.logger.error({ error }, 'Failed to collect process metrics');
|
|
88
|
-
return
|
|
89
|
-
cpu: 0,
|
|
90
|
-
memory: 0,
|
|
91
|
-
requests: 0,
|
|
92
|
-
errors: 1,
|
|
93
|
-
};
|
|
86
|
+
return null;
|
|
94
87
|
}
|
|
95
88
|
}
|
|
96
89
|
storeMetrics(processId, metrics) {
|
|
97
90
|
const history = this.metricsHistory.get(processId) || [];
|
|
91
|
+
const latencies = this.latencyTracking.get(processId);
|
|
98
92
|
const timestampedMetrics = {
|
|
99
93
|
...metrics,
|
|
94
|
+
...(latencies && latencies.length > 0 ? { latency: this.calculateLatencyMetrics(latencies) } : {}),
|
|
100
95
|
timestamp: Date.now(),
|
|
101
96
|
};
|
|
102
97
|
history.push(timestampedMetrics);
|
|
@@ -104,10 +99,6 @@ export class ProcessMetricsCollector {
|
|
|
104
99
|
history.shift();
|
|
105
100
|
}
|
|
106
101
|
this.metricsHistory.set(processId, history);
|
|
107
|
-
const latencies = this.latencyTracking.get(processId);
|
|
108
|
-
if (latencies && latencies.length > 0) {
|
|
109
|
-
metrics.latency = this.calculateLatencyMetrics(latencies);
|
|
110
|
-
}
|
|
111
102
|
}
|
|
112
103
|
calculateLatencyMetrics(latencies) {
|
|
113
104
|
const sorted = [...latencies].sort((a, b) => a - b);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"process-metrics.js","sourceRoot":"","sources":["../src/process-metrics.ts"],"names":[],"mappings":"AAYA,MAAM,OAAO,uBAAuB;IAYlC,YAA6B,MAAe;QAAf,WAAM,GAAN,MAAM,CAAS;QAXpC,eAAU,GAAG,IAAI,GAAG,EAA0B,CAAC;QAC/C,mBAAc,GAAG,IAAI,GAAG,EAA6B,CAAC;QACtD,oBAAe,GAAG,IAAI,GAAG,EAAoB,CAAC;QAO9C,sBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;IAEC,CAAC;IAKhD,eAAe,CAAC,SAAiB,EAAE,KAAwB,EAAE,
|
|
1
|
+
{"version":3,"file":"process-metrics.js","sourceRoot":"","sources":["../src/process-metrics.ts"],"names":[],"mappings":"AAYA,MAAM,OAAO,uBAAuB;IAYlC,YAA6B,MAAe;QAAf,WAAM,GAAN,MAAM,CAAS;QAXpC,eAAU,GAAG,IAAI,GAAG,EAA0B,CAAC;QAC/C,mBAAc,GAAG,IAAI,GAAG,EAA6B,CAAC;QACtD,oBAAe,GAAG,IAAI,GAAG,EAAoB,CAAC;QAO9C,sBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;IAEC,CAAC;IAKhD,eAAe,CAAC,SAAiB,EAAE,KAAwB,EAAE,QAAQ,GAAW,IAAI;QAClF,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACnC,OAAO;QACT,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,EAAE,6BAA6B,CAAC,CAAC;QAGhE,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACvC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAGxC,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YACvC,IAAI,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC;gBAAE,OAAO;YAClD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACtC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;gBAGjD,IAAI,OAAO,EAAE,CAAC;oBACZ,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBACxC,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,2BAA2B,CAAC,CAAC;YACvE,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC,EAAE,QAAQ,CAAC,CAAC;QAEb,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IAC5C,CAAC;IAKD,cAAc,CAAC,SAAiB;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,SAAS,EAAE,CAAC;YACd,aAAa,CAAC,SAAS,CAAC,CAAC;YACzB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAClC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACtC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACvC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAEzC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,EAAE,4BAA4B,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAKD,UAAU,CAAC,SAAiB;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACnD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC;IAC7C,CAAC;IAKD,iBAAiB,CAAC,SAAiB;QACjC,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;IAClD,CAAC;IAKD,oBAAoB,CAAC,SAAiB,EAAE,UAAU,GAAW,KAAK;QAChE,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACnD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAS,CAAC,SAAS,IAAI,GAAG,GAAI,CAAS,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC;QAE7G,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;IAKD,YAAY,CAAC,SAAiB,EAAE,QAAgB;QAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QAC5D,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAGzB,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YAC5B,SAAS,CAAC,KAAK,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACjD,CAAC;IASO,KAAK,CAAC,cAAc,CAAC,KAAwB;QAanD,IAAI,CAAC;YACH,MAAM,UAAU,GAAI,KAA2D,CAAC,YAAY,CAAC;YAC7F,IAAI,OAAO,UAAU,KAAK,UAAU,EAAE,CAAC;gBACrC,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,MAAM,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,mCAAmC,CAAC,CAAC;YAClE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAKO,YAAY,CAAC,SAAiB,EAAE,OAAwB;QAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QAUzD,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACtD,MAAM,kBAAkB,GAAoB;YAC1C,GAAG,OAAO;YACV,GAAG,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClG,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACH,CAAC;QAErB,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAGjC,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC;IAKO,uBAAuB,CAAC,SAAmB;QACjD,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpD,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;QAE1B,OAAO;YACL,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAChC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAChC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAChC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAChC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAChC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG;SAC9C,CAAC;IACJ,CAAC;IAKO,UAAU,CAAC,MAAgB,EAAE,CAAS;QAC5C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAElC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACvD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC;IAKO,gBAAgB,CAAC,OAA0B;QACjD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO;gBACL,GAAG,EAAE,CAAC;gBACN,MAAM,EAAE,CAAC;gBACT,QAAQ,EAAE,CAAC;gBACX,MAAM,EAAE,CAAC;aACV,CAAC;QACJ,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CACxB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACX,GAAG,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;YACpB,MAAM,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM;YAC7B,QAAQ,EAAE,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC;YACjD,MAAM,EAAE,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;SAC5C,CAAC,EACF;YACE,GAAG,EAAE,CAAC;YACN,MAAM,EAAE,CAAC;YACT,QAAQ,EAAE,CAAC;YACX,MAAM,EAAE,CAAC;SACV,CACF,CAAC;QAEF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;QAG7B,IAAI,OAAoC,CAAC;QACzC,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAQ,CAAC,CAAC;QAE/E,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,OAAO,GAAG;gBACR,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACnD,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACnD,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACnD,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACnD,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACnD,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;aACtD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,GAAG,EAAE,GAAG,CAAC,GAAG,GAAG,KAAK;YACpB,MAAM,EAAE,GAAG,CAAC,MAAM,GAAG,KAAK;YAC1B,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,OAAO;SACR,CAAC;IACJ,CAAC;IAKO,OAAO,CAAC,OAAiB;QAC/B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACnC,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAC7D,CAAC;CACF"}
|
package/dist/process-pool.d.ts
CHANGED
|
@@ -30,6 +30,7 @@ export declare class ProcessPool<T> {
|
|
|
30
30
|
private unhealthyWorkers;
|
|
31
31
|
private readonly healthFailures;
|
|
32
32
|
private readonly memoryLimitBytes;
|
|
33
|
+
private readonly cpuSamples;
|
|
33
34
|
private readonly memoryWarningThreshold;
|
|
34
35
|
private readonly queueBatchSize;
|
|
35
36
|
private queueProcessingScheduled;
|
|
@@ -88,6 +89,8 @@ export declare class ProcessPool<T> {
|
|
|
88
89
|
private healthCheckRunning;
|
|
89
90
|
private setupHealthMonitoring;
|
|
90
91
|
private autoScaleRunning;
|
|
92
|
+
private autoScaleConsecutiveFailures;
|
|
93
|
+
private autoScaleNextRetryAt;
|
|
91
94
|
private setupAutoScaling;
|
|
92
95
|
private recyclingRunning;
|
|
93
96
|
private setupRecycling;
|
|
@@ -102,6 +105,7 @@ export declare class ProcessPool<T> {
|
|
|
102
105
|
private scaleDown;
|
|
103
106
|
private waitForActiveRequests;
|
|
104
107
|
private calculateThroughput;
|
|
108
|
+
private withDerivedPercentages;
|
|
105
109
|
private calculateSaturation;
|
|
106
110
|
private handleCircuitBreakerFailure;
|
|
107
111
|
private circuitBreakerHalfOpenTimer?;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"process-pool.d.ts","sourceRoot":"","sources":["../src/process-pool.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AAEjE,OAAO,KAAK,EAAE,mBAAmB,EAAgB,YAAY,EAAE,eAAe,EAAmB,MAAM,YAAY,CAAC;AAOpH,qBAAa,qBAAsB,SAAQ,KAAK;IAC9C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"process-pool.d.ts","sourceRoot":"","sources":["../src/process-pool.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AAEjE,OAAO,KAAK,EAAE,mBAAmB,EAAgB,YAAY,EAAE,eAAe,EAAmB,MAAM,YAAY,CAAC;AAOpH,qBAAa,qBAAsB,SAAQ,KAAK;IAC9C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,YAAY,SAAS,EAAE,MAAM,EAAE,YAAY,SAAO,EAIjD;CACF;AA8DD,qBAAa,WAAW,CAAC,CAAC;IAsEtB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IAEnC,OAAO,CAAC,QAAQ,CAAC,MAAM;IAxEzB,OAAO,CAAC,OAAO,CAAoC;IACnD,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAK;IACnC,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,kBAAkB,CAAK;IAG/B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,kBAAkB,CAAK;IAG/B,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,cAAc,CAAc;IACpC,OAAO,CAAC,YAAY,CAAqF;IAGzG,OAAO,CAAC,gBAAgB,CAAC,CAAiB;IAC1C,OAAO,CAAC,YAAY,CAAC,CAAiB;IACtC,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,kBAAkB,CAAC,CAAiB;IAC5C,OAAO,CAAC,gBAAgB,CAAqB;IAU7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAwB;IAGvD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAK1C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsD;IACjF,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAGhD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAc;IAC7C,OAAO,CAAC,wBAAwB,CAAS;IAGzC,OAAO,CAAC,cAAc,CAKpB;IAGF,OAAO,CAAC,OAAO,CAAsB;IAGrC,OAAO,CAAC,WAAW,CAAsB;IAGzC,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,wBAAwB,CAAS;IAEzC,YACmB,OAAO,EAAE,eAAe,EACxB,kBAAkB,EAAE,MAAM,GAAG,CAAC,KAAK,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,EACzE,OAAO,EAAE,mBAAmB,EACX,MAAM,EAAE,OAAO,EAOjC;IAMD,OAAO,CAAC,gBAAgB;IAsBxB,OAAO,KAAK,WAAW,GAItB;IAKD,OAAO,CAAC,gBAAgB;IAuClB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CA8ChC;IAKD,IAAI,IAAI,IAAI,MAAM,CAEjB;IAKD,IAAI,MAAM,IAAI,MAAM,CAMnB;IAKD,IAAI,OAAO,IAAI,MAAM,CAEpB;IAKD,YAAY,IAAI,MAAM,EAAE,CAEvB;IAYD,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KAAE,GAAG,IAAI,CAQtF;IAKD,IAAI,OAAO,IAAI,YAAY,CA0D1B;IAKD,IAAI,mBAAmB,IAAI;QACzB,KAAK,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;QACvC,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,OAAO,CAAC;KACpB,CAcA;IAKK,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA2C1C;IAKK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAoB3B;IAKK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAkE7B;IAgBK,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAgC1D;IAyBD,OAAO,CAAC,WAAW;YAUL,WAAW;YAwEX,cAAc;IA8C5B,OAAO,CAAC,6BAA6B;IAOrC,OAAO,CAAC,iBAAiB;YAWX,YAAY;IA8D1B,OAAO,CAAC,gBAAgB;IAYxB,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,cAAc;IAyBtB,OAAO,CAAC,kBAAkB;IAU1B,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,eAAe;IAavB,OAAO,CAAC,sBAAsB;IAsB9B,OAAO,CAAC,wBAAwB;IAoChC,OAAO,CAAC,uBAAuB;IA8B/B,OAAO,CAAC,eAAe;YA2BT,eAAe;YAiFf,YAAY;YAwDZ,YAAY;IAkB1B,OAAO,CAAC,iBAAiB;IAuDzB,OAAO,CAAC,iBAAiB;IAiBzB,OAAO,CAAC,kBAAkB;IAoD1B,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,qBAAqB;IA2I7B,OAAO,CAAC,gBAAgB,CAAS;IAWjC,OAAO,CAAC,4BAA4B,CAAK;IACzC,OAAO,CAAC,oBAAoB,CAAK;IACjC,OAAO,CAAC,gBAAgB;IAgIxB,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,cAAc;IAkEtB,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,qBAAqB;IA8E7B,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,wBAAwB;YAsElB,aAAa;YAyBb,aAAa;YAeb,UAAU;YAoBV,OAAO;YAWP,SAAS;YAaT,qBAAqB;IAgBnC,OAAO,CAAC,mBAAmB;IA8B3B,OAAO,CAAC,sBAAsB;IA0B9B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,2BAA2B;IAoBnC,OAAO,CAAC,2BAA2B,CAAC,CAAiB;IAKrD,OAAO,CAAC,kBAAkB;IA8B1B,OAAO,CAAC,mBAAmB;IAkB3B,OAAO,CAAC,iBAAiB;IAQzB,OAAO,CAAC,IAAI;IAOZ,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,GAAG,IAAI,CAE1D;IAKD,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,GAAG,IAAI,CAE3D;CACF"}
|
package/dist/process-pool.js
CHANGED
|
@@ -33,6 +33,7 @@ export class ProcessPool {
|
|
|
33
33
|
this.scaleHistory = [];
|
|
34
34
|
this.unhealthyWorkers = new Set();
|
|
35
35
|
this.healthFailures = new FailureTracker();
|
|
36
|
+
this.cpuSamples = new Map();
|
|
36
37
|
this.queueBatchSize = 10;
|
|
37
38
|
this.queueProcessingScheduled = false;
|
|
38
39
|
this.circuitBreaker = {
|
|
@@ -46,6 +47,8 @@ export class ProcessPool {
|
|
|
46
47
|
this.healthyWorkersCacheValid = false;
|
|
47
48
|
this.healthCheckRunning = false;
|
|
48
49
|
this.autoScaleRunning = false;
|
|
50
|
+
this.autoScaleConsecutiveFailures = 0;
|
|
51
|
+
this.autoScaleNextRetryAt = 0;
|
|
49
52
|
this.recyclingRunning = false;
|
|
50
53
|
this.memoryMonitorRunning = false;
|
|
51
54
|
this.heartbeatRunning = false;
|
|
@@ -173,19 +176,35 @@ export class ProcessPool {
|
|
|
173
176
|
let totalCpu = 0;
|
|
174
177
|
let totalMemory = 0;
|
|
175
178
|
let healthyWorkers = 0;
|
|
179
|
+
let totalCpuPercent = 0;
|
|
180
|
+
let cpuPercentSamples = 0;
|
|
181
|
+
let totalMemoryPercent = 0;
|
|
182
|
+
let memoryPercentSamples = 0;
|
|
176
183
|
this.workers.forEach((worker) => {
|
|
177
184
|
if (worker.metrics) {
|
|
178
185
|
totalCpu += worker.metrics.cpu;
|
|
179
186
|
totalMemory += worker.metrics.memory;
|
|
187
|
+
if (typeof worker.metrics.cpuPercent === 'number') {
|
|
188
|
+
totalCpuPercent += worker.metrics.cpuPercent;
|
|
189
|
+
cpuPercentSamples++;
|
|
190
|
+
}
|
|
191
|
+
if (typeof worker.metrics.memoryPercent === 'number') {
|
|
192
|
+
totalMemoryPercent += worker.metrics.memoryPercent;
|
|
193
|
+
memoryPercentSamples++;
|
|
194
|
+
}
|
|
180
195
|
}
|
|
181
196
|
if (worker.health === 'healthy')
|
|
182
197
|
healthyWorkers++;
|
|
183
198
|
});
|
|
184
199
|
const avgCpu = this.workers.size > 0 ? totalCpu / this.workers.size : 0;
|
|
185
200
|
const avgMemory = this.workers.size > 0 ? totalMemory / this.workers.size : 0;
|
|
201
|
+
const avgCpuPercent = cpuPercentSamples > 0 ? totalCpuPercent / cpuPercentSamples : undefined;
|
|
202
|
+
const avgMemoryPercent = memoryPercentSamples > 0 ? totalMemoryPercent / memoryPercentSamples : undefined;
|
|
186
203
|
return {
|
|
187
204
|
cpu: avgCpu,
|
|
188
205
|
memory: avgMemory,
|
|
206
|
+
...(avgCpuPercent !== undefined ? { cpuPercent: avgCpuPercent } : {}),
|
|
207
|
+
...(avgMemoryPercent !== undefined ? { memoryPercent: avgMemoryPercent } : {}),
|
|
189
208
|
queueSize: this.queue.length,
|
|
190
209
|
activeWorkers: this.active,
|
|
191
210
|
totalWorkers: this.workers.size,
|
|
@@ -336,6 +355,9 @@ export class ProcessPool {
|
|
|
336
355
|
: {};
|
|
337
356
|
const proxy = await this.manager.spawn(this.processPathOrClass, {
|
|
338
357
|
name: `${this.processName}-pool-${workerIndex}`,
|
|
358
|
+
...(this.poolOptions.requestTimeout !== undefined && {
|
|
359
|
+
requestTimeout: this.poolOptions.requestTimeout,
|
|
360
|
+
}),
|
|
339
361
|
...baseOptions,
|
|
340
362
|
...factoryOptions,
|
|
341
363
|
dependencies: {
|
|
@@ -621,7 +643,7 @@ export class ProcessPool {
|
|
|
621
643
|
if (this.poolOptions.circuitBreaker?.enabled) {
|
|
622
644
|
this.handleCircuitBreakerFailure();
|
|
623
645
|
}
|
|
624
|
-
if (worker.errors > (this.poolOptions.healthCheck?.unhealthyThreshold
|
|
646
|
+
if (worker.errors > (this.poolOptions.healthCheck?.unhealthyThreshold ?? 3)) {
|
|
625
647
|
worker.health = 'unhealthy';
|
|
626
648
|
this.unhealthyWorkers.add(worker.id);
|
|
627
649
|
this.invalidateHealthyWorkersCache();
|
|
@@ -638,7 +660,7 @@ export class ProcessPool {
|
|
|
638
660
|
}
|
|
639
661
|
}
|
|
640
662
|
async queueRequest(method, args) {
|
|
641
|
-
if (this.queue.length >= (this.poolOptions.maxQueueSize
|
|
663
|
+
if (this.queue.length >= (this.poolOptions.maxQueueSize ?? 100)) {
|
|
642
664
|
throw new PoolBackpressureError(this.queue.length);
|
|
643
665
|
}
|
|
644
666
|
return new Promise((resolve, reject) => {
|
|
@@ -665,7 +687,9 @@ export class ProcessPool {
|
|
|
665
687
|
queueSize: this.queue.length,
|
|
666
688
|
method,
|
|
667
689
|
});
|
|
668
|
-
this.processQueue()
|
|
690
|
+
void this.processQueue().catch((err) => {
|
|
691
|
+
this.logger.error({ err, class: this.processName }, 'Process pool queue processing failed');
|
|
692
|
+
});
|
|
669
693
|
});
|
|
670
694
|
}
|
|
671
695
|
async processQueue() {
|
|
@@ -705,7 +729,9 @@ export class ProcessPool {
|
|
|
705
729
|
processed++;
|
|
706
730
|
}
|
|
707
731
|
if (this.queue.length > 0) {
|
|
708
|
-
this.processQueue()
|
|
732
|
+
void this.processQueue().catch((err) => {
|
|
733
|
+
this.logger.error({ err, class: this.processName }, 'Process pool queue processing failed');
|
|
734
|
+
});
|
|
709
735
|
}
|
|
710
736
|
}
|
|
711
737
|
selectBestFromTwo(workers, idx1, idx2) {
|
|
@@ -739,6 +765,11 @@ export class ProcessPool {
|
|
|
739
765
|
case 'recovery':
|
|
740
766
|
this.logger.info(payload, 'Worker health check recovered');
|
|
741
767
|
break;
|
|
768
|
+
default: {
|
|
769
|
+
const unexpected = decision.level;
|
|
770
|
+
this.logger.warn({ ...payload, level: unexpected }, 'Unhandled failure-tracker level');
|
|
771
|
+
break;
|
|
772
|
+
}
|
|
742
773
|
}
|
|
743
774
|
}
|
|
744
775
|
setupHealthMonitoring() {
|
|
@@ -776,9 +807,6 @@ export class ProcessPool {
|
|
|
776
807
|
this.emitHealthDecision(this.healthFailures.recordSuccess(worker.id), worker.id, undefined);
|
|
777
808
|
}
|
|
778
809
|
}
|
|
779
|
-
if ('__getMetrics' in worker.proxy) {
|
|
780
|
-
worker.metrics = await worker.proxy.__getMetrics();
|
|
781
|
-
}
|
|
782
810
|
}
|
|
783
811
|
catch (error) {
|
|
784
812
|
const decision = this.healthFailures.recordFailure(worker.id);
|
|
@@ -786,6 +814,16 @@ export class ProcessPool {
|
|
|
786
814
|
worker.health = 'unhealthy';
|
|
787
815
|
worker.consecutiveFailures++;
|
|
788
816
|
}
|
|
817
|
+
try {
|
|
818
|
+
const raw = (await worker.proxy
|
|
819
|
+
.__getMetrics?.());
|
|
820
|
+
if (raw) {
|
|
821
|
+
worker.metrics = this.withDerivedPercentages(worker.id, raw);
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
catch (error) {
|
|
825
|
+
this.logger.debug({ workerId: worker.id, error }, 'Worker metrics unavailable this tick');
|
|
826
|
+
}
|
|
789
827
|
return previousHealth !== worker.health;
|
|
790
828
|
};
|
|
791
829
|
for (let i = 0; i < workers.length; i += HEALTH_CHECK_CONCURRENCY) {
|
|
@@ -798,7 +836,7 @@ export class ProcessPool {
|
|
|
798
836
|
if (healthChanged) {
|
|
799
837
|
this.invalidateHealthyWorkersCache();
|
|
800
838
|
}
|
|
801
|
-
const unhealthyThreshold = this.poolOptions.healthCheck?.unhealthyThreshold
|
|
839
|
+
const unhealthyThreshold = this.poolOptions.healthCheck?.unhealthyThreshold ?? 3;
|
|
802
840
|
for (const worker of workers) {
|
|
803
841
|
if (worker.health === 'unhealthy' && worker.consecutiveFailures >= unhealthyThreshold && !this.isShuttingDown) {
|
|
804
842
|
this.logger.info({
|
|
@@ -820,45 +858,68 @@ export class ProcessPool {
|
|
|
820
858
|
if (!this.poolOptions.autoScale?.enabled)
|
|
821
859
|
return;
|
|
822
860
|
const checkInterval = this.poolOptions.autoScale?.checkInterval ?? 10_000;
|
|
861
|
+
const FAILURE_BACKOFF_MAX_MS = 5 * 60_000;
|
|
823
862
|
this.autoScaleTimer = setInterval(async () => {
|
|
824
863
|
if (this.isShuttingDown || this.isDraining)
|
|
825
864
|
return;
|
|
826
865
|
if (this.autoScaleRunning)
|
|
827
866
|
return;
|
|
867
|
+
const now = Date.now();
|
|
868
|
+
if (now < this.autoScaleNextRetryAt)
|
|
869
|
+
return;
|
|
828
870
|
this.autoScaleRunning = true;
|
|
829
871
|
try {
|
|
830
|
-
const
|
|
831
|
-
const cooldownPeriod = this.poolOptions.autoScale?.cooldownPeriod || 60000;
|
|
872
|
+
const cooldownPeriod = this.poolOptions.autoScale?.cooldownPeriod ?? 60000;
|
|
832
873
|
if (now - this.lastScaleCheck < cooldownPeriod)
|
|
833
874
|
return;
|
|
834
875
|
const metrics = this.metrics;
|
|
835
876
|
const config = this.poolOptions.autoScale;
|
|
836
877
|
const queueThreshold = config.queueThreshold ?? 50;
|
|
837
|
-
const
|
|
838
|
-
|
|
839
|
-
|
|
878
|
+
const targetCpu = config.targetCPU ?? 70;
|
|
879
|
+
const targetMemory = config.targetMemory ?? 80;
|
|
880
|
+
const cpuOverTarget = metrics.cpuPercent !== undefined && metrics.cpuPercent > targetCpu;
|
|
881
|
+
const memoryOverTarget = metrics.memoryPercent !== undefined && metrics.memoryPercent > targetMemory;
|
|
882
|
+
const shouldScaleUp = cpuOverTarget ||
|
|
883
|
+
memoryOverTarget ||
|
|
884
|
+
(metrics.saturation ?? 0) > (config.scaleUpThreshold ?? 0.8) ||
|
|
840
885
|
this.queue.length > queueThreshold;
|
|
841
|
-
const scaleDownCpu =
|
|
842
|
-
const scaleDownMemory =
|
|
843
|
-
const
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
886
|
+
const scaleDownCpu = targetCpu * 0.4;
|
|
887
|
+
const scaleDownMemory = targetMemory * 0.5;
|
|
888
|
+
const cpuIdle = metrics.cpuPercent !== undefined && metrics.cpuPercent < scaleDownCpu;
|
|
889
|
+
const memoryIdle = metrics.memoryPercent !== undefined && metrics.memoryPercent < scaleDownMemory;
|
|
890
|
+
const shouldScaleDown = cpuIdle &&
|
|
891
|
+
memoryIdle &&
|
|
892
|
+
(metrics.saturation ?? 0) < (config.scaleDownThreshold ?? 0.3);
|
|
893
|
+
if (shouldScaleUp && this.workers.size < (config.max ?? 10)) {
|
|
847
894
|
const queuePressure = this.queue.length / queueThreshold;
|
|
848
895
|
const scaleStep = Math.max(1, Math.ceil(queuePressure));
|
|
849
896
|
const maxWorkers = config.max ?? 10;
|
|
850
897
|
const newSize = Math.min(this.workers.size + scaleStep, maxWorkers);
|
|
851
898
|
await this.scale(newSize);
|
|
852
899
|
this.lastScaleCheck = now;
|
|
900
|
+
this.autoScaleConsecutiveFailures = 0;
|
|
853
901
|
}
|
|
854
|
-
else if (shouldScaleDown && this.workers.size > (config.min
|
|
855
|
-
const newSize = Math.max(this.workers.size - 1, config.min
|
|
902
|
+
else if (shouldScaleDown && this.workers.size > (config.min ?? 1)) {
|
|
903
|
+
const newSize = Math.max(this.workers.size - 1, config.min ?? 1);
|
|
856
904
|
await this.scale(newSize);
|
|
857
905
|
this.lastScaleCheck = now;
|
|
906
|
+
this.autoScaleConsecutiveFailures = 0;
|
|
907
|
+
}
|
|
908
|
+
else {
|
|
909
|
+
this.autoScaleConsecutiveFailures = 0;
|
|
858
910
|
}
|
|
859
911
|
}
|
|
860
912
|
catch (error) {
|
|
861
|
-
this.
|
|
913
|
+
this.autoScaleConsecutiveFailures += 1;
|
|
914
|
+
const failures = this.autoScaleConsecutiveFailures;
|
|
915
|
+
const backoffMultiplier = Math.min(2 ** (failures - 1), Math.ceil(FAILURE_BACKOFF_MAX_MS / checkInterval));
|
|
916
|
+
const backoffMs = Math.min(checkInterval * backoffMultiplier, FAILURE_BACKOFF_MAX_MS);
|
|
917
|
+
this.autoScaleNextRetryAt = Date.now() + backoffMs;
|
|
918
|
+
this.logger.error({
|
|
919
|
+
err: error instanceof Error ? error : new Error(String(error)),
|
|
920
|
+
consecutiveFailures: failures,
|
|
921
|
+
nextRetryInMs: backoffMs,
|
|
922
|
+
}, 'Error during auto-scaling check');
|
|
862
923
|
}
|
|
863
924
|
finally {
|
|
864
925
|
this.autoScaleRunning = false;
|
|
@@ -924,10 +985,11 @@ export class ProcessPool {
|
|
|
924
985
|
try {
|
|
925
986
|
if ('__getMetrics' in worker.proxy) {
|
|
926
987
|
const metrics = await worker.proxy.__getMetrics();
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
worker.
|
|
930
|
-
|
|
988
|
+
const footprint = typeof metrics?.memoryRss === 'number' ? metrics.memoryRss : metrics?.memory;
|
|
989
|
+
if (typeof footprint === 'number') {
|
|
990
|
+
worker.memoryUsage = footprint;
|
|
991
|
+
worker.peakMemoryUsage = Math.max(worker.peakMemoryUsage, footprint);
|
|
992
|
+
const memoryRatio = footprint / this.memoryLimitBytes;
|
|
931
993
|
if (memoryRatio > 0.9) {
|
|
932
994
|
worker.weight = 10;
|
|
933
995
|
}
|
|
@@ -1024,10 +1086,11 @@ export class ProcessPool {
|
|
|
1024
1086
|
}
|
|
1025
1087
|
async sendHeartbeat(worker) {
|
|
1026
1088
|
try {
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
return
|
|
1089
|
+
const getHealth = worker.proxy.__getHealth;
|
|
1090
|
+
if (typeof getHealth !== 'function') {
|
|
1091
|
+
return false;
|
|
1030
1092
|
}
|
|
1093
|
+
await getHealth.call(worker.proxy);
|
|
1031
1094
|
return true;
|
|
1032
1095
|
}
|
|
1033
1096
|
catch {
|
|
@@ -1086,6 +1149,26 @@ export class ProcessPool {
|
|
|
1086
1149
|
const duration = (Date.now() - firstEntry.timestamp) / 1000;
|
|
1087
1150
|
return duration > 0 ? this.successfulRequests / duration : 0;
|
|
1088
1151
|
}
|
|
1152
|
+
withDerivedPercentages(workerId, raw) {
|
|
1153
|
+
const now = Date.now();
|
|
1154
|
+
const derived = { ...raw };
|
|
1155
|
+
if (typeof raw.cpu === 'number') {
|
|
1156
|
+
const previous = this.cpuSamples.get(workerId);
|
|
1157
|
+
if (previous) {
|
|
1158
|
+
const elapsedSeconds = (now - previous.at) / 1000;
|
|
1159
|
+
const cpuSeconds = raw.cpu - previous.seconds;
|
|
1160
|
+
if (elapsedSeconds > 0 && cpuSeconds >= 0) {
|
|
1161
|
+
derived.cpuPercent = (cpuSeconds / elapsedSeconds) * 100;
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
this.cpuSamples.set(workerId, { seconds: raw.cpu, at: now });
|
|
1165
|
+
}
|
|
1166
|
+
const footprint = typeof raw.memoryRss === 'number' ? raw.memoryRss : raw.memory;
|
|
1167
|
+
if (typeof footprint === 'number' && this.memoryLimitBytes > 0) {
|
|
1168
|
+
derived.memoryPercent = (footprint / this.memoryLimitBytes) * 100;
|
|
1169
|
+
}
|
|
1170
|
+
return derived;
|
|
1171
|
+
}
|
|
1089
1172
|
calculateSaturation() {
|
|
1090
1173
|
const totalCapacity = this.workers.size * 2;
|
|
1091
1174
|
const currentLoad = this.active + this.queue.length;
|