@naturalcycles/backend-lib 9.84.2 → 9.85.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -15,6 +15,12 @@ import type { Integer, NonNegativeInteger, NumberOfMilliseconds, NumberOfPercent
|
|
|
15
15
|
* and the interval timer fires late (see `intervalMs`).
|
|
16
16
|
* Use `pause()` / `resume()` to keep such periods out of the delay samples,
|
|
17
17
|
* e.g. pause when the last in-flight request finishes and resume when the next one arrives.
|
|
18
|
+
*
|
|
19
|
+
* Besides the delay histogram, a watchdog timer measures the largest stall of each interval
|
|
20
|
+
* together with the CPU the event loop thread consumed across it (`stallMs` / `stallCpuMs`),
|
|
21
|
+
* which tells a stall caused by own work (sync code, GC) from one caused by the thread not
|
|
22
|
+
* being scheduled.
|
|
23
|
+
* The longest single GC pause is reported too (`gcMaxTime` / `gcMaxKind`).
|
|
18
24
|
*/
|
|
19
25
|
export declare class EventLoopMonitor {
|
|
20
26
|
constructor(cfg?: EventLoopMonitorCfg);
|
|
@@ -23,6 +29,22 @@ export declare class EventLoopMonitor {
|
|
|
23
29
|
private lastElu;
|
|
24
30
|
private lastReportedAt;
|
|
25
31
|
private paused;
|
|
32
|
+
/**
|
|
33
|
+
* performance.now() of the last resume() (or construction), to tell gc pauses that overlapped
|
|
34
|
+
* a paused period from those that ran while sampling was active.
|
|
35
|
+
*/
|
|
36
|
+
private resumedAt;
|
|
37
|
+
private readonly watchdogInterval;
|
|
38
|
+
private watchdog?;
|
|
39
|
+
/**
|
|
40
|
+
* performance.now() and process CPU (microseconds) at the last watchdog tick.
|
|
41
|
+
*/
|
|
42
|
+
private watchdogAt;
|
|
43
|
+
private watchdogCpuUs;
|
|
44
|
+
private maxStallMs;
|
|
45
|
+
private maxStallCpuMs;
|
|
46
|
+
private gcMaxTime;
|
|
47
|
+
private gcMaxKind?;
|
|
26
48
|
/**
|
|
27
49
|
* Undefined until the first interval has completed.
|
|
28
50
|
*/
|
|
@@ -32,7 +54,7 @@ export declare class EventLoopMonitor {
|
|
|
32
54
|
gcTotalTime: number;
|
|
33
55
|
stop(): void;
|
|
34
56
|
/**
|
|
35
|
-
* Stops sampling event loop delay until `resume()` is called.
|
|
57
|
+
* Stops sampling event loop delay (and the stall watchdog) until `resume()` is called.
|
|
36
58
|
* GC and ELU keep being tracked, and the samples recorded so far are kept.
|
|
37
59
|
*
|
|
38
60
|
* Meant for periods when the process is known to be CPU-throttled, e.g. a CloudRun instance
|
|
@@ -43,16 +65,31 @@ export declare class EventLoopMonitor {
|
|
|
43
65
|
/**
|
|
44
66
|
* Resumes sampling event loop delay after `pause()`.
|
|
45
67
|
* Safe to call from within the event that ended the throttled period (e.g. the middleware of
|
|
46
|
-
* the request that woke the instance): the histogram
|
|
47
|
-
* so the stall that preceded it in the same loop iteration is not sampled.
|
|
68
|
+
* the request that woke the instance): the histogram and the watchdog restart their clocks
|
|
69
|
+
* from this moment, so the stall that preceded it in the same loop iteration is not sampled.
|
|
48
70
|
*/
|
|
49
71
|
resume(): void;
|
|
72
|
+
private startWatchdog;
|
|
73
|
+
private stopWatchdog;
|
|
74
|
+
/**
|
|
75
|
+
* A tick that fires late means the loop could not turn in between: a long synchronous callback,
|
|
76
|
+
* a long GC pause, or the process not being scheduled at all. The CPU consumed across the gap
|
|
77
|
+
* tells these apart. Node reschedules a late interval from the actual fire time, so one stall
|
|
78
|
+
* produces exactly one late tick.
|
|
79
|
+
*/
|
|
80
|
+
private onWatchdogTick;
|
|
50
81
|
}
|
|
82
|
+
export type GcKind = 'major' | 'minor' | 'incremental' | 'weakcb' | 'unknown';
|
|
51
83
|
export interface EventLoopMonitorCfg {
|
|
52
84
|
/**
|
|
53
85
|
* Defaults to 60_000 ms
|
|
54
86
|
*/
|
|
55
87
|
measureInterval?: NumberOfMilliseconds;
|
|
88
|
+
/**
|
|
89
|
+
* Period of the stall watchdog timer, see `EventLoopStats.stallMs`.
|
|
90
|
+
* Defaults to 1000 ms.
|
|
91
|
+
*/
|
|
92
|
+
watchdogInterval?: NumberOfMilliseconds;
|
|
56
93
|
/**
|
|
57
94
|
* Callback to be invoked with EventLoopStats.
|
|
58
95
|
* Called every `measureInterval` milliseconds.
|
|
@@ -91,6 +128,31 @@ export interface EventLoopStats {
|
|
|
91
128
|
* too "lossy")
|
|
92
129
|
*/
|
|
93
130
|
gcCPU: Integer;
|
|
131
|
+
/**
|
|
132
|
+
* Longest single GC pause in the last interval, among the pauses that ran while sampling was
|
|
133
|
+
* active (a pause overlapping a paused, CPU-throttled period is stretched by the throttling).
|
|
134
|
+
* `gcCount` and `gcTotalTime` are not gated.
|
|
135
|
+
*/
|
|
136
|
+
gcMaxTime: NumberOfMilliseconds;
|
|
137
|
+
/**
|
|
138
|
+
* Kind of the longest GC pause. Undefined when no such GC pause ran in the interval.
|
|
139
|
+
*/
|
|
140
|
+
gcMaxKind: GcKind | undefined;
|
|
141
|
+
/**
|
|
142
|
+
* Largest stall of the interval, as seen by the watchdog timer: how late its tick fired,
|
|
143
|
+
* 0 when every tick was on time. Covers anything that kept the loop from turning:
|
|
144
|
+
* a long synchronous callback, a long GC pause, or the process not being scheduled at all.
|
|
145
|
+
* Underestimates the stall by up to `watchdogInterval` (the tick may have been due anywhere
|
|
146
|
+
* within its period when the stall began). Not measured while paused.
|
|
147
|
+
*/
|
|
148
|
+
stallMs: NumberOfMilliseconds;
|
|
149
|
+
/**
|
|
150
|
+
* CPU time of the event loop thread consumed between the watchdog tick before the largest
|
|
151
|
+
* stall and the late tick that measured it.
|
|
152
|
+
* Close to `stallMs` means the thread was running (own code or GC, see `gcMaxTime`),
|
|
153
|
+
* close to 0 means it was not scheduled (platform pause, CPU throttling).
|
|
154
|
+
*/
|
|
155
|
+
stallCpuMs: NumberOfMilliseconds;
|
|
94
156
|
/**
|
|
95
157
|
* Actual wall-clock duration of the interval covered by these stats.
|
|
96
158
|
* Normally equals `measureInterval`, but is longer when the timer fired late,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { monitorEventLoopDelay, performance, PerformanceObserver } from 'node:perf_hooks';
|
|
1
|
+
import { constants, monitorEventLoopDelay, performance, PerformanceObserver } from 'node:perf_hooks';
|
|
2
2
|
/*
|
|
3
3
|
Healthy gcCPU Ranges
|
|
4
4
|
┌────────┬──────────────────────────────────────────────┐
|
|
@@ -31,10 +31,17 @@ import { monitorEventLoopDelay, performance, PerformanceObserver } from 'node:pe
|
|
|
31
31
|
* and the interval timer fires late (see `intervalMs`).
|
|
32
32
|
* Use `pause()` / `resume()` to keep such periods out of the delay samples,
|
|
33
33
|
* e.g. pause when the last in-flight request finishes and resume when the next one arrives.
|
|
34
|
+
*
|
|
35
|
+
* Besides the delay histogram, a watchdog timer measures the largest stall of each interval
|
|
36
|
+
* together with the CPU the event loop thread consumed across it (`stallMs` / `stallCpuMs`),
|
|
37
|
+
* which tells a stall caused by own work (sync code, GC) from one caused by the thread not
|
|
38
|
+
* being scheduled.
|
|
39
|
+
* The longest single GC pause is reported too (`gcMaxTime` / `gcMaxKind`).
|
|
34
40
|
*/
|
|
35
41
|
export class EventLoopMonitor {
|
|
36
42
|
constructor(cfg = {}) {
|
|
37
|
-
const { measureInterval = 60_000 } = cfg;
|
|
43
|
+
const { measureInterval = 60_000, watchdogInterval = 1000 } = cfg;
|
|
44
|
+
this.watchdogInterval = watchdogInterval;
|
|
38
45
|
const eldOptions = {
|
|
39
46
|
// Node >=24.19 / >=26.5: sample once per event loop iteration (uv_prepare/uv_check hooks),
|
|
40
47
|
// which measures actual per-iteration delay instead of timer jitter.
|
|
@@ -46,16 +53,26 @@ export class EventLoopMonitor {
|
|
|
46
53
|
this.eld.enable();
|
|
47
54
|
this.lastElu = performance.eventLoopUtilization();
|
|
48
55
|
this.lastReportedAt = performance.now();
|
|
56
|
+
this.resumedAt = this.lastReportedAt;
|
|
49
57
|
this.po = new PerformanceObserver(list => {
|
|
50
58
|
for (const entry of list.getEntries()) {
|
|
51
59
|
this.gcCount++;
|
|
52
60
|
this.gcTotalTime += entry.duration;
|
|
61
|
+
// A pause overlapping a paused (CPU-throttled) period is stretched by the throttling and
|
|
62
|
+
// says nothing about GC: only pauses that ran while sampling was active compete for the max
|
|
63
|
+
if (this.paused || entry.startTime < this.resumedAt)
|
|
64
|
+
continue;
|
|
65
|
+
if (entry.duration > this.gcMaxTime) {
|
|
66
|
+
this.gcMaxTime = entry.duration;
|
|
67
|
+
this.gcMaxKind = getGcKind(entry);
|
|
68
|
+
}
|
|
53
69
|
}
|
|
54
70
|
});
|
|
55
71
|
this.po.observe({ entryTypes: ['gc'] });
|
|
72
|
+
this.startWatchdog();
|
|
56
73
|
this.interval = setInterval(() => {
|
|
57
74
|
// Delay stats are reported in **nanoseconds**
|
|
58
|
-
const { eld, gcCount, gcTotalTime } = this;
|
|
75
|
+
const { eld, gcCount, gcTotalTime, gcMaxKind } = this;
|
|
59
76
|
const p50 = Math.round(eld.percentile(50) / 1e6);
|
|
60
77
|
const p90 = Math.round(eld.percentile(90) / 1e6);
|
|
61
78
|
const p99 = Math.round(eld.percentile(99) / 1e6);
|
|
@@ -72,6 +89,9 @@ export class EventLoopMonitor {
|
|
|
72
89
|
this.lastElu = currentElu;
|
|
73
90
|
const elu = Math.round(deltaElu.utilization * 100);
|
|
74
91
|
const gcCPU = Math.round((gcTotalTime / intervalMs) * 10_000);
|
|
92
|
+
const gcMaxTime = Math.round(this.gcMaxTime);
|
|
93
|
+
const stallMs = Math.round(this.maxStallMs);
|
|
94
|
+
const stallCpuMs = Math.round(this.maxStallCpuMs);
|
|
75
95
|
this.lastStats = {
|
|
76
96
|
p50,
|
|
77
97
|
p90,
|
|
@@ -82,12 +102,20 @@ export class EventLoopMonitor {
|
|
|
82
102
|
gcCount,
|
|
83
103
|
gcTotalTime,
|
|
84
104
|
gcCPU,
|
|
105
|
+
gcMaxTime,
|
|
106
|
+
gcMaxKind,
|
|
107
|
+
stallMs,
|
|
108
|
+
stallCpuMs,
|
|
85
109
|
intervalMs,
|
|
86
110
|
};
|
|
87
111
|
cfg.onStats?.(this.lastStats);
|
|
88
112
|
eld.reset();
|
|
89
113
|
this.gcCount = 0;
|
|
90
114
|
this.gcTotalTime = 0;
|
|
115
|
+
this.gcMaxTime = 0;
|
|
116
|
+
this.gcMaxKind = undefined;
|
|
117
|
+
this.maxStallMs = 0;
|
|
118
|
+
this.maxStallCpuMs = 0;
|
|
91
119
|
}, measureInterval);
|
|
92
120
|
}
|
|
93
121
|
interval;
|
|
@@ -95,6 +123,22 @@ export class EventLoopMonitor {
|
|
|
95
123
|
lastElu;
|
|
96
124
|
lastReportedAt;
|
|
97
125
|
paused = false;
|
|
126
|
+
/**
|
|
127
|
+
* performance.now() of the last resume() (or construction), to tell gc pauses that overlapped
|
|
128
|
+
* a paused period from those that ran while sampling was active.
|
|
129
|
+
*/
|
|
130
|
+
resumedAt;
|
|
131
|
+
watchdogInterval;
|
|
132
|
+
watchdog;
|
|
133
|
+
/**
|
|
134
|
+
* performance.now() and process CPU (microseconds) at the last watchdog tick.
|
|
135
|
+
*/
|
|
136
|
+
watchdogAt = 0;
|
|
137
|
+
watchdogCpuUs = 0;
|
|
138
|
+
maxStallMs = 0;
|
|
139
|
+
maxStallCpuMs = 0;
|
|
140
|
+
gcMaxTime = 0;
|
|
141
|
+
gcMaxKind;
|
|
98
142
|
/**
|
|
99
143
|
* Undefined until the first interval has completed.
|
|
100
144
|
*/
|
|
@@ -104,11 +148,12 @@ export class EventLoopMonitor {
|
|
|
104
148
|
gcTotalTime = 0;
|
|
105
149
|
stop() {
|
|
106
150
|
clearInterval(this.interval);
|
|
151
|
+
this.stopWatchdog();
|
|
107
152
|
this.eld.disable();
|
|
108
153
|
this.po.disconnect();
|
|
109
154
|
}
|
|
110
155
|
/**
|
|
111
|
-
* Stops sampling event loop delay until `resume()` is called.
|
|
156
|
+
* Stops sampling event loop delay (and the stall watchdog) until `resume()` is called.
|
|
112
157
|
* GC and ELU keep being tracked, and the samples recorded so far are kept.
|
|
113
158
|
*
|
|
114
159
|
* Meant for periods when the process is known to be CPU-throttled, e.g. a CloudRun instance
|
|
@@ -118,17 +163,69 @@ export class EventLoopMonitor {
|
|
|
118
163
|
pause() {
|
|
119
164
|
this.paused = true;
|
|
120
165
|
this.eld.disable();
|
|
166
|
+
this.stopWatchdog();
|
|
121
167
|
}
|
|
122
168
|
/**
|
|
123
169
|
* Resumes sampling event loop delay after `pause()`.
|
|
124
170
|
* Safe to call from within the event that ended the throttled period (e.g. the middleware of
|
|
125
|
-
* the request that woke the instance): the histogram
|
|
126
|
-
* so the stall that preceded it in the same loop iteration is not sampled.
|
|
171
|
+
* the request that woke the instance): the histogram and the watchdog restart their clocks
|
|
172
|
+
* from this moment, so the stall that preceded it in the same loop iteration is not sampled.
|
|
127
173
|
*/
|
|
128
174
|
resume() {
|
|
129
175
|
if (!this.paused)
|
|
130
176
|
return;
|
|
131
177
|
this.paused = false;
|
|
178
|
+
this.resumedAt = performance.now();
|
|
132
179
|
this.eld.enable();
|
|
180
|
+
this.startWatchdog();
|
|
181
|
+
}
|
|
182
|
+
startWatchdog() {
|
|
183
|
+
this.watchdogAt = performance.now();
|
|
184
|
+
this.watchdogCpuUs = getCpuUs();
|
|
185
|
+
this.watchdog = setInterval(() => this.onWatchdogTick(), this.watchdogInterval);
|
|
186
|
+
// Must not keep the process alive on its own (the main interval already does, until `stop()`)
|
|
187
|
+
this.watchdog.unref();
|
|
188
|
+
}
|
|
189
|
+
stopWatchdog() {
|
|
190
|
+
clearInterval(this.watchdog);
|
|
191
|
+
this.watchdog = undefined;
|
|
133
192
|
}
|
|
193
|
+
/**
|
|
194
|
+
* A tick that fires late means the loop could not turn in between: a long synchronous callback,
|
|
195
|
+
* a long GC pause, or the process not being scheduled at all. The CPU consumed across the gap
|
|
196
|
+
* tells these apart. Node reschedules a late interval from the actual fire time, so one stall
|
|
197
|
+
* produces exactly one late tick.
|
|
198
|
+
*/
|
|
199
|
+
onWatchdogTick() {
|
|
200
|
+
const now = performance.now();
|
|
201
|
+
const cpuUs = getCpuUs();
|
|
202
|
+
const stallMs = now - this.watchdogAt - this.watchdogInterval;
|
|
203
|
+
if (stallMs > this.maxStallMs) {
|
|
204
|
+
this.maxStallMs = stallMs;
|
|
205
|
+
this.maxStallCpuMs = (cpuUs - this.watchdogCpuUs) / 1000;
|
|
206
|
+
}
|
|
207
|
+
this.watchdogAt = now;
|
|
208
|
+
this.watchdogCpuUs = cpuUs;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
function getGcKind(entry) {
|
|
212
|
+
// `detail` is only declared on Node's PerformanceNodeEntry type, which getEntries() doesn't return
|
|
213
|
+
const kind = entry.detail?.kind;
|
|
214
|
+
if (kind === constants.NODE_PERFORMANCE_GC_MAJOR)
|
|
215
|
+
return 'major';
|
|
216
|
+
if (kind === constants.NODE_PERFORMANCE_GC_MINOR)
|
|
217
|
+
return 'minor';
|
|
218
|
+
if (kind === constants.NODE_PERFORMANCE_GC_INCREMENTAL)
|
|
219
|
+
return 'incremental';
|
|
220
|
+
if (kind === constants.NODE_PERFORMANCE_GC_WEAKCB)
|
|
221
|
+
return 'weakcb';
|
|
222
|
+
return 'unknown';
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* CPU time of the current (event loop) thread in microseconds, so that worker threads,
|
|
226
|
+
* the libuv threadpool and parallel GC helpers don't count.
|
|
227
|
+
*/
|
|
228
|
+
function getCpuUs() {
|
|
229
|
+
const { user, system } = process.threadCpuUsage();
|
|
230
|
+
return user + system;
|
|
134
231
|
}
|
package/package.json
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
1
|
+
import type {
|
|
2
|
+
ELDHistogram,
|
|
3
|
+
EventLoopMonitorOptions,
|
|
4
|
+
EventLoopUtilization,
|
|
5
|
+
PerformanceEntry,
|
|
6
|
+
} from 'node:perf_hooks'
|
|
7
|
+
import { constants, monitorEventLoopDelay, performance, PerformanceObserver } from 'node:perf_hooks'
|
|
3
8
|
import type {
|
|
4
9
|
Integer,
|
|
5
10
|
NonNegativeInteger,
|
|
@@ -40,10 +45,17 @@ import type {
|
|
|
40
45
|
* and the interval timer fires late (see `intervalMs`).
|
|
41
46
|
* Use `pause()` / `resume()` to keep such periods out of the delay samples,
|
|
42
47
|
* e.g. pause when the last in-flight request finishes and resume when the next one arrives.
|
|
48
|
+
*
|
|
49
|
+
* Besides the delay histogram, a watchdog timer measures the largest stall of each interval
|
|
50
|
+
* together with the CPU the event loop thread consumed across it (`stallMs` / `stallCpuMs`),
|
|
51
|
+
* which tells a stall caused by own work (sync code, GC) from one caused by the thread not
|
|
52
|
+
* being scheduled.
|
|
53
|
+
* The longest single GC pause is reported too (`gcMaxTime` / `gcMaxKind`).
|
|
43
54
|
*/
|
|
44
55
|
export class EventLoopMonitor {
|
|
45
56
|
constructor(cfg: EventLoopMonitorCfg = {}) {
|
|
46
|
-
const { measureInterval = 60_000 } = cfg
|
|
57
|
+
const { measureInterval = 60_000, watchdogInterval = 1000 } = cfg
|
|
58
|
+
this.watchdogInterval = watchdogInterval
|
|
47
59
|
|
|
48
60
|
const eldOptions: EventLoopMonitorOptions = {
|
|
49
61
|
// Node >=24.19 / >=26.5: sample once per event loop iteration (uv_prepare/uv_check hooks),
|
|
@@ -57,19 +69,29 @@ export class EventLoopMonitor {
|
|
|
57
69
|
|
|
58
70
|
this.lastElu = performance.eventLoopUtilization()
|
|
59
71
|
this.lastReportedAt = performance.now()
|
|
72
|
+
this.resumedAt = this.lastReportedAt
|
|
60
73
|
|
|
61
74
|
this.po = new PerformanceObserver(list => {
|
|
62
75
|
for (const entry of list.getEntries()) {
|
|
63
76
|
this.gcCount++
|
|
64
77
|
this.gcTotalTime += entry.duration
|
|
78
|
+
// A pause overlapping a paused (CPU-throttled) period is stretched by the throttling and
|
|
79
|
+
// says nothing about GC: only pauses that ran while sampling was active compete for the max
|
|
80
|
+
if (this.paused || entry.startTime < this.resumedAt) continue
|
|
81
|
+
if (entry.duration > this.gcMaxTime) {
|
|
82
|
+
this.gcMaxTime = entry.duration
|
|
83
|
+
this.gcMaxKind = getGcKind(entry)
|
|
84
|
+
}
|
|
65
85
|
}
|
|
66
86
|
})
|
|
67
87
|
|
|
68
88
|
this.po.observe({ entryTypes: ['gc'] })
|
|
69
89
|
|
|
90
|
+
this.startWatchdog()
|
|
91
|
+
|
|
70
92
|
this.interval = setInterval(() => {
|
|
71
93
|
// Delay stats are reported in **nanoseconds**
|
|
72
|
-
const { eld, gcCount, gcTotalTime } = this
|
|
94
|
+
const { eld, gcCount, gcTotalTime, gcMaxKind } = this
|
|
73
95
|
const p50 = Math.round(eld.percentile(50) / 1e6)
|
|
74
96
|
const p90 = Math.round(eld.percentile(90) / 1e6)
|
|
75
97
|
const p99 = Math.round(eld.percentile(99) / 1e6)
|
|
@@ -89,6 +111,9 @@ export class EventLoopMonitor {
|
|
|
89
111
|
|
|
90
112
|
const elu = Math.round(deltaElu.utilization * 100)
|
|
91
113
|
const gcCPU = Math.round((gcTotalTime / intervalMs) * 10_000)
|
|
114
|
+
const gcMaxTime = Math.round(this.gcMaxTime)
|
|
115
|
+
const stallMs = Math.round(this.maxStallMs)
|
|
116
|
+
const stallCpuMs = Math.round(this.maxStallCpuMs)
|
|
92
117
|
|
|
93
118
|
this.lastStats = {
|
|
94
119
|
p50,
|
|
@@ -100,6 +125,10 @@ export class EventLoopMonitor {
|
|
|
100
125
|
gcCount,
|
|
101
126
|
gcTotalTime,
|
|
102
127
|
gcCPU,
|
|
128
|
+
gcMaxTime,
|
|
129
|
+
gcMaxKind,
|
|
130
|
+
stallMs,
|
|
131
|
+
stallCpuMs,
|
|
103
132
|
intervalMs,
|
|
104
133
|
}
|
|
105
134
|
|
|
@@ -108,6 +137,10 @@ export class EventLoopMonitor {
|
|
|
108
137
|
eld.reset()
|
|
109
138
|
this.gcCount = 0
|
|
110
139
|
this.gcTotalTime = 0
|
|
140
|
+
this.gcMaxTime = 0
|
|
141
|
+
this.gcMaxKind = undefined
|
|
142
|
+
this.maxStallMs = 0
|
|
143
|
+
this.maxStallCpuMs = 0
|
|
111
144
|
}, measureInterval)
|
|
112
145
|
}
|
|
113
146
|
|
|
@@ -116,6 +149,22 @@ export class EventLoopMonitor {
|
|
|
116
149
|
private lastElu: EventLoopUtilization
|
|
117
150
|
private lastReportedAt: NumberOfMilliseconds
|
|
118
151
|
private paused = false
|
|
152
|
+
/**
|
|
153
|
+
* performance.now() of the last resume() (or construction), to tell gc pauses that overlapped
|
|
154
|
+
* a paused period from those that ran while sampling was active.
|
|
155
|
+
*/
|
|
156
|
+
private resumedAt: NumberOfMilliseconds
|
|
157
|
+
private readonly watchdogInterval: NumberOfMilliseconds
|
|
158
|
+
private watchdog?: NodeJS.Timeout
|
|
159
|
+
/**
|
|
160
|
+
* performance.now() and process CPU (microseconds) at the last watchdog tick.
|
|
161
|
+
*/
|
|
162
|
+
private watchdogAt = 0
|
|
163
|
+
private watchdogCpuUs = 0
|
|
164
|
+
private maxStallMs = 0
|
|
165
|
+
private maxStallCpuMs = 0
|
|
166
|
+
private gcMaxTime = 0
|
|
167
|
+
private gcMaxKind?: GcKind
|
|
119
168
|
/**
|
|
120
169
|
* Undefined until the first interval has completed.
|
|
121
170
|
*/
|
|
@@ -126,12 +175,13 @@ export class EventLoopMonitor {
|
|
|
126
175
|
|
|
127
176
|
stop(): void {
|
|
128
177
|
clearInterval(this.interval)
|
|
178
|
+
this.stopWatchdog()
|
|
129
179
|
this.eld.disable()
|
|
130
180
|
this.po.disconnect()
|
|
131
181
|
}
|
|
132
182
|
|
|
133
183
|
/**
|
|
134
|
-
* Stops sampling event loop delay until `resume()` is called.
|
|
184
|
+
* Stops sampling event loop delay (and the stall watchdog) until `resume()` is called.
|
|
135
185
|
* GC and ELU keep being tracked, and the samples recorded so far are kept.
|
|
136
186
|
*
|
|
137
187
|
* Meant for periods when the process is known to be CPU-throttled, e.g. a CloudRun instance
|
|
@@ -141,28 +191,88 @@ export class EventLoopMonitor {
|
|
|
141
191
|
pause(): void {
|
|
142
192
|
this.paused = true
|
|
143
193
|
this.eld.disable()
|
|
194
|
+
this.stopWatchdog()
|
|
144
195
|
}
|
|
145
196
|
|
|
146
197
|
/**
|
|
147
198
|
* Resumes sampling event loop delay after `pause()`.
|
|
148
199
|
* Safe to call from within the event that ended the throttled period (e.g. the middleware of
|
|
149
|
-
* the request that woke the instance): the histogram
|
|
150
|
-
* so the stall that preceded it in the same loop iteration is not sampled.
|
|
200
|
+
* the request that woke the instance): the histogram and the watchdog restart their clocks
|
|
201
|
+
* from this moment, so the stall that preceded it in the same loop iteration is not sampled.
|
|
151
202
|
*/
|
|
152
203
|
resume(): void {
|
|
153
204
|
if (!this.paused) return
|
|
154
205
|
this.paused = false
|
|
206
|
+
this.resumedAt = performance.now()
|
|
155
207
|
this.eld.enable()
|
|
208
|
+
this.startWatchdog()
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
private startWatchdog(): void {
|
|
212
|
+
this.watchdogAt = performance.now()
|
|
213
|
+
this.watchdogCpuUs = getCpuUs()
|
|
214
|
+
this.watchdog = setInterval(() => this.onWatchdogTick(), this.watchdogInterval)
|
|
215
|
+
// Must not keep the process alive on its own (the main interval already does, until `stop()`)
|
|
216
|
+
this.watchdog.unref()
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
private stopWatchdog(): void {
|
|
220
|
+
clearInterval(this.watchdog)
|
|
221
|
+
this.watchdog = undefined
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* A tick that fires late means the loop could not turn in between: a long synchronous callback,
|
|
226
|
+
* a long GC pause, or the process not being scheduled at all. The CPU consumed across the gap
|
|
227
|
+
* tells these apart. Node reschedules a late interval from the actual fire time, so one stall
|
|
228
|
+
* produces exactly one late tick.
|
|
229
|
+
*/
|
|
230
|
+
private onWatchdogTick(): void {
|
|
231
|
+
const now = performance.now()
|
|
232
|
+
const cpuUs = getCpuUs()
|
|
233
|
+
const stallMs = now - this.watchdogAt - this.watchdogInterval
|
|
234
|
+
if (stallMs > this.maxStallMs) {
|
|
235
|
+
this.maxStallMs = stallMs
|
|
236
|
+
this.maxStallCpuMs = (cpuUs - this.watchdogCpuUs) / 1000
|
|
237
|
+
}
|
|
238
|
+
this.watchdogAt = now
|
|
239
|
+
this.watchdogCpuUs = cpuUs
|
|
156
240
|
}
|
|
157
241
|
|
|
158
242
|
// cfg: Required<EventLoopMonitorCfg>
|
|
159
243
|
}
|
|
160
244
|
|
|
245
|
+
function getGcKind(entry: PerformanceEntry): GcKind {
|
|
246
|
+
// `detail` is only declared on Node's PerformanceNodeEntry type, which getEntries() doesn't return
|
|
247
|
+
const kind = (entry as { detail?: { kind?: number } }).detail?.kind
|
|
248
|
+
if (kind === constants.NODE_PERFORMANCE_GC_MAJOR) return 'major'
|
|
249
|
+
if (kind === constants.NODE_PERFORMANCE_GC_MINOR) return 'minor'
|
|
250
|
+
if (kind === constants.NODE_PERFORMANCE_GC_INCREMENTAL) return 'incremental'
|
|
251
|
+
if (kind === constants.NODE_PERFORMANCE_GC_WEAKCB) return 'weakcb'
|
|
252
|
+
return 'unknown'
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* CPU time of the current (event loop) thread in microseconds, so that worker threads,
|
|
257
|
+
* the libuv threadpool and parallel GC helpers don't count.
|
|
258
|
+
*/
|
|
259
|
+
function getCpuUs(): number {
|
|
260
|
+
const { user, system } = process.threadCpuUsage()
|
|
261
|
+
return user + system
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
export type GcKind = 'major' | 'minor' | 'incremental' | 'weakcb' | 'unknown'
|
|
265
|
+
|
|
161
266
|
export interface EventLoopMonitorCfg {
|
|
162
267
|
/**
|
|
163
268
|
* Defaults to 60_000 ms
|
|
164
269
|
*/
|
|
165
270
|
measureInterval?: NumberOfMilliseconds
|
|
271
|
+
/**
|
|
272
|
+
* Period of the stall watchdog timer, see `EventLoopStats.stallMs`.
|
|
273
|
+
* Defaults to 1000 ms.
|
|
274
|
+
*/
|
|
275
|
+
watchdogInterval?: NumberOfMilliseconds
|
|
166
276
|
/**
|
|
167
277
|
* Callback to be invoked with EventLoopStats.
|
|
168
278
|
* Called every `measureInterval` milliseconds.
|
|
@@ -202,6 +312,31 @@ export interface EventLoopStats {
|
|
|
202
312
|
* too "lossy")
|
|
203
313
|
*/
|
|
204
314
|
gcCPU: Integer
|
|
315
|
+
/**
|
|
316
|
+
* Longest single GC pause in the last interval, among the pauses that ran while sampling was
|
|
317
|
+
* active (a pause overlapping a paused, CPU-throttled period is stretched by the throttling).
|
|
318
|
+
* `gcCount` and `gcTotalTime` are not gated.
|
|
319
|
+
*/
|
|
320
|
+
gcMaxTime: NumberOfMilliseconds
|
|
321
|
+
/**
|
|
322
|
+
* Kind of the longest GC pause. Undefined when no such GC pause ran in the interval.
|
|
323
|
+
*/
|
|
324
|
+
gcMaxKind: GcKind | undefined
|
|
325
|
+
/**
|
|
326
|
+
* Largest stall of the interval, as seen by the watchdog timer: how late its tick fired,
|
|
327
|
+
* 0 when every tick was on time. Covers anything that kept the loop from turning:
|
|
328
|
+
* a long synchronous callback, a long GC pause, or the process not being scheduled at all.
|
|
329
|
+
* Underestimates the stall by up to `watchdogInterval` (the tick may have been due anywhere
|
|
330
|
+
* within its period when the stall began). Not measured while paused.
|
|
331
|
+
*/
|
|
332
|
+
stallMs: NumberOfMilliseconds
|
|
333
|
+
/**
|
|
334
|
+
* CPU time of the event loop thread consumed between the watchdog tick before the largest
|
|
335
|
+
* stall and the late tick that measured it.
|
|
336
|
+
* Close to `stallMs` means the thread was running (own code or GC, see `gcMaxTime`),
|
|
337
|
+
* close to 0 means it was not scheduled (platform pause, CPU throttling).
|
|
338
|
+
*/
|
|
339
|
+
stallCpuMs: NumberOfMilliseconds
|
|
205
340
|
/**
|
|
206
341
|
* Actual wall-clock duration of the interval covered by these stats.
|
|
207
342
|
* Normally equals `measureInterval`, but is longer when the timer fired late,
|