@indigoai-us/hq-cloud 6.16.3 → 6.16.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-events.test.js +22 -12
- package/dist/bin/sync-runner-events.test.js.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.d.ts +238 -84
- package/dist/bin/sync-runner-heap-telemetry.d.ts.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.js +385 -131
- package/dist/bin/sync-runner-heap-telemetry.js.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.test.js +539 -204
- package/dist/bin/sync-runner-heap-telemetry.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +65 -15
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +47 -9
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +32 -14
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/package.json +1 -1
|
@@ -7,31 +7,45 @@
|
|
|
7
7
|
* runner shipped no memory telemetry at all, so nothing in the abort evidence
|
|
8
8
|
* could name which structure held the memory. This module closes that gap: it
|
|
9
9
|
* samples the process heap on a bounded cadence, attributes live entry counts
|
|
10
|
-
* to a fixed set of long-lived structures (the "census"), and —
|
|
11
|
-
*
|
|
12
|
-
*
|
|
10
|
+
* to a fixed set of long-lived structures (the "census"), and — before V8
|
|
11
|
+
* aborts — recycles the process so the desktop supervisor respawns a fresh one.
|
|
12
|
+
*
|
|
13
|
+
* The governor keys on the POST-GC LIVE HEAP, not the instantaneous
|
|
14
|
+
* `used_heap_size`. Between major collections `used_heap_size` is live-set plus
|
|
15
|
+
* not-yet-collected garbage, so it sawtooths by a large fraction of the ceiling
|
|
16
|
+
* from minute to minute; a rule that watches it can sit at a sub-threshold
|
|
17
|
+
* sample forever while the true live set climbs to the abort point (the
|
|
18
|
+
* 2026-09-01 regression: a full production day produced ZERO recycle verdicts
|
|
19
|
+
* against 1462 samples while the runner kept aborting). The monotonic quantity
|
|
20
|
+
* is the live set that survives a major GC — exactly what V8 uses for its own
|
|
21
|
+
* OOM verdict — so the primary signal is read on each observed MAJOR GC. A
|
|
22
|
+
* single instantaneous backstop remains for the case where the live signal is
|
|
23
|
+
* unavailable and the heap is already minutes from aborting.
|
|
13
24
|
*
|
|
14
25
|
* The whole module is numeric-only and allocation-light by construction:
|
|
15
26
|
*
|
|
16
27
|
* - Every emitted value is a number, and every census site is a FIXED
|
|
17
|
-
* identifier token, never a path/filename/company/message.
|
|
18
|
-
*
|
|
28
|
+
* identifier token, never a path/filename/company/message. Property keys are
|
|
29
|
+
* additionally shaped to survive the vault's ingest contract: they contain
|
|
30
|
+
* none of its denied key substrings and end in `_count`/`_bytes`/`_ms`, so
|
|
31
|
+
* the census attribution is actually stored rather than silently dropped.
|
|
19
32
|
* - The controller retains only the FIRST sample (the growth baseline) and
|
|
20
33
|
* the LATEST sample. It never accumulates history — a memory diagnostic
|
|
21
34
|
* that leaks would regress the very defect it measures.
|
|
22
|
-
* - All readers, timers, and emit callbacks are injected, so
|
|
23
|
-
* driven deterministically by tests with no real allocation,
|
|
24
|
-
* growth, no wall-clock timing, and no network.
|
|
35
|
+
* - All readers, timers, the GC observer, and emit callbacks are injected, so
|
|
36
|
+
* the module is driven deterministically by tests with no real allocation,
|
|
37
|
+
* no real heap growth, no wall-clock timing, and no network.
|
|
25
38
|
*
|
|
26
39
|
* Diagnostics must never affect sync: a throwing reader, a faulty census
|
|
27
|
-
* entry, or a rejecting emit callback degrades to
|
|
28
|
-
* watch cycle.
|
|
40
|
+
* entry, an unavailable GC entry API, or a rejecting emit callback degrades to
|
|
41
|
+
* silence and never disturbs a watch cycle.
|
|
29
42
|
*/
|
|
43
|
+
import * as perfHooks from "node:perf_hooks";
|
|
30
44
|
/**
|
|
31
45
|
* Census tokens are fixed identifiers, never user-derived strings. The pattern
|
|
32
46
|
* is deliberately stricter than the telemetry sanitizer's SAFE_KEY so a token
|
|
33
|
-
* can always be prefixed (`census_
|
|
34
|
-
* the 64-character key budget.
|
|
47
|
+
* can always be prefixed (`census_`) and suffixed (`_count`, `_growth_count`)
|
|
48
|
+
* and still fit inside the 64-character key budget.
|
|
35
49
|
*/
|
|
36
50
|
const CENSUS_TOKEN = /^[A-Za-z][A-Za-z0-9_]{0,48}$/;
|
|
37
51
|
/**
|
|
@@ -93,11 +107,18 @@ export function sampleRunnerHeap(deps) {
|
|
|
93
107
|
function finiteNumber(value) {
|
|
94
108
|
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
95
109
|
}
|
|
110
|
+
/** Parts-per-thousand, the integer spelling every fraction is emitted as. */
|
|
111
|
+
function toPermille(fraction) {
|
|
112
|
+
return Math.round(fraction * 1000);
|
|
113
|
+
}
|
|
96
114
|
/**
|
|
97
115
|
* Flatten a sample into numeric telemetry properties. Raw heap figures carry a
|
|
98
|
-
* `heap_`/`rss_`/`external_` prefix; each census site becomes
|
|
99
|
-
* Every value is a number
|
|
100
|
-
*
|
|
116
|
+
* `heap_`/`rss_`/`external_` prefix; each census site becomes
|
|
117
|
+
* `census_<token>_count`. Every value is a number AND every key ends in
|
|
118
|
+
* `_count`/`_bytes`/`_ms` and contains none of the vault ingest contract's
|
|
119
|
+
* denied substrings, so the record is both accepted by the local sanitizer and
|
|
120
|
+
* actually STORED by ingest (unlike the pre-fix `census_<token>` spelling,
|
|
121
|
+
* which ingest silently dropped, so the attribution never reached the store).
|
|
101
122
|
*/
|
|
102
123
|
export function heapSampleCloudProperties(sample) {
|
|
103
124
|
const props = {
|
|
@@ -109,18 +130,18 @@ export function heapSampleCloudProperties(sample) {
|
|
|
109
130
|
external_bytes: sample.externalBytes,
|
|
110
131
|
array_buffers_bytes: sample.arrayBuffersBytes,
|
|
111
132
|
uptime_ms: sample.uptimeMs,
|
|
112
|
-
|
|
133
|
+
census_total_count: sample.censusTotal,
|
|
113
134
|
};
|
|
114
135
|
for (const { site, count } of sample.census) {
|
|
115
|
-
props[`census_${site}`] = count;
|
|
136
|
+
props[`census_${site}_count`] = count;
|
|
116
137
|
}
|
|
117
138
|
return props;
|
|
118
139
|
}
|
|
119
140
|
/** Flatten a pressure record into numeric-only telemetry properties. */
|
|
120
141
|
export function heapPressureCloudProperties(record) {
|
|
121
142
|
const props = {
|
|
122
|
-
|
|
123
|
-
|
|
143
|
+
pressure_used_permille_count: toPermille(record.usedFraction),
|
|
144
|
+
pressure_threshold_permille_count: toPermille(record.thresholdFraction),
|
|
124
145
|
heap_used_bytes: record.usedHeapBytes,
|
|
125
146
|
heap_limit_bytes: record.heapLimitBytes,
|
|
126
147
|
rss_bytes: record.rssBytes,
|
|
@@ -128,8 +149,8 @@ export function heapPressureCloudProperties(record) {
|
|
|
128
149
|
uptime_ms: record.uptimeMs,
|
|
129
150
|
};
|
|
130
151
|
for (const { site, count, growth } of record.topSites) {
|
|
131
|
-
props[
|
|
132
|
-
props[
|
|
152
|
+
props[`${site}_count`] = count;
|
|
153
|
+
props[`${site}_growth_count`] = growth;
|
|
133
154
|
}
|
|
134
155
|
return props;
|
|
135
156
|
}
|
|
@@ -140,26 +161,35 @@ export function heapPressureStderrLine(record) {
|
|
|
140
161
|
.join(",");
|
|
141
162
|
return ("hq-sync-runner: heap-pressure " +
|
|
142
163
|
`used=${record.usedHeapBytes} limit=${record.heapLimitBytes} ` +
|
|
143
|
-
`
|
|
164
|
+
`permille=${toPermille(record.usedFraction)} rss=${record.rssBytes} ` +
|
|
144
165
|
`uptime_ms=${record.uptimeMs} top=${top}`);
|
|
145
166
|
}
|
|
146
167
|
/** Flatten a recycle record into numeric-only telemetry properties. */
|
|
147
168
|
export function heapRecycleCloudProperties(record) {
|
|
148
169
|
const props = {
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
170
|
+
recycle_used_permille_count: toPermille(record.usedFraction),
|
|
171
|
+
recycle_live_permille_count: toPermille(record.liveFraction),
|
|
172
|
+
recycle_live_floor_permille_count: toPermille(record.recycleLiveFraction),
|
|
173
|
+
recycle_emergency_floor_permille_count: toPermille(record.emergencyFraction),
|
|
174
|
+
recycle_emergency_trigger_count: record.trigger === "emergency" ? 1 : 0,
|
|
175
|
+
recycle_sustain_window_ms: record.sustainWindowMs,
|
|
176
|
+
recycle_observed_sustain_ms: record.observedSustainMs,
|
|
153
177
|
recycle_min_uptime_ms: record.minUptimeMs,
|
|
154
178
|
heap_used_bytes: record.usedHeapBytes,
|
|
179
|
+
heap_live_bytes: record.liveHeapBytes,
|
|
155
180
|
heap_limit_bytes: record.heapLimitBytes,
|
|
156
181
|
rss_bytes: record.rssBytes,
|
|
157
182
|
external_bytes: record.externalBytes,
|
|
183
|
+
array_buffers_bytes: record.arrayBuffersBytes,
|
|
184
|
+
rss_growth_bytes: record.rssGrowthBytes,
|
|
185
|
+
external_growth_bytes: record.externalGrowthBytes,
|
|
186
|
+
array_buffers_growth_bytes: record.arrayBuffersGrowthBytes,
|
|
158
187
|
uptime_ms: record.uptimeMs,
|
|
188
|
+
census_total_count: record.censusTotal,
|
|
159
189
|
};
|
|
160
190
|
for (const { site, count, growth } of record.topSites) {
|
|
161
|
-
props[
|
|
162
|
-
props[
|
|
191
|
+
props[`${site}_count`] = count;
|
|
192
|
+
props[`${site}_growth_count`] = growth;
|
|
163
193
|
}
|
|
164
194
|
return props;
|
|
165
195
|
}
|
|
@@ -177,20 +207,28 @@ export function heapRecycleStderrLine(record) {
|
|
|
177
207
|
.map((entry) => `${entry.site}:${entry.growth}`)
|
|
178
208
|
.join(",");
|
|
179
209
|
return ("hq-sync-runner: heap-recycle " +
|
|
180
|
-
`
|
|
181
|
-
`
|
|
182
|
-
`
|
|
210
|
+
`trigger=${record.trigger} ` +
|
|
211
|
+
`used=${record.usedHeapBytes} live=${record.liveHeapBytes} limit=${record.heapLimitBytes} ` +
|
|
212
|
+
`live_permille=${toPermille(record.liveFraction)} rss=${record.rssBytes} ` +
|
|
213
|
+
`uptime_ms=${record.uptimeMs} sustain_ms=${record.observedSustainMs} top=${top}`);
|
|
183
214
|
}
|
|
184
215
|
const DEFAULT_PRESSURE_THRESHOLD = 0.75;
|
|
185
216
|
const DEFAULT_TOP_SITE_COUNT = 8;
|
|
186
|
-
const
|
|
187
|
-
const
|
|
188
|
-
const
|
|
217
|
+
const DEFAULT_RECYCLE_LIVE_FRACTION = 0.6;
|
|
218
|
+
const DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS = 120_000;
|
|
219
|
+
const DEFAULT_RECYCLE_EMERGENCY_FRACTION = 0.9;
|
|
220
|
+
const DEFAULT_RECYCLE_MIN_UPTIME_MS = 35 * 60_000;
|
|
189
221
|
/**
|
|
190
|
-
* Owns the sampling state machine
|
|
191
|
-
*
|
|
192
|
-
*
|
|
193
|
-
*
|
|
222
|
+
* Owns the sampling state machine. It keeps the baseline and latest sample, the
|
|
223
|
+
* pressure escalation, and the two recycle paths:
|
|
224
|
+
* - {@link observeMajorGc}, the PRIMARY path, driven by the GC observer: each
|
|
225
|
+
* (throttled) call reads the post-GC live heap and tracks the sustain
|
|
226
|
+
* window that a live recycle requires.
|
|
227
|
+
* - {@link evaluate} and the emergency arm of {@link sample}, the instantaneous
|
|
228
|
+
* BACKSTOP: they recycle on a single sample at/above the emergency fraction
|
|
229
|
+
* even if the GC signal never arrives or the sampler timer is starved.
|
|
230
|
+
* Every fraction is derived from the injected `heap_size_limit`, so the governor
|
|
231
|
+
* self-calibrates to whatever `--max-old-space-size` the desktop supplied.
|
|
194
232
|
*/
|
|
195
233
|
export class RunnerHeapTelemetry {
|
|
196
234
|
readers;
|
|
@@ -199,14 +237,17 @@ export class RunnerHeapTelemetry {
|
|
|
199
237
|
threshold;
|
|
200
238
|
topSiteCount;
|
|
201
239
|
recycleEnabled;
|
|
202
|
-
|
|
203
|
-
|
|
240
|
+
recycleLiveFraction;
|
|
241
|
+
recycleSustainWindowMs;
|
|
242
|
+
recycleEmergencyFraction;
|
|
204
243
|
recycleMinUptimeMs;
|
|
205
244
|
baseline = null;
|
|
206
245
|
latest = null;
|
|
207
246
|
pressureSignalled = false;
|
|
208
|
-
recycleSustainedCount = 0;
|
|
209
247
|
recycleSignalled = false;
|
|
248
|
+
/** Uptime (ms) of the first live observation at/above the floor in the
|
|
249
|
+
* current consecutive run; null when the last live reading dipped below it. */
|
|
250
|
+
liveAnchorUptimeMs = null;
|
|
210
251
|
constructor(options) {
|
|
211
252
|
this.readers = options.readers;
|
|
212
253
|
this.census = options.census;
|
|
@@ -219,16 +260,11 @@ export class RunnerHeapTelemetry {
|
|
|
219
260
|
const count = options.topSiteCount ?? DEFAULT_TOP_SITE_COUNT;
|
|
220
261
|
this.topSiteCount = Number.isFinite(count) && count > 0 ? Math.trunc(count) : DEFAULT_TOP_SITE_COUNT;
|
|
221
262
|
this.recycleEnabled = options.recycleEnabled ?? true;
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
const sustained = options.recycleSustainedSamples ?? DEFAULT_RECYCLE_SUSTAINED_SAMPLES;
|
|
228
|
-
this.recycleSustainedSamples =
|
|
229
|
-
Number.isFinite(sustained) && sustained >= 1
|
|
230
|
-
? Math.trunc(sustained)
|
|
231
|
-
: DEFAULT_RECYCLE_SUSTAINED_SAMPLES;
|
|
263
|
+
this.recycleLiveFraction = fractionOrDefault(options.recycleLiveFraction, DEFAULT_RECYCLE_LIVE_FRACTION);
|
|
264
|
+
const window = options.recycleSustainWindowMs ?? DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
|
|
265
|
+
this.recycleSustainWindowMs =
|
|
266
|
+
Number.isFinite(window) && window >= 0 ? Math.trunc(window) : DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
|
|
267
|
+
this.recycleEmergencyFraction = fractionOrDefault(options.recycleEmergencyFraction, DEFAULT_RECYCLE_EMERGENCY_FRACTION);
|
|
232
268
|
const minUptime = options.recycleMinUptimeMs ?? DEFAULT_RECYCLE_MIN_UPTIME_MS;
|
|
233
269
|
this.recycleMinUptimeMs =
|
|
234
270
|
Number.isFinite(minUptime) && minUptime >= 0
|
|
@@ -237,78 +273,151 @@ export class RunnerHeapTelemetry {
|
|
|
237
273
|
}
|
|
238
274
|
/**
|
|
239
275
|
* Capture the growth baseline when sampling starts, before the first interval,
|
|
240
|
-
* WITHOUT emitting or arming
|
|
241
|
-
*
|
|
242
|
-
*
|
|
243
|
-
*
|
|
276
|
+
* WITHOUT emitting or arming any signal. Without this, a runner that is already
|
|
277
|
+
* above the threshold at its first scheduled sample would make that same high
|
|
278
|
+
* sample its own baseline, so a crossing record would report zero growth for
|
|
279
|
+
* every site and lose the attribution.
|
|
244
280
|
*/
|
|
245
281
|
primeBaseline() {
|
|
246
|
-
|
|
247
|
-
readers: this.readers,
|
|
248
|
-
census: this.census,
|
|
249
|
-
sessionStartedAtMs: this.sessionStartedAtMs,
|
|
250
|
-
});
|
|
251
|
-
this.baseline ??= sample;
|
|
252
|
-
this.latest = sample;
|
|
253
|
-
return sample;
|
|
282
|
+
return this.buildSample();
|
|
254
283
|
}
|
|
284
|
+
/**
|
|
285
|
+
* The interval sampler tick. Emits a sample and (at most once per rising
|
|
286
|
+
* crossing) a pressure record. Its recycle output is the EMERGENCY backstop
|
|
287
|
+
* only — the instantaneous sawtooth is deliberately NOT the primary verdict.
|
|
288
|
+
*/
|
|
255
289
|
sample() {
|
|
256
|
-
const sample =
|
|
257
|
-
readers: this.readers,
|
|
258
|
-
census: this.census,
|
|
259
|
-
sessionStartedAtMs: this.sessionStartedAtMs,
|
|
260
|
-
});
|
|
261
|
-
// Retain ONLY the baseline (first sample) and the latest. Growth is the
|
|
262
|
-
// difference between the two; no per-sample history is ever kept.
|
|
263
|
-
this.baseline ??= sample;
|
|
264
|
-
this.latest = sample;
|
|
290
|
+
const sample = this.buildSample();
|
|
265
291
|
return {
|
|
266
292
|
sample,
|
|
267
293
|
pressure: this.detectPressure(sample),
|
|
268
|
-
recycle: this.
|
|
294
|
+
recycle: this.detectEmergency(sample),
|
|
269
295
|
};
|
|
270
296
|
}
|
|
271
297
|
/**
|
|
272
|
-
* The governor
|
|
273
|
-
*
|
|
274
|
-
*
|
|
275
|
-
*
|
|
276
|
-
*
|
|
277
|
-
*
|
|
298
|
+
* The PRIMARY governor path, driven by an observed MAJOR GC. The caller
|
|
299
|
+
* ({@link nodeMajorGcObserver}) only invokes this for a GC whose reading it
|
|
300
|
+
* could capture PROMPTLY — a callback delayed by a blocked event loop is
|
|
301
|
+
* skipped upstream, because `getHeapStatistics()` read late would be an
|
|
302
|
+
* instantaneous sawtooth value (live set plus everything allocated since the
|
|
303
|
+
* GC), not the post-GC live floor.
|
|
304
|
+
*
|
|
305
|
+
* The anchor is maintained on EVERY observed major GC via a cheap
|
|
306
|
+
* `getHeapStatistics()` read (no throttle), so a post-GC reading that dips
|
|
307
|
+
* below the floor ALWAYS invalidates the sustain run — a throttle that skipped
|
|
308
|
+
* such a reading would let a stale anchor fake a continuous run. The fuller
|
|
309
|
+
* sample (census snapshot + memory deltas) is taken only at the verdict, so a
|
|
310
|
+
* GC storm never costs more than the O(1) statistics read here.
|
|
278
311
|
*/
|
|
279
|
-
|
|
280
|
-
if (!this.recycleEnabled)
|
|
312
|
+
observeMajorGc() {
|
|
313
|
+
if (!this.recycleEnabled || this.recycleSignalled)
|
|
281
314
|
return null;
|
|
282
|
-
|
|
315
|
+
const stats = this.readers.getHeapStatistics();
|
|
316
|
+
const heapLimitBytes = finiteNumber(stats.heap_size_limit);
|
|
317
|
+
if (heapLimitBytes <= 0)
|
|
318
|
+
return null;
|
|
319
|
+
const liveFraction = finiteNumber(stats.used_heap_size) / heapLimitBytes;
|
|
320
|
+
const uptimeMs = Math.max(0, Math.trunc(this.readers.monotonicNowMs() - this.sessionStartedAtMs));
|
|
321
|
+
// A genuine post-GC reading already abort-adjacent: recycle now, bypassing
|
|
322
|
+
// both the sustain window and the uptime floor.
|
|
323
|
+
if (liveFraction >= this.recycleEmergencyFraction) {
|
|
324
|
+
return this.produceRecycleFromCurrent("emergency", 0);
|
|
325
|
+
}
|
|
326
|
+
if (liveFraction < this.recycleLiveFraction) {
|
|
327
|
+
// The live floor dipped: reset the anchor so the window only ever measures
|
|
328
|
+
// a genuinely continuous run at/above the floor.
|
|
329
|
+
this.liveAnchorUptimeMs = null;
|
|
283
330
|
return null;
|
|
284
|
-
const usedFraction = sample.usedHeapBytes / sample.heapLimitBytes;
|
|
285
|
-
if (usedFraction >= this.recycleFraction) {
|
|
286
|
-
this.recycleSustainedCount += 1;
|
|
287
331
|
}
|
|
288
|
-
|
|
289
|
-
//
|
|
290
|
-
|
|
291
|
-
this.recycleSustainedCount = 0;
|
|
332
|
+
if (this.liveAnchorUptimeMs === null) {
|
|
333
|
+
// First qualifying observation. A second, at least a window later, confirms.
|
|
334
|
+
this.liveAnchorUptimeMs = uptimeMs;
|
|
292
335
|
return null;
|
|
293
336
|
}
|
|
294
|
-
|
|
337
|
+
const observedSustainMs = Math.max(0, uptimeMs - this.liveAnchorUptimeMs);
|
|
338
|
+
if (observedSustainMs < this.recycleSustainWindowMs)
|
|
295
339
|
return null;
|
|
296
|
-
if (
|
|
340
|
+
if (uptimeMs < this.recycleMinUptimeMs)
|
|
297
341
|
return null;
|
|
298
|
-
|
|
342
|
+
return this.produceRecycleFromCurrent("live", observedSustainMs);
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* The loop-top BACKSTOP against timer starvation (the observed 11-minute
|
|
346
|
+
* sampler hole). Reads the instantaneous heap and runs the emergency check
|
|
347
|
+
* only — an instantaneous reading can never confirm the post-GC live floor,
|
|
348
|
+
* so it must not feed the primary path's sustain window. Cheap: an O(1)
|
|
349
|
+
* statistics read, sampling the census only if it actually fires.
|
|
350
|
+
*/
|
|
351
|
+
evaluate() {
|
|
352
|
+
if (!this.recycleEnabled || this.recycleSignalled)
|
|
299
353
|
return null;
|
|
354
|
+
const stats = this.readers.getHeapStatistics();
|
|
355
|
+
const heapLimitBytes = finiteNumber(stats.heap_size_limit);
|
|
356
|
+
if (heapLimitBytes <= 0)
|
|
357
|
+
return null;
|
|
358
|
+
const usedFraction = finiteNumber(stats.used_heap_size) / heapLimitBytes;
|
|
359
|
+
if (usedFraction < this.recycleEmergencyFraction)
|
|
360
|
+
return null;
|
|
361
|
+
return this.produceRecycleFromCurrent("emergency", 0);
|
|
362
|
+
}
|
|
363
|
+
buildSample() {
|
|
364
|
+
const sample = sampleRunnerHeap({
|
|
365
|
+
readers: this.readers,
|
|
366
|
+
census: this.census,
|
|
367
|
+
sessionStartedAtMs: this.sessionStartedAtMs,
|
|
368
|
+
});
|
|
369
|
+
// Retain ONLY the baseline (first sample) and the latest. Growth is the
|
|
370
|
+
// difference between the two; no per-sample history is ever kept.
|
|
371
|
+
this.baseline ??= sample;
|
|
372
|
+
this.latest = sample;
|
|
373
|
+
return sample;
|
|
374
|
+
}
|
|
375
|
+
/** The instantaneous emergency backstop: a single sample at/above the
|
|
376
|
+
* emergency fraction, bypassing the sustain window and the uptime floor. */
|
|
377
|
+
detectEmergency(sample) {
|
|
378
|
+
if (!this.recycleEnabled || this.recycleSignalled)
|
|
379
|
+
return null;
|
|
380
|
+
if (sample.heapLimitBytes <= 0)
|
|
381
|
+
return null;
|
|
382
|
+
const usedFraction = sample.usedHeapBytes / sample.heapLimitBytes;
|
|
383
|
+
if (usedFraction < this.recycleEmergencyFraction)
|
|
384
|
+
return null;
|
|
385
|
+
return this.produceRecycle(sample, "emergency", usedFraction, 0);
|
|
386
|
+
}
|
|
387
|
+
/**
|
|
388
|
+
* Produce a verdict from a freshly-taken sample. Called only once the cheap
|
|
389
|
+
* per-GC (or loop-top) path has decided a recycle is due, so the census
|
|
390
|
+
* snapshot and memory deltas are computed exactly once per session.
|
|
391
|
+
*/
|
|
392
|
+
produceRecycleFromCurrent(trigger, observedSustainMs) {
|
|
393
|
+
const sample = this.buildSample();
|
|
394
|
+
const liveFraction = sample.heapLimitBytes > 0 ? sample.usedHeapBytes / sample.heapLimitBytes : 0;
|
|
395
|
+
return this.produceRecycle(sample, trigger, liveFraction, observedSustainMs);
|
|
396
|
+
}
|
|
397
|
+
produceRecycle(sample, trigger, liveFraction, observedSustainMs) {
|
|
300
398
|
this.recycleSignalled = true;
|
|
399
|
+
const usedFraction = sample.heapLimitBytes > 0 ? sample.usedHeapBytes / sample.heapLimitBytes : 0;
|
|
400
|
+
const baseline = this.baseline;
|
|
301
401
|
return {
|
|
402
|
+
trigger,
|
|
302
403
|
usedHeapBytes: sample.usedHeapBytes,
|
|
404
|
+
liveHeapBytes: sample.usedHeapBytes,
|
|
303
405
|
heapLimitBytes: sample.heapLimitBytes,
|
|
304
406
|
usedFraction,
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
407
|
+
liveFraction,
|
|
408
|
+
recycleLiveFraction: this.recycleLiveFraction,
|
|
409
|
+
emergencyFraction: this.recycleEmergencyFraction,
|
|
410
|
+
sustainWindowMs: this.recycleSustainWindowMs,
|
|
411
|
+
observedSustainMs,
|
|
308
412
|
minUptimeMs: this.recycleMinUptimeMs,
|
|
413
|
+
uptimeMs: sample.uptimeMs,
|
|
309
414
|
rssBytes: sample.rssBytes,
|
|
310
415
|
externalBytes: sample.externalBytes,
|
|
311
|
-
|
|
416
|
+
arrayBuffersBytes: sample.arrayBuffersBytes,
|
|
417
|
+
rssGrowthBytes: sample.rssBytes - (baseline?.rssBytes ?? sample.rssBytes),
|
|
418
|
+
externalGrowthBytes: sample.externalBytes - (baseline?.externalBytes ?? sample.externalBytes),
|
|
419
|
+
arrayBuffersGrowthBytes: sample.arrayBuffersBytes - (baseline?.arrayBuffersBytes ?? sample.arrayBuffersBytes),
|
|
420
|
+
censusTotal: sample.censusTotal,
|
|
312
421
|
topSites: this.topSitesByGrowth(sample),
|
|
313
422
|
};
|
|
314
423
|
}
|
|
@@ -355,18 +464,104 @@ export class RunnerHeapTelemetry {
|
|
|
355
464
|
return this.latest;
|
|
356
465
|
}
|
|
357
466
|
}
|
|
467
|
+
function fractionOrDefault(value, fallback) {
|
|
468
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 && value <= 1
|
|
469
|
+
? value
|
|
470
|
+
: fallback;
|
|
471
|
+
}
|
|
472
|
+
/**
|
|
473
|
+
* The maximum age (ms) of a major-GC PerformanceEntry, at the moment its
|
|
474
|
+
* observer callback runs, for the post-GC heap read to be trustworthy. GC
|
|
475
|
+
* entries are delivered ASYNCHRONOUSLY, so when the event loop was blocked the
|
|
476
|
+
* callback fires long after the collection and a `getHeapStatistics()` read then
|
|
477
|
+
* reflects the live set PLUS everything allocated since — an instantaneous
|
|
478
|
+
* sawtooth value, not the post-GC live floor. A reading older than this is
|
|
479
|
+
* skipped rather than fed to the sustain window, so only promptly-captured
|
|
480
|
+
* (genuinely post-GC) readings can arm a live recycle.
|
|
481
|
+
*/
|
|
482
|
+
export const MAX_FRESH_GC_READING_MS = 250;
|
|
483
|
+
/**
|
|
484
|
+
* Whether a major-GC reading taken at `nowMs` is fresh enough to treat as a
|
|
485
|
+
* post-GC live-heap snapshot, given the entry's end time (`entry.startTime +
|
|
486
|
+
* entry.duration`). Exported for deterministic testing of the staleness gate.
|
|
487
|
+
*/
|
|
488
|
+
export function isFreshGcReading(gcEndMs, nowMs, maxAgeMs = MAX_FRESH_GC_READING_MS) {
|
|
489
|
+
if (!Number.isFinite(gcEndMs) || !Number.isFinite(nowMs))
|
|
490
|
+
return false;
|
|
491
|
+
return nowMs - gcEndMs <= maxAgeMs;
|
|
492
|
+
}
|
|
493
|
+
/**
|
|
494
|
+
* Production major-GC observer over `perf_hooks`. Everything is wrapped so a
|
|
495
|
+
* runtime without the GC PerformanceEntry API, or one whose entries omit
|
|
496
|
+
* `detail.kind`, degrades to a silent no-op rather than throwing into the
|
|
497
|
+
* runner — in that degraded case the instantaneous emergency backstop and the
|
|
498
|
+
* pressure diagnostics remain active, so the governor is never worse than base.
|
|
499
|
+
*
|
|
500
|
+
* It only invokes the callback for a major GC whose reading it can capture
|
|
501
|
+
* PROMPTLY ({@link isFreshGcReading}): a callback delayed by a blocked event
|
|
502
|
+
* loop would read an inflated instantaneous heap rather than the post-GC live
|
|
503
|
+
* floor, so it is skipped.
|
|
504
|
+
*/
|
|
505
|
+
export function nodeMajorGcObserver() {
|
|
506
|
+
let observer = null;
|
|
507
|
+
return {
|
|
508
|
+
start(onMajorGc) {
|
|
509
|
+
try {
|
|
510
|
+
const PerformanceObserverCtor = perfHooks.PerformanceObserver;
|
|
511
|
+
const majorKind = perfHooks.constants?.NODE_PERFORMANCE_GC_MAJOR;
|
|
512
|
+
if (typeof PerformanceObserverCtor !== "function" || typeof majorKind !== "number") {
|
|
513
|
+
// No usable major-GC signal on this runtime: stay silent.
|
|
514
|
+
observer = null;
|
|
515
|
+
return;
|
|
516
|
+
}
|
|
517
|
+
observer = new PerformanceObserverCtor((list) => {
|
|
518
|
+
try {
|
|
519
|
+
const nowMs = perfHooks.performance.now();
|
|
520
|
+
for (const entry of list.getEntries()) {
|
|
521
|
+
const kind = entry.detail?.kind;
|
|
522
|
+
if (typeof kind !== "number" || kind !== majorKind)
|
|
523
|
+
continue;
|
|
524
|
+
const gcEndMs = entry.startTime + (entry.duration ?? 0);
|
|
525
|
+
if (!isFreshGcReading(gcEndMs, nowMs))
|
|
526
|
+
continue;
|
|
527
|
+
onMajorGc();
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
catch {
|
|
531
|
+
/* a mis-shaped entry or a throwing callback must never surface */
|
|
532
|
+
}
|
|
533
|
+
});
|
|
534
|
+
observer.observe({ entryTypes: ["gc"] });
|
|
535
|
+
}
|
|
536
|
+
catch {
|
|
537
|
+
observer = null;
|
|
538
|
+
}
|
|
539
|
+
},
|
|
540
|
+
stop() {
|
|
541
|
+
try {
|
|
542
|
+
observer?.disconnect();
|
|
543
|
+
}
|
|
544
|
+
catch {
|
|
545
|
+
/* disconnecting is best-effort during shutdown */
|
|
546
|
+
}
|
|
547
|
+
observer = null;
|
|
548
|
+
},
|
|
549
|
+
};
|
|
550
|
+
}
|
|
358
551
|
const DEFAULT_SAMPLE_INTERVAL_MS = 60_000;
|
|
359
552
|
/**
|
|
360
|
-
* Start periodic sampling
|
|
361
|
-
*
|
|
362
|
-
*
|
|
363
|
-
*
|
|
553
|
+
* Start periodic sampling and, when a GC observer is supplied, the primary
|
|
554
|
+
* live-heap path. The interval timer is `unref`'d so it can never hold the
|
|
555
|
+
* process open, and `stop()` clears it and disconnects the GC observer on every
|
|
556
|
+
* shutdown path. Each tick, GC callback, and emit — and the loop-top
|
|
557
|
+
* `evaluate()` — is wrapped so a fault degrades to silence and never affects the
|
|
558
|
+
* watch cycle.
|
|
364
559
|
*/
|
|
365
560
|
export function startRunnerHeapSampling(options) {
|
|
366
561
|
const telemetry = new RunnerHeapTelemetry(options);
|
|
367
562
|
// Capture the baseline immediately, before the first interval, so the first
|
|
368
|
-
// pressure crossing
|
|
369
|
-
// against a first sample that may already sit at the ceiling.
|
|
563
|
+
// pressure crossing (and every growth delta) attributes against session start
|
|
564
|
+
// rather than against a first sample that may already sit at the ceiling.
|
|
370
565
|
try {
|
|
371
566
|
telemetry.primeBaseline();
|
|
372
567
|
}
|
|
@@ -378,6 +573,16 @@ export function startRunnerHeapSampling(options) {
|
|
|
378
573
|
? Math.trunc(options.intervalMs)
|
|
379
574
|
: DEFAULT_SAMPLE_INTERVAL_MS;
|
|
380
575
|
let stopped = false;
|
|
576
|
+
// One recycle verdict per session, from whichever path reaches it first. The
|
|
577
|
+
// telemetry latches internally, so this callback fires at most once.
|
|
578
|
+
const fireRecycle = (record) => {
|
|
579
|
+
try {
|
|
580
|
+
options.onRecycle?.(record);
|
|
581
|
+
}
|
|
582
|
+
catch (err) {
|
|
583
|
+
warn(options.log, err);
|
|
584
|
+
}
|
|
585
|
+
};
|
|
381
586
|
const tick = () => {
|
|
382
587
|
if (stopped)
|
|
383
588
|
return;
|
|
@@ -397,23 +602,50 @@ export function startRunnerHeapSampling(options) {
|
|
|
397
602
|
warn(options.log, err);
|
|
398
603
|
}
|
|
399
604
|
}
|
|
400
|
-
if (recycle)
|
|
605
|
+
if (recycle)
|
|
606
|
+
fireRecycle(recycle);
|
|
607
|
+
}
|
|
608
|
+
catch (err) {
|
|
609
|
+
warn(options.log, err);
|
|
610
|
+
}
|
|
611
|
+
};
|
|
612
|
+
const handle = scheduler.setInterval(tick, intervalMs);
|
|
613
|
+
handle.unref?.();
|
|
614
|
+
// The primary path: a major GC drives one (throttled) post-GC live evaluation.
|
|
615
|
+
const gcObserver = options.gcObserver;
|
|
616
|
+
if (gcObserver) {
|
|
617
|
+
try {
|
|
618
|
+
gcObserver.start(() => {
|
|
619
|
+
if (stopped)
|
|
620
|
+
return;
|
|
401
621
|
try {
|
|
402
|
-
|
|
622
|
+
const recycle = telemetry.observeMajorGc();
|
|
623
|
+
if (recycle)
|
|
624
|
+
fireRecycle(recycle);
|
|
403
625
|
}
|
|
404
626
|
catch (err) {
|
|
405
627
|
warn(options.log, err);
|
|
406
628
|
}
|
|
407
|
-
}
|
|
629
|
+
});
|
|
408
630
|
}
|
|
409
631
|
catch (err) {
|
|
410
632
|
warn(options.log, err);
|
|
411
633
|
}
|
|
412
|
-
}
|
|
413
|
-
const handle = scheduler.setInterval(tick, intervalMs);
|
|
414
|
-
handle.unref?.();
|
|
634
|
+
}
|
|
415
635
|
return {
|
|
416
636
|
telemetry,
|
|
637
|
+
evaluate: () => {
|
|
638
|
+
if (stopped)
|
|
639
|
+
return;
|
|
640
|
+
try {
|
|
641
|
+
const recycle = telemetry.evaluate();
|
|
642
|
+
if (recycle)
|
|
643
|
+
fireRecycle(recycle);
|
|
644
|
+
}
|
|
645
|
+
catch (err) {
|
|
646
|
+
warn(options.log, err);
|
|
647
|
+
}
|
|
648
|
+
},
|
|
417
649
|
stop: () => {
|
|
418
650
|
if (stopped)
|
|
419
651
|
return;
|
|
@@ -424,6 +656,12 @@ export function startRunnerHeapSampling(options) {
|
|
|
424
656
|
catch {
|
|
425
657
|
/* clearing is best-effort during shutdown */
|
|
426
658
|
}
|
|
659
|
+
try {
|
|
660
|
+
gcObserver?.stop();
|
|
661
|
+
}
|
|
662
|
+
catch {
|
|
663
|
+
/* disconnecting is best-effort during shutdown */
|
|
664
|
+
}
|
|
427
665
|
},
|
|
428
666
|
};
|
|
429
667
|
}
|
|
@@ -443,13 +681,15 @@ function warn(log, err) {
|
|
|
443
681
|
export const HEAP_SAMPLE_INTERVAL_ENV = "HQ_SYNC_RUNNER_HEAP_SAMPLE_MS";
|
|
444
682
|
export const HEAP_PRESSURE_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_PRESSURE_FRACTION";
|
|
445
683
|
export const HEAP_RECYCLE_ENABLED_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_ENABLED";
|
|
446
|
-
export const
|
|
447
|
-
export const
|
|
684
|
+
export const HEAP_RECYCLE_LIVE_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_LIVE_FRACTION";
|
|
685
|
+
export const HEAP_RECYCLE_SUSTAIN_WINDOW_MS_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_SUSTAIN_WINDOW_MS";
|
|
686
|
+
export const HEAP_RECYCLE_EMERGENCY_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_EMERGENCY_FRACTION";
|
|
448
687
|
export const HEAP_RECYCLE_MIN_UPTIME_MS_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_MIN_UPTIME_MS";
|
|
449
688
|
export const DEFAULT_HEAP_SAMPLE_INTERVAL_MS = DEFAULT_SAMPLE_INTERVAL_MS;
|
|
450
689
|
export const DEFAULT_HEAP_PRESSURE_FRACTION = DEFAULT_PRESSURE_THRESHOLD;
|
|
451
|
-
export const
|
|
452
|
-
export const
|
|
690
|
+
export const DEFAULT_HEAP_RECYCLE_LIVE_FRACTION = DEFAULT_RECYCLE_LIVE_FRACTION;
|
|
691
|
+
export const DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS = DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
|
|
692
|
+
export const DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION = DEFAULT_RECYCLE_EMERGENCY_FRACTION;
|
|
453
693
|
export const DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS = DEFAULT_RECYCLE_MIN_UPTIME_MS;
|
|
454
694
|
/**
|
|
455
695
|
* Parse the sampling-interval env knob (whole milliseconds). Rejects the ENTIRE
|
|
@@ -493,39 +733,53 @@ export function resolveHeapRecycleEnabled(raw) {
|
|
|
493
733
|
return !(trimmed === "0" || trimmed === "false" || trimmed === "no" || trimmed === "off");
|
|
494
734
|
}
|
|
495
735
|
/**
|
|
496
|
-
* Parse the
|
|
497
|
-
* or out-of-range input. Rejects the entire string (e.g. "0.
|
|
498
|
-
* rather than accepting a numeric prefix.
|
|
736
|
+
* Parse the primary live-fraction knob; falls back to the default (0.60) on
|
|
737
|
+
* garbage or out-of-range input. Rejects the entire string (e.g. "0.6x" falls
|
|
738
|
+
* back) rather than accepting a numeric prefix.
|
|
739
|
+
*/
|
|
740
|
+
export function resolveHeapRecycleLiveFraction(raw) {
|
|
741
|
+
if (raw === undefined)
|
|
742
|
+
return DEFAULT_RECYCLE_LIVE_FRACTION;
|
|
743
|
+
const trimmed = raw.trim();
|
|
744
|
+
if (!/^\d*\.?\d+$/.test(trimmed))
|
|
745
|
+
return DEFAULT_RECYCLE_LIVE_FRACTION;
|
|
746
|
+
const value = Number.parseFloat(trimmed);
|
|
747
|
+
return Number.isFinite(value) && value > 0 && value <= 1 ? value : DEFAULT_RECYCLE_LIVE_FRACTION;
|
|
748
|
+
}
|
|
749
|
+
/**
|
|
750
|
+
* Parse the emergency-fraction knob; falls back to the default (0.90) on garbage
|
|
751
|
+
* or out-of-range input. Rejects the entire string rather than a numeric prefix.
|
|
499
752
|
*/
|
|
500
|
-
export function
|
|
753
|
+
export function resolveHeapRecycleEmergencyFraction(raw) {
|
|
501
754
|
if (raw === undefined)
|
|
502
|
-
return
|
|
755
|
+
return DEFAULT_RECYCLE_EMERGENCY_FRACTION;
|
|
503
756
|
const trimmed = raw.trim();
|
|
504
757
|
if (!/^\d*\.?\d+$/.test(trimmed))
|
|
505
|
-
return
|
|
758
|
+
return DEFAULT_RECYCLE_EMERGENCY_FRACTION;
|
|
506
759
|
const value = Number.parseFloat(trimmed);
|
|
507
|
-
return Number.isFinite(value) && value > 0 && value <= 1
|
|
760
|
+
return Number.isFinite(value) && value > 0 && value <= 1
|
|
761
|
+
? value
|
|
762
|
+
: DEFAULT_RECYCLE_EMERGENCY_FRACTION;
|
|
508
763
|
}
|
|
509
764
|
/**
|
|
510
|
-
* Parse the
|
|
511
|
-
*
|
|
512
|
-
*
|
|
513
|
-
*
|
|
765
|
+
* Parse the sustain-window knob (whole milliseconds, >= 0). 0 is a VALID value
|
|
766
|
+
* here — it removes the window so two qualifying observations at the same uptime
|
|
767
|
+
* suffice, which the deterministic tests rely on — so only a non-integer or
|
|
768
|
+
* negative value falls back to the 120-second default.
|
|
514
769
|
*/
|
|
515
|
-
export function
|
|
770
|
+
export function resolveHeapRecycleSustainWindowMs(raw) {
|
|
516
771
|
if (raw === undefined)
|
|
517
|
-
return
|
|
772
|
+
return DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
|
|
518
773
|
const trimmed = raw.trim();
|
|
519
774
|
if (!/^\d+$/.test(trimmed))
|
|
520
|
-
return
|
|
775
|
+
return DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
|
|
521
776
|
const value = Number.parseInt(trimmed, 10);
|
|
522
|
-
return Number.isFinite(value) && value >=
|
|
777
|
+
return Number.isFinite(value) && value >= 0 ? value : DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
|
|
523
778
|
}
|
|
524
779
|
/**
|
|
525
|
-
* Parse the minimum-uptime-floor knob (whole milliseconds, >= 0).
|
|
526
|
-
*
|
|
527
|
-
*
|
|
528
|
-
* falls back to the 30-minute default.
|
|
780
|
+
* Parse the minimum-uptime-floor knob (whole milliseconds, >= 0). 0 is a VALID
|
|
781
|
+
* value here — it disables the floor, which the deterministic tests rely on — so
|
|
782
|
+
* only a non-integer or negative value falls back to the 35-minute default.
|
|
529
783
|
*/
|
|
530
784
|
export function resolveHeapRecycleMinUptimeMs(raw) {
|
|
531
785
|
if (raw === undefined)
|