@indigoai-us/hq-cloud 6.16.3 → 6.16.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,31 +7,45 @@
7
7
  * runner shipped no memory telemetry at all, so nothing in the abort evidence
8
8
  * could name which structure held the memory. This module closes that gap: it
9
9
  * samples the process heap on a bounded cadence, attributes live entry counts
10
- * to a fixed set of long-lived structures (the "census"), and — the moment the
11
- * used heap first crosses a configurable fraction of the ceiling — emits one
12
- * escalation record ranking those structures by growth since the session began.
10
+ * to a fixed set of long-lived structures (the "census"), and — before V8
11
+ * aborts — recycles the process so the desktop supervisor respawns a fresh one.
12
+ *
13
+ * The governor keys on the POST-GC LIVE HEAP, not the instantaneous
14
+ * `used_heap_size`. Between major collections `used_heap_size` is live-set plus
15
+ * not-yet-collected garbage, so it sawtooths by a large fraction of the ceiling
16
+ * from minute to minute; a rule that watches it can sit at a sub-threshold
17
+ * sample forever while the true live set climbs to the abort point (the
18
+ * 2026-09-01 regression: a full production day produced ZERO recycle verdicts
19
+ * against 1462 samples while the runner kept aborting). The monotonic quantity
20
+ * is the live set that survives a major GC — exactly what V8 uses for its own
21
+ * OOM verdict — so the primary signal is read on each observed MAJOR GC. A
22
+ * single instantaneous backstop remains for the case where the live signal is
23
+ * unavailable and the heap is already minutes from aborting.
13
24
  *
14
25
  * The whole module is numeric-only and allocation-light by construction:
15
26
  *
16
27
  * - Every emitted value is a number, and every census site is a FIXED
17
- * identifier token, never a path/filename/company/message. Records
18
- * therefore pass the existing telemetry sanitizer unchanged.
28
+ * identifier token, never a path/filename/company/message. Property keys are
29
+ * additionally shaped to survive the vault's ingest contract: they contain
30
+ * none of its denied key substrings and end in `_count`/`_bytes`/`_ms`, so
31
+ * the census attribution is actually stored rather than silently dropped.
19
32
  * - The controller retains only the FIRST sample (the growth baseline) and
20
33
  * the LATEST sample. It never accumulates history — a memory diagnostic
21
34
  * that leaks would regress the very defect it measures.
22
- * - All readers, timers, and emit callbacks are injected, so the module is
23
- * driven deterministically by tests with no real allocation, no real heap
24
- * growth, no wall-clock timing, and no network.
35
+ * - All readers, timers, the GC observer, and emit callbacks are injected, so
36
+ * the module is driven deterministically by tests with no real allocation,
37
+ * no real heap growth, no wall-clock timing, and no network.
25
38
  *
26
39
  * Diagnostics must never affect sync: a throwing reader, a faulty census
27
- * entry, or a rejecting emit callback degrades to silence and never disturbs a
28
- * watch cycle.
40
+ * entry, an unavailable GC entry API, or a rejecting emit callback degrades to
41
+ * silence and never disturbs a watch cycle.
29
42
  */
43
+ import * as perfHooks from "node:perf_hooks";
30
44
  /**
31
45
  * Census tokens are fixed identifiers, never user-derived strings. The pattern
32
46
  * is deliberately stricter than the telemetry sanitizer's SAFE_KEY so a token
33
- * can always be prefixed (`census_`, `count_`, `growth_`) and still fit inside
34
- * the 64-character key budget.
47
+ * can always be prefixed (`census_`) and suffixed (`_count`, `_growth_count`)
48
+ * and still fit inside the 64-character key budget.
35
49
  */
36
50
  const CENSUS_TOKEN = /^[A-Za-z][A-Za-z0-9_]{0,48}$/;
37
51
  /**
@@ -93,13 +107,29 @@ export function sampleRunnerHeap(deps) {
93
107
  function finiteNumber(value) {
94
108
  return typeof value === "number" && Number.isFinite(value) ? value : 0;
95
109
  }
110
+ /** Fold the four cadence counters into a property bag, ahead of per-site keys. */
111
+ function appendCounters(props, counters) {
112
+ if (!counters)
113
+ return;
114
+ props.heap_observations_count = counters.heapObservationsCount;
115
+ props.heap_interval_observations_count = counters.heapIntervalObservationsCount;
116
+ props.heap_checkpoint_observations_count = counters.heapCheckpointObservationsCount;
117
+ props.ms_since_previous_observation_ms = counters.msSincePreviousObservationMs;
118
+ }
119
+ /** Parts-per-thousand, the integer spelling every fraction is emitted as. */
120
+ function toPermille(fraction) {
121
+ return Math.round(fraction * 1000);
122
+ }
96
123
  /**
97
124
  * Flatten a sample into numeric telemetry properties. Raw heap figures carry a
98
- * `heap_`/`rss_`/`external_` prefix; each census site becomes `census_<token>`.
99
- * Every value is a number, so the record survives the telemetry sanitizer with
100
- * no weakening of its path/name/message rejection.
125
+ * `heap_`/`rss_`/`external_` prefix; each census site becomes
126
+ * `census_<token>_count`. Every value is a number AND every key ends in
127
+ * `_count`/`_bytes`/`_ms` and contains none of the vault ingest contract's
128
+ * denied substrings, so the record is both accepted by the local sanitizer and
129
+ * actually STORED by ingest (unlike the pre-fix `census_<token>` spelling,
130
+ * which ingest silently dropped, so the attribution never reached the store).
101
131
  */
102
- export function heapSampleCloudProperties(sample) {
132
+ export function heapSampleCloudProperties(sample, counters) {
103
133
  const props = {
104
134
  heap_used_bytes: sample.usedHeapBytes,
105
135
  heap_total_bytes: sample.totalHeapBytes,
@@ -109,27 +139,31 @@ export function heapSampleCloudProperties(sample) {
109
139
  external_bytes: sample.externalBytes,
110
140
  array_buffers_bytes: sample.arrayBuffersBytes,
111
141
  uptime_ms: sample.uptimeMs,
112
- census_total: sample.censusTotal,
142
+ census_total_count: sample.censusTotal,
113
143
  };
144
+ appendCounters(props, counters);
114
145
  for (const { site, count } of sample.census) {
115
- props[`census_${site}`] = count;
146
+ props[`census_${site}_count`] = count;
116
147
  }
117
148
  return props;
118
149
  }
119
150
  /** Flatten a pressure record into numeric-only telemetry properties. */
120
- export function heapPressureCloudProperties(record) {
151
+ export function heapPressureCloudProperties(record, counters) {
121
152
  const props = {
122
- pressure_used_fraction_milli: Math.round(record.usedFraction * 1000),
123
- pressure_threshold_milli: Math.round(record.thresholdFraction * 1000),
153
+ pressure_used_permille_count: toPermille(record.usedFraction),
154
+ pressure_threshold_permille_count: toPermille(record.thresholdFraction),
124
155
  heap_used_bytes: record.usedHeapBytes,
125
156
  heap_limit_bytes: record.heapLimitBytes,
157
+ heap_effective_limit_bytes: record.effectiveLimitBytes,
158
+ heap_declared_limit_bytes: record.declaredCeilingBytes,
126
159
  rss_bytes: record.rssBytes,
127
160
  external_bytes: record.externalBytes,
128
161
  uptime_ms: record.uptimeMs,
129
162
  };
163
+ appendCounters(props, counters);
130
164
  for (const { site, count, growth } of record.topSites) {
131
- props[`count_${site}`] = count;
132
- props[`growth_${site}`] = growth;
165
+ props[`${site}_count`] = count;
166
+ props[`${site}_growth_count`] = growth;
133
167
  }
134
168
  return props;
135
169
  }
@@ -140,26 +174,45 @@ export function heapPressureStderrLine(record) {
140
174
  .join(",");
141
175
  return ("hq-sync-runner: heap-pressure " +
142
176
  `used=${record.usedHeapBytes} limit=${record.heapLimitBytes} ` +
143
- `frac_milli=${Math.round(record.usedFraction * 1000)} rss=${record.rssBytes} ` +
177
+ `eff=${record.effectiveLimitBytes} ` +
178
+ `permille=${toPermille(record.usedFraction)} rss=${record.rssBytes} ` +
144
179
  `uptime_ms=${record.uptimeMs} top=${top}`);
145
180
  }
146
181
  /** Flatten a recycle record into numeric-only telemetry properties. */
147
- export function heapRecycleCloudProperties(record) {
182
+ export function heapRecycleCloudProperties(record, counters) {
148
183
  const props = {
149
- recycle_used_fraction_milli: Math.round(record.usedFraction * 1000),
150
- recycle_fraction_milli: Math.round(record.recycleFraction * 1000),
151
- recycle_sustained_samples: record.sustainedSamples,
152
- recycle_observed_sustained_samples: record.observedSustainedSamples,
184
+ recycle_used_permille_count: toPermille(record.usedFraction),
185
+ recycle_live_permille_count: toPermille(record.liveFraction),
186
+ recycle_live_floor_permille_count: toPermille(record.recycleLiveFraction),
187
+ recycle_pinned_floor_permille_count: toPermille(record.pinnedFraction),
188
+ recycle_emergency_floor_permille_count: toPermille(record.emergencyFraction),
189
+ recycle_hard_exit_floor_permille_count: toPermille(record.hardExitFraction),
190
+ recycle_emergency_trigger_count: record.trigger === "emergency" ? 1 : 0,
191
+ recycle_pinned_trigger_count: record.trigger === "pinned" ? 1 : 0,
192
+ recycle_latch_timeout_count: record.reason === "latch_timeout" ? 1 : 0,
193
+ recycle_hard_fraction_count: record.reason === "hard_fraction" ? 1 : 0,
194
+ recycle_post_latch_observations_count: record.postLatchObservations,
195
+ recycle_sustain_window_ms: record.sustainWindowMs,
196
+ recycle_observed_sustain_ms: record.observedSustainMs,
153
197
  recycle_min_uptime_ms: record.minUptimeMs,
154
198
  heap_used_bytes: record.usedHeapBytes,
199
+ heap_live_bytes: record.liveHeapBytes,
155
200
  heap_limit_bytes: record.heapLimitBytes,
201
+ heap_effective_limit_bytes: record.effectiveLimitBytes,
202
+ heap_declared_limit_bytes: record.declaredCeilingBytes,
156
203
  rss_bytes: record.rssBytes,
157
204
  external_bytes: record.externalBytes,
205
+ array_buffers_bytes: record.arrayBuffersBytes,
206
+ rss_growth_bytes: record.rssGrowthBytes,
207
+ external_growth_bytes: record.externalGrowthBytes,
208
+ array_buffers_growth_bytes: record.arrayBuffersGrowthBytes,
158
209
  uptime_ms: record.uptimeMs,
210
+ census_total_count: record.censusTotal,
159
211
  };
212
+ appendCounters(props, counters);
160
213
  for (const { site, count, growth } of record.topSites) {
161
- props[`count_${site}`] = count;
162
- props[`growth_${site}`] = growth;
214
+ props[`${site}_count`] = count;
215
+ props[`${site}_growth_count`] = growth;
163
216
  }
164
217
  return props;
165
218
  }
@@ -177,21 +230,24 @@ export function heapRecycleStderrLine(record) {
177
230
  .map((entry) => `${entry.site}:${entry.growth}`)
178
231
  .join(",");
179
232
  return ("hq-sync-runner: heap-recycle " +
180
- `used=${record.usedHeapBytes} limit=${record.heapLimitBytes} ` +
181
- `frac_milli=${Math.round(record.usedFraction * 1000)} rss=${record.rssBytes} ` +
182
- `uptime_ms=${record.uptimeMs} sustained=${record.observedSustainedSamples} top=${top}`);
233
+ `trigger=${record.trigger} reason=${record.reason} ` +
234
+ `used=${record.usedHeapBytes} live=${record.liveHeapBytes} limit=${record.heapLimitBytes} ` +
235
+ `eff=${record.effectiveLimitBytes} ` +
236
+ `live_permille=${toPermille(record.liveFraction)} rss=${record.rssBytes} ` +
237
+ `uptime_ms=${record.uptimeMs} sustain_ms=${record.observedSustainMs} top=${top}`);
183
238
  }
184
239
  const DEFAULT_PRESSURE_THRESHOLD = 0.75;
185
240
  const DEFAULT_TOP_SITE_COUNT = 8;
186
- const DEFAULT_RECYCLE_FRACTION = 0.78;
187
- const DEFAULT_RECYCLE_SUSTAINED_SAMPLES = 2;
188
- const DEFAULT_RECYCLE_MIN_UPTIME_MS = 30 * 60_000;
189
- /**
190
- * Owns the sampling state machine: it keeps the baseline and latest sample, and
191
- * decides — once per rising crossing of the threshold — when to escalate. The
192
- * threshold is derived from the injected `heap_size_limit`, so it self-calibrates
193
- * to whatever `--max-old-space-size` the desktop supplied.
194
- */
241
+ const DEFAULT_RECYCLE_LIVE_FRACTION = 0.6;
242
+ const DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS = 120_000;
243
+ const DEFAULT_RECYCLE_EMERGENCY_FRACTION = 0.9;
244
+ const DEFAULT_RECYCLE_MIN_UPTIME_MS = 35 * 60_000;
245
+ const DEFAULT_RECYCLE_PINNED_FRACTION = 0.84;
246
+ const DEFAULT_RECYCLE_PINNED_SUSTAIN_WINDOW_MS = 20_000;
247
+ const DEFAULT_HARD_EXIT_FRACTION = 0.955;
248
+ const DEFAULT_LATCH_HARD_EXIT_MS = 60_000;
249
+ const DEFAULT_LATCH_HARD_EXIT_OBSERVATIONS = 240;
250
+ const NO_VERDICT = Object.freeze({ recycle: null, hardExit: null });
195
251
  export class RunnerHeapTelemetry {
196
252
  readers;
197
253
  census;
@@ -199,14 +255,35 @@ export class RunnerHeapTelemetry {
199
255
  threshold;
200
256
  topSiteCount;
201
257
  recycleEnabled;
202
- recycleFraction;
203
- recycleSustainedSamples;
258
+ declaredCeilingBytes;
259
+ recycleLiveFraction;
260
+ recycleSustainWindowMs;
261
+ recycleEmergencyFraction;
204
262
  recycleMinUptimeMs;
263
+ pinnedFraction;
264
+ pinnedSustainWindowMs;
265
+ hardExitFraction;
266
+ latchHardExitMs;
267
+ latchHardExitObservations;
205
268
  baseline = null;
206
269
  latest = null;
207
270
  pressureSignalled = false;
208
- recycleSustainedCount = 0;
209
271
  recycleSignalled = false;
272
+ hardExitSignalled = false;
273
+ /** Uptime (ms) of the first live observation at/above the floor in the
274
+ * current consecutive run; null when the last live reading dipped below it. */
275
+ liveAnchorUptimeMs = null;
276
+ /** Uptime (ms) of the first instantaneous observation at/above the pinned
277
+ * floor in the current consecutive run; null when a reading dipped below it. */
278
+ pinnedAnchorUptimeMs = null;
279
+ /** Consecutive instantaneous observations at/above the pinned floor. */
280
+ pinnedObservationCount = 0;
281
+ /** Uptime (ms) at which the FIRST cooperative latch (any tier) armed. */
282
+ latchedAtUptimeMs = null;
283
+ /** The tier that produced the cooperative latch, carried onto a hard exit. */
284
+ latchedTrigger = null;
285
+ /** Observations seen since the cooperative latch armed (drives latch_timeout). */
286
+ postLatchObservations = 0;
210
287
  constructor(options) {
211
288
  this.readers = options.readers;
212
289
  this.census = options.census;
@@ -219,40 +296,224 @@ export class RunnerHeapTelemetry {
219
296
  const count = options.topSiteCount ?? DEFAULT_TOP_SITE_COUNT;
220
297
  this.topSiteCount = Number.isFinite(count) && count > 0 ? Math.trunc(count) : DEFAULT_TOP_SITE_COUNT;
221
298
  this.recycleEnabled = options.recycleEnabled ?? true;
222
- const recycleFraction = options.recycleFraction ?? DEFAULT_RECYCLE_FRACTION;
223
- this.recycleFraction =
224
- Number.isFinite(recycleFraction) && recycleFraction > 0 && recycleFraction <= 1
225
- ? recycleFraction
226
- : DEFAULT_RECYCLE_FRACTION;
227
- const sustained = options.recycleSustainedSamples ?? DEFAULT_RECYCLE_SUSTAINED_SAMPLES;
228
- this.recycleSustainedSamples =
229
- Number.isFinite(sustained) && sustained >= 1
230
- ? Math.trunc(sustained)
231
- : DEFAULT_RECYCLE_SUSTAINED_SAMPLES;
299
+ const declared = options.declaredCeilingBytes;
300
+ this.declaredCeilingBytes =
301
+ typeof declared === "number" && Number.isFinite(declared) && declared > 0
302
+ ? Math.trunc(declared)
303
+ : null;
304
+ this.recycleLiveFraction = fractionOrDefault(options.recycleLiveFraction, DEFAULT_RECYCLE_LIVE_FRACTION);
305
+ const window = options.recycleSustainWindowMs ?? DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
306
+ this.recycleSustainWindowMs =
307
+ Number.isFinite(window) && window >= 0 ? Math.trunc(window) : DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
308
+ this.recycleEmergencyFraction = fractionOrDefault(options.recycleEmergencyFraction, DEFAULT_RECYCLE_EMERGENCY_FRACTION);
232
309
  const minUptime = options.recycleMinUptimeMs ?? DEFAULT_RECYCLE_MIN_UPTIME_MS;
233
310
  this.recycleMinUptimeMs =
234
311
  Number.isFinite(minUptime) && minUptime >= 0
235
312
  ? Math.trunc(minUptime)
236
313
  : DEFAULT_RECYCLE_MIN_UPTIME_MS;
314
+ this.pinnedFraction = fractionOrDefault(options.recyclePinnedFraction, DEFAULT_RECYCLE_PINNED_FRACTION);
315
+ const pinnedWindow = options.recyclePinnedSustainWindowMs ?? DEFAULT_RECYCLE_PINNED_SUSTAIN_WINDOW_MS;
316
+ this.pinnedSustainWindowMs =
317
+ Number.isFinite(pinnedWindow) && pinnedWindow >= 0
318
+ ? Math.trunc(pinnedWindow)
319
+ : DEFAULT_RECYCLE_PINNED_SUSTAIN_WINDOW_MS;
320
+ this.hardExitFraction = fractionOrDefault(options.hardExitFraction, DEFAULT_HARD_EXIT_FRACTION);
321
+ const latchMs = options.latchHardExitMs ?? DEFAULT_LATCH_HARD_EXIT_MS;
322
+ this.latchHardExitMs =
323
+ Number.isFinite(latchMs) && latchMs >= 0 ? Math.trunc(latchMs) : DEFAULT_LATCH_HARD_EXIT_MS;
324
+ const latchObs = options.latchHardExitObservations ?? DEFAULT_LATCH_HARD_EXIT_OBSERVATIONS;
325
+ this.latchHardExitObservations =
326
+ Number.isFinite(latchObs) && latchObs >= 1
327
+ ? Math.trunc(latchObs)
328
+ : DEFAULT_LATCH_HARD_EXIT_OBSERVATIONS;
329
+ }
330
+ /**
331
+ * The one denominator every fraction is computed against: the declared
332
+ * `--max-old-space-size` when known (it is the real V8 ceiling the aborts
333
+ * respect), else V8's reported `heap_size_limit`. Never larger than reported.
334
+ */
335
+ effectiveLimit(heapLimitBytes) {
336
+ if (this.declaredCeilingBytes === null)
337
+ return heapLimitBytes;
338
+ return Math.min(this.declaredCeilingBytes, heapLimitBytes);
339
+ }
340
+ uptimeNow() {
341
+ return Math.max(0, Math.trunc(this.readers.monotonicNowMs() - this.sessionStartedAtMs));
237
342
  }
238
343
  /**
239
344
  * Capture the growth baseline when sampling starts, before the first interval,
240
- * WITHOUT emitting or arming the pressure signal. Without this, a runner that
241
- * is already above the threshold at its first scheduled sample would make that
242
- * same high-water sample its own baseline, so the single crossing record it
243
- * emits would report zero growth for every site and lose the attribution.
345
+ * WITHOUT emitting or arming any signal. Without this, a runner that is already
346
+ * above the threshold at its first scheduled sample would make that same high
347
+ * sample its own baseline, so a crossing record would report zero growth for
348
+ * every site and lose the attribution.
244
349
  */
245
350
  primeBaseline() {
246
- const sample = sampleRunnerHeap({
247
- readers: this.readers,
248
- census: this.census,
249
- sessionStartedAtMs: this.sessionStartedAtMs,
250
- });
251
- this.baseline ??= sample;
252
- this.latest = sample;
253
- return sample;
351
+ return this.buildSample();
254
352
  }
353
+ /**
354
+ * The interval sampler tick. Emits a sample and (at most once per rising
355
+ * crossing) a pressure record, then runs the instantaneous tiers (pinned,
356
+ * emergency, hard). The primary post-GC live verdict comes from
357
+ * {@link observeMajorGc}; this instantaneous path catches the starved
358
+ * synchronous stretch where no post-GC reading is ever taken.
359
+ */
255
360
  sample() {
361
+ const sample = this.buildSample();
362
+ const pressure = this.detectPressure(sample);
363
+ const { recycle, hardExit } = this.observeInstant(sample.usedHeapBytes, sample.heapLimitBytes, sample.uptimeMs, () => sample);
364
+ return { sample, pressure, recycle, hardExit };
365
+ }
366
+ /**
367
+ * The PRIMARY governor path, driven by an observed MAJOR GC. The caller
368
+ * ({@link nodeMajorGcObserver}) only invokes this for a GC whose reading it
369
+ * could capture PROMPTLY — a callback delayed by a blocked event loop is
370
+ * skipped upstream, because `getHeapStatistics()` read late would be an
371
+ * instantaneous sawtooth value (live set plus everything allocated since the
372
+ * GC), not the post-GC live floor.
373
+ *
374
+ * The anchor is maintained on EVERY observed major GC via a cheap
375
+ * `getHeapStatistics()` read (no throttle), so a post-GC reading that dips
376
+ * below the floor ALWAYS invalidates the sustain run — a throttle that skipped
377
+ * such a reading would let a stale anchor fake a continuous run. The fuller
378
+ * sample (census snapshot + memory deltas) is taken only at the verdict, so a
379
+ * GC storm never costs more than the O(1) statistics read here. The fraction
380
+ * is denominated in the EFFECTIVE limit (min(declared, reported)) — the only
381
+ * change from the shipped live-tier logic.
382
+ */
383
+ observeMajorGc() {
384
+ if (!this.recycleEnabled)
385
+ return NO_VERDICT;
386
+ const stats = this.readers.getHeapStatistics();
387
+ const heapLimitBytes = finiteNumber(stats.heap_size_limit);
388
+ const effective = this.effectiveLimit(heapLimitBytes);
389
+ if (effective <= 0)
390
+ return NO_VERDICT;
391
+ const liveFraction = finiteNumber(stats.used_heap_size) / effective;
392
+ const uptimeMs = this.uptimeNow();
393
+ // T4 hard: a post-GC reading is the cleanest instantaneous value, so the
394
+ // hard-fraction and the aged-latch backstops are evaluated here too.
395
+ const hardExit = this.checkHardExit(liveFraction, uptimeMs);
396
+ if (hardExit)
397
+ return { recycle: null, hardExit };
398
+ if (this.recycleSignalled)
399
+ return NO_VERDICT;
400
+ // T3 emergency: a genuine post-GC reading already abort-adjacent — cooperative
401
+ // recycle now, bypassing both the sustain window and the uptime floor.
402
+ if (liveFraction >= this.recycleEmergencyFraction) {
403
+ return { recycle: this.armCooperativeLatch("emergency", 0), hardExit: null };
404
+ }
405
+ // T1 live: the post-GC live floor held across the sustain window.
406
+ if (liveFraction < this.recycleLiveFraction) {
407
+ // The live floor dipped: reset the anchor so the window only ever measures
408
+ // a genuinely continuous run at/above the floor.
409
+ this.liveAnchorUptimeMs = null;
410
+ return NO_VERDICT;
411
+ }
412
+ if (this.liveAnchorUptimeMs === null) {
413
+ // First qualifying observation. A second, at least a window later, confirms.
414
+ this.liveAnchorUptimeMs = uptimeMs;
415
+ return NO_VERDICT;
416
+ }
417
+ const observedSustainMs = Math.max(0, uptimeMs - this.liveAnchorUptimeMs);
418
+ if (observedSustainMs < this.recycleSustainWindowMs)
419
+ return NO_VERDICT;
420
+ if (uptimeMs < this.recycleMinUptimeMs)
421
+ return NO_VERDICT;
422
+ return { recycle: this.armCooperativeLatch("live", observedSustainMs), hardExit: null };
423
+ }
424
+ /**
425
+ * The loop-top / work-driven BACKSTOP against timer starvation (the observed
426
+ * 11-minute sampler hole). Reads the instantaneous heap and runs the
427
+ * instantaneous tiers (hard, emergency, pinned). An instantaneous reading can
428
+ * never confirm the post-GC live floor, so it never feeds the live tier's
429
+ * sustain window. Cheap: an O(1) statistics read, sampling the census only if
430
+ * a verdict actually fires.
431
+ */
432
+ evaluate() {
433
+ if (!this.recycleEnabled)
434
+ return NO_VERDICT;
435
+ const stats = this.readers.getHeapStatistics();
436
+ const heapLimitBytes = finiteNumber(stats.heap_size_limit);
437
+ const usedHeapBytes = finiteNumber(stats.used_heap_size);
438
+ const uptimeMs = this.uptimeNow();
439
+ return this.observeInstant(usedHeapBytes, heapLimitBytes, uptimeMs, () => this.buildSample());
440
+ }
441
+ /**
442
+ * The shared instantaneous tier pipeline (interval tick, loop-top, and the
443
+ * work-driven checkpoint all reach it). Order: T4 hard (hard-fraction, then an
444
+ * aged-out latch) → T3 emergency (single observation, bypasses window+floor) →
445
+ * T2 pinned (instantaneous fraction sustained across the pinned window past the
446
+ * uptime floor, with any dip resetting the anchor). The full sample is built
447
+ * only at a verdict, via `buildVerdictSample`.
448
+ */
449
+ observeInstant(usedHeapBytes, heapLimitBytes, uptimeMs, buildVerdictSample) {
450
+ if (!this.recycleEnabled)
451
+ return NO_VERDICT;
452
+ const effective = this.effectiveLimit(heapLimitBytes);
453
+ if (effective <= 0)
454
+ return NO_VERDICT;
455
+ const usedFraction = usedHeapBytes / effective;
456
+ const hardExit = this.checkHardExit(usedFraction, uptimeMs, buildVerdictSample);
457
+ if (hardExit)
458
+ return { recycle: null, hardExit };
459
+ if (this.recycleSignalled)
460
+ return NO_VERDICT;
461
+ // T3 emergency: single instantaneous observation, bypasses window + floor.
462
+ if (usedFraction >= this.recycleEmergencyFraction) {
463
+ return { recycle: this.armCooperativeLatch("emergency", 0, buildVerdictSample), hardExit: null };
464
+ }
465
+ // T2 pinned: instantaneous fraction held across the pinned window.
466
+ if (usedFraction < this.pinnedFraction) {
467
+ this.pinnedAnchorUptimeMs = null;
468
+ this.pinnedObservationCount = 0;
469
+ return NO_VERDICT;
470
+ }
471
+ if (this.pinnedAnchorUptimeMs === null) {
472
+ this.pinnedAnchorUptimeMs = uptimeMs;
473
+ this.pinnedObservationCount = 1;
474
+ return NO_VERDICT;
475
+ }
476
+ this.pinnedObservationCount += 1;
477
+ const observedSustainMs = Math.max(0, uptimeMs - this.pinnedAnchorUptimeMs);
478
+ if (observedSustainMs < this.pinnedSustainWindowMs)
479
+ return NO_VERDICT;
480
+ if (this.pinnedObservationCount < 2)
481
+ return NO_VERDICT;
482
+ if (uptimeMs < this.recycleMinUptimeMs)
483
+ return NO_VERDICT;
484
+ return {
485
+ recycle: this.armCooperativeLatch("pinned", observedSustainMs, buildVerdictSample),
486
+ hardExit: null,
487
+ };
488
+ }
489
+ /**
490
+ * T4 hard-exit evaluation, run at EVERY observation point. Fires at most once
491
+ * per session (independent of the cooperative latch), on either arm:
492
+ * (a) `hard_fraction` — the instantaneous fraction is at/above the hard line;
493
+ * (b) `latch_timeout` — a cooperative latch aged past its time OR observation
494
+ * deadline without the loop honouring it.
495
+ * The `latch_timeout` observation counter advances on every later observation,
496
+ * never on the observation that armed the latch (that call sees a still-false
497
+ * `recycleSignalled` before the cooperative tier runs).
498
+ */
499
+ checkHardExit(fraction, uptimeMs, buildVerdictSample) {
500
+ if (!this.recycleEnabled || this.hardExitSignalled)
501
+ return null;
502
+ const build = buildVerdictSample ?? (() => this.buildSample());
503
+ if (Number.isFinite(fraction) && fraction >= this.hardExitFraction) {
504
+ return this.armHardExit(build(), "hard_fraction");
505
+ }
506
+ if (this.recycleSignalled && this.latchedAtUptimeMs !== null) {
507
+ this.postLatchObservations += 1;
508
+ const agedMs = uptimeMs - this.latchedAtUptimeMs;
509
+ if (agedMs >= this.latchHardExitMs ||
510
+ this.postLatchObservations >= this.latchHardExitObservations) {
511
+ return this.armHardExit(build(), "latch_timeout");
512
+ }
513
+ }
514
+ return null;
515
+ }
516
+ buildSample() {
256
517
  const sample = sampleRunnerHeap({
257
518
  readers: this.readers,
258
519
  census: this.census,
@@ -262,60 +523,63 @@ export class RunnerHeapTelemetry {
262
523
  // difference between the two; no per-sample history is ever kept.
263
524
  this.baseline ??= sample;
264
525
  this.latest = sample;
265
- return {
266
- sample,
267
- pressure: this.detectPressure(sample),
268
- recycle: this.detectRecycle(sample),
269
- };
526
+ return sample;
270
527
  }
271
- /**
272
- * The governor decision. Returns a record at most once per session, and only
273
- * when the heap has stayed at or above the recycle fraction for a sustained
274
- * run of consecutive samples AND the process has outlived the uptime floor.
275
- * The sustained counter is maintained on every sample so the streak is real
276
- * consecutive pressure, but the verdict itself is gated on the floor, so a
277
- * recycle can never fire early however high the heap climbs.
278
- */
279
- detectRecycle(sample) {
280
- if (!this.recycleEnabled)
281
- return null;
282
- if (sample.heapLimitBytes <= 0)
283
- return null;
284
- const usedFraction = sample.usedHeapBytes / sample.heapLimitBytes;
285
- if (usedFraction >= this.recycleFraction) {
286
- this.recycleSustainedCount += 1;
287
- }
288
- else {
289
- // A dip below the fraction breaks the streak: a single spike can never
290
- // accumulate the sustained run the verdict requires.
291
- this.recycleSustainedCount = 0;
292
- return null;
293
- }
294
- if (this.recycleSignalled)
295
- return null;
296
- if (sample.uptimeMs < this.recycleMinUptimeMs)
297
- return null;
298
- if (this.recycleSustainedCount < this.recycleSustainedSamples)
299
- return null;
528
+ /** Arm the one cooperative latch (from any tier) and build its verdict. */
529
+ armCooperativeLatch(trigger, observedSustainMs, buildVerdictSample) {
300
530
  this.recycleSignalled = true;
531
+ const sample = (buildVerdictSample ?? (() => this.buildSample()))();
532
+ if (this.latchedAtUptimeMs === null) {
533
+ this.latchedAtUptimeMs = sample.uptimeMs;
534
+ this.latchedTrigger = trigger;
535
+ this.postLatchObservations = 0;
536
+ }
537
+ return this.buildRecord(sample, trigger, "cooperative", observedSustainMs);
538
+ }
539
+ /** Arm the one hard exit and build its verdict (`hard_fraction`/`latch_timeout`). */
540
+ armHardExit(sample, reason) {
541
+ this.hardExitSignalled = true;
542
+ const trigger = this.latchedTrigger ?? "emergency";
543
+ return this.buildRecord(sample, trigger, reason, 0);
544
+ }
545
+ buildRecord(sample, trigger, reason, observedSustainMs) {
546
+ const effective = this.effectiveLimit(sample.heapLimitBytes);
547
+ const usedFraction = effective > 0 ? sample.usedHeapBytes / effective : 0;
548
+ const baseline = this.baseline;
301
549
  return {
550
+ trigger,
551
+ reason,
302
552
  usedHeapBytes: sample.usedHeapBytes,
553
+ liveHeapBytes: sample.usedHeapBytes,
303
554
  heapLimitBytes: sample.heapLimitBytes,
555
+ effectiveLimitBytes: effective,
556
+ declaredCeilingBytes: this.declaredCeilingBytes ?? 0,
304
557
  usedFraction,
305
- recycleFraction: this.recycleFraction,
306
- sustainedSamples: this.recycleSustainedSamples,
307
- observedSustainedSamples: this.recycleSustainedCount,
558
+ liveFraction: usedFraction,
559
+ recycleLiveFraction: this.recycleLiveFraction,
560
+ pinnedFraction: this.pinnedFraction,
561
+ emergencyFraction: this.recycleEmergencyFraction,
562
+ hardExitFraction: this.hardExitFraction,
563
+ sustainWindowMs: this.recycleSustainWindowMs,
564
+ observedSustainMs,
308
565
  minUptimeMs: this.recycleMinUptimeMs,
566
+ postLatchObservations: reason === "latch_timeout" ? this.postLatchObservations : 0,
567
+ uptimeMs: sample.uptimeMs,
309
568
  rssBytes: sample.rssBytes,
310
569
  externalBytes: sample.externalBytes,
311
- uptimeMs: sample.uptimeMs,
570
+ arrayBuffersBytes: sample.arrayBuffersBytes,
571
+ rssGrowthBytes: sample.rssBytes - (baseline?.rssBytes ?? sample.rssBytes),
572
+ externalGrowthBytes: sample.externalBytes - (baseline?.externalBytes ?? sample.externalBytes),
573
+ arrayBuffersGrowthBytes: sample.arrayBuffersBytes - (baseline?.arrayBuffersBytes ?? sample.arrayBuffersBytes),
574
+ censusTotal: sample.censusTotal,
312
575
  topSites: this.topSitesByGrowth(sample),
313
576
  };
314
577
  }
315
578
  detectPressure(sample) {
316
- if (sample.heapLimitBytes <= 0)
579
+ const effective = this.effectiveLimit(sample.heapLimitBytes);
580
+ if (effective <= 0)
317
581
  return null;
318
- const usedFraction = sample.usedHeapBytes / sample.heapLimitBytes;
582
+ const usedFraction = sample.usedHeapBytes / effective;
319
583
  if (usedFraction < this.threshold) {
320
584
  // Re-arm below the threshold so a genuine later crossing escalates again.
321
585
  this.pressureSignalled = false;
@@ -327,6 +591,8 @@ export class RunnerHeapTelemetry {
327
591
  return {
328
592
  usedHeapBytes: sample.usedHeapBytes,
329
593
  heapLimitBytes: sample.heapLimitBytes,
594
+ effectiveLimitBytes: effective,
595
+ declaredCeilingBytes: this.declaredCeilingBytes ?? 0,
330
596
  usedFraction,
331
597
  thresholdFraction: this.threshold,
332
598
  rssBytes: sample.rssBytes,
@@ -355,18 +621,107 @@ export class RunnerHeapTelemetry {
355
621
  return this.latest;
356
622
  }
357
623
  }
624
+ function fractionOrDefault(value, fallback) {
625
+ return typeof value === "number" && Number.isFinite(value) && value > 0 && value <= 1
626
+ ? value
627
+ : fallback;
628
+ }
629
+ /**
630
+ * The maximum age (ms) of a major-GC PerformanceEntry, at the moment its
631
+ * observer callback runs, for the post-GC heap read to be trustworthy. GC
632
+ * entries are delivered ASYNCHRONOUSLY, so when the event loop was blocked the
633
+ * callback fires long after the collection and a `getHeapStatistics()` read then
634
+ * reflects the live set PLUS everything allocated since — an instantaneous
635
+ * sawtooth value, not the post-GC live floor. A reading older than this is
636
+ * skipped rather than fed to the sustain window, so only promptly-captured
637
+ * (genuinely post-GC) readings can arm a live recycle.
638
+ */
639
+ export const MAX_FRESH_GC_READING_MS = 250;
640
+ /**
641
+ * Whether a major-GC reading taken at `nowMs` is fresh enough to treat as a
642
+ * post-GC live-heap snapshot, given the entry's end time (`entry.startTime +
643
+ * entry.duration`). Exported for deterministic testing of the staleness gate.
644
+ */
645
+ export function isFreshGcReading(gcEndMs, nowMs, maxAgeMs = MAX_FRESH_GC_READING_MS) {
646
+ if (!Number.isFinite(gcEndMs) || !Number.isFinite(nowMs))
647
+ return false;
648
+ return nowMs - gcEndMs <= maxAgeMs;
649
+ }
650
+ /**
651
+ * Production major-GC observer over `perf_hooks`. Everything is wrapped so a
652
+ * runtime without the GC PerformanceEntry API, or one whose entries omit
653
+ * `detail.kind`, degrades to a silent no-op rather than throwing into the
654
+ * runner — in that degraded case the instantaneous emergency backstop and the
655
+ * pressure diagnostics remain active, so the governor is never worse than base.
656
+ *
657
+ * It only invokes the callback for a major GC whose reading it can capture
658
+ * PROMPTLY ({@link isFreshGcReading}): a callback delayed by a blocked event
659
+ * loop would read an inflated instantaneous heap rather than the post-GC live
660
+ * floor, so it is skipped.
661
+ */
662
+ export function nodeMajorGcObserver() {
663
+ let observer = null;
664
+ return {
665
+ start(onMajorGc) {
666
+ try {
667
+ const PerformanceObserverCtor = perfHooks.PerformanceObserver;
668
+ const majorKind = perfHooks.constants?.NODE_PERFORMANCE_GC_MAJOR;
669
+ if (typeof PerformanceObserverCtor !== "function" || typeof majorKind !== "number") {
670
+ // No usable major-GC signal on this runtime: stay silent.
671
+ observer = null;
672
+ return;
673
+ }
674
+ observer = new PerformanceObserverCtor((list) => {
675
+ try {
676
+ const nowMs = perfHooks.performance.now();
677
+ for (const entry of list.getEntries()) {
678
+ const kind = entry.detail?.kind;
679
+ if (typeof kind !== "number" || kind !== majorKind)
680
+ continue;
681
+ const gcEndMs = entry.startTime + (entry.duration ?? 0);
682
+ if (!isFreshGcReading(gcEndMs, nowMs))
683
+ continue;
684
+ onMajorGc();
685
+ }
686
+ }
687
+ catch {
688
+ /* a mis-shaped entry or a throwing callback must never surface */
689
+ }
690
+ });
691
+ observer.observe({ entryTypes: ["gc"] });
692
+ }
693
+ catch {
694
+ observer = null;
695
+ }
696
+ },
697
+ stop() {
698
+ try {
699
+ observer?.disconnect();
700
+ }
701
+ catch {
702
+ /* disconnecting is best-effort during shutdown */
703
+ }
704
+ observer = null;
705
+ },
706
+ };
707
+ }
358
708
  const DEFAULT_SAMPLE_INTERVAL_MS = 60_000;
709
+ const DEFAULT_CHECKPOINT_MS = 250;
359
710
  /**
360
- * Start periodic sampling. The timer is `unref`'d so it can never hold the
361
- * process open, and `stop()` clears it on every shutdown path. Each tick — and
362
- * every emit callback it drives — is wrapped so a fault degrades to silence and
363
- * never affects the watch cycle.
711
+ * Start periodic sampling and, when a GC observer is supplied, the primary
712
+ * live-heap path. The interval timer is `unref`'d so it can never hold the
713
+ * process open, and `stop()` clears it and disconnects the GC observer on every
714
+ * shutdown path. The interval tick, the work-driven checkpoint, and the loop-top
715
+ * `evaluate()` share ONE observation pipeline, one last-observation timestamp,
716
+ * and one set of cadence counters. Each observation — and every emit callback it
717
+ * drives — is wrapped so a fault degrades to silence and never affects the watch
718
+ * cycle.
364
719
  */
365
720
  export function startRunnerHeapSampling(options) {
366
721
  const telemetry = new RunnerHeapTelemetry(options);
367
722
  // Capture the baseline immediately, before the first interval, so the first
368
- // pressure crossing attributes growth against session start rather than
369
- // against a first sample that may already sit at the ceiling.
723
+ // pressure crossing (and every growth delta) attributes against session start
724
+ // rather than against a first sample that may already sit at the ceiling.
370
725
  try {
371
726
  telemetry.primeBaseline();
372
727
  }
@@ -377,43 +732,128 @@ export function startRunnerHeapSampling(options) {
377
732
  const intervalMs = Number.isFinite(options.intervalMs) && options.intervalMs > 0
378
733
  ? Math.trunc(options.intervalMs)
379
734
  : DEFAULT_SAMPLE_INTERVAL_MS;
735
+ const checkpointMs = options.checkpointMs === undefined
736
+ ? DEFAULT_CHECKPOINT_MS
737
+ : Number.isFinite(options.checkpointMs) && options.checkpointMs >= 0
738
+ ? Math.trunc(options.checkpointMs)
739
+ : DEFAULT_CHECKPOINT_MS;
380
740
  let stopped = false;
381
- const tick = () => {
382
- if (stopped)
383
- return;
384
- try {
385
- const { sample, pressure, recycle } = telemetry.sample();
741
+ // Shared observation state — interval, checkpoint, and loop-top feed the SAME
742
+ // pipeline, one last-observation timestamp, and one set of cadence counters.
743
+ let lastObservationAtMs = null;
744
+ let observationCount = 0;
745
+ let intervalObservationCount = 0;
746
+ let checkpointObservationCount = 0;
747
+ const snapshotCounters = (msSincePrev) => ({
748
+ heapObservationsCount: observationCount,
749
+ heapIntervalObservationsCount: intervalObservationCount,
750
+ heapCheckpointObservationsCount: checkpointObservationCount,
751
+ msSincePreviousObservationMs: msSincePrev,
752
+ });
753
+ // Deliver at most one verdict: a hard exit preempts the cooperative recycle on
754
+ // the same observation (the process is leaving now); both callbacks are wrapped
755
+ // so a fault degrades to silence and never disturbs a watch cycle.
756
+ const deliver = (recycle, hardExit, counters) => {
757
+ if (hardExit && options.onHardExit) {
386
758
  try {
387
- options.onSample(sample);
759
+ options.onHardExit(hardExit, counters);
388
760
  }
389
761
  catch (err) {
390
762
  warn(options.log, err);
391
763
  }
392
- if (pressure) {
764
+ }
765
+ else if (recycle) {
766
+ try {
767
+ options.onRecycle?.(recycle, counters);
768
+ }
769
+ catch (err) {
770
+ warn(options.log, err);
771
+ }
772
+ }
773
+ };
774
+ // The one observation pipeline. `source` distinguishes the 60 s interval tick
775
+ // (which emits a cloud sample) from a checkpoint/loop-top (which never does).
776
+ // `gated` is true only for the work-driven checkpoint: it is throttled to
777
+ // `checkpointMs` and disabled entirely when `checkpointMs <= 0`. The loop-top
778
+ // `evaluate()` passes gated=false so it keeps working with checkpoints off,
779
+ // preserving the shipped behaviour exactly.
780
+ const runObservation = (source, gated) => {
781
+ if (stopped)
782
+ return;
783
+ try {
784
+ const now = options.readers.monotonicNowMs();
785
+ if (gated) {
786
+ if (checkpointMs <= 0)
787
+ return; // checkpoints disabled
788
+ if (lastObservationAtMs !== null && now - lastObservationAtMs < checkpointMs) {
789
+ return; // below the gate: one clock compare and out
790
+ }
791
+ }
792
+ const msSincePrev = lastObservationAtMs === null ? 0 : Math.max(0, Math.trunc(now - lastObservationAtMs));
793
+ lastObservationAtMs = now;
794
+ observationCount += 1;
795
+ if (source === "interval")
796
+ intervalObservationCount += 1;
797
+ else
798
+ checkpointObservationCount += 1;
799
+ const counters = snapshotCounters(msSincePrev);
800
+ if (source === "interval") {
801
+ const { sample, pressure, recycle, hardExit } = telemetry.sample();
393
802
  try {
394
- options.onPressure(pressure);
803
+ options.onSample(sample, counters);
395
804
  }
396
805
  catch (err) {
397
806
  warn(options.log, err);
398
807
  }
808
+ if (pressure) {
809
+ try {
810
+ options.onPressure(pressure, counters);
811
+ }
812
+ catch (err) {
813
+ warn(options.log, err);
814
+ }
815
+ }
816
+ deliver(recycle, hardExit, counters);
399
817
  }
400
- if (recycle) {
818
+ else {
819
+ const { recycle, hardExit } = telemetry.evaluate();
820
+ deliver(recycle, hardExit, counters);
821
+ }
822
+ }
823
+ catch (err) {
824
+ warn(options.log, err);
825
+ }
826
+ };
827
+ const handle = scheduler.setInterval(() => runObservation("interval", false), intervalMs);
828
+ handle.unref?.();
829
+ // The primary path: a major GC drives one post-GC live evaluation. It also
830
+ // evaluates the hard tiers (a post-GC reading is the cleanest instantaneous
831
+ // value), so a latch that aged out or a hard-fraction crossing hard-exits here
832
+ // too. It is orthogonal to the cadence gate, so it never touches its counters.
833
+ const gcObserver = options.gcObserver;
834
+ if (gcObserver) {
835
+ try {
836
+ gcObserver.start(() => {
837
+ if (stopped)
838
+ return;
401
839
  try {
402
- options.onRecycle?.(recycle);
840
+ const { recycle, hardExit } = telemetry.observeMajorGc();
841
+ if (recycle || hardExit)
842
+ deliver(recycle, hardExit, snapshotCounters(0));
403
843
  }
404
844
  catch (err) {
405
845
  warn(options.log, err);
406
846
  }
407
- }
847
+ });
408
848
  }
409
849
  catch (err) {
410
850
  warn(options.log, err);
411
851
  }
412
- };
413
- const handle = scheduler.setInterval(tick, intervalMs);
414
- handle.unref?.();
852
+ }
415
853
  return {
416
854
  telemetry,
855
+ evaluate: () => runObservation("checkpoint", false),
856
+ maybeObserve: () => runObservation("checkpoint", true),
417
857
  stop: () => {
418
858
  if (stopped)
419
859
  return;
@@ -424,9 +864,43 @@ export function startRunnerHeapSampling(options) {
424
864
  catch {
425
865
  /* clearing is best-effort during shutdown */
426
866
  }
867
+ try {
868
+ gcObserver?.stop();
869
+ }
870
+ catch {
871
+ /* disconnecting is best-effort during shutdown */
872
+ }
427
873
  },
428
874
  };
429
875
  }
876
+ /**
877
+ * Perform a deliberate hard exit for a `hard_fraction`/`latch_timeout` verdict.
878
+ * Writes the additive `heap-recycle` NDJSON record to stdout and the compact
879
+ * numeric stderr line — through SYNCHRONOUS injected writers, so both records
880
+ * land even when the streams are buffered or blocked — and only THEN exits. The
881
+ * exit code and the writers are injected so the exact write-before-exit ordering
882
+ * is unit-testable and the code stays the runner's single benign-exit constant.
883
+ */
884
+ export function performHeapHardExit(record, io, exitCode) {
885
+ // Both diagnostic writes are BEST-EFFORT: if the supervisor or a pipeline has
886
+ // closed stdout/stderr, a synchronous write can throw — but the exit is the
887
+ // whole point of this path (it replaces the V8 abort), so it must still run.
888
+ // The verdict is already latched one-shot, so a swallowed write with no exit
889
+ // would let the process climb on to the OOM this path exists to prevent.
890
+ try {
891
+ io.writeStdout(`${JSON.stringify({ type: "heap-recycle", ...record })}\n`);
892
+ }
893
+ catch {
894
+ /* stream closed — the fd-1 record is lost, but the exit still happens */
895
+ }
896
+ try {
897
+ io.writeStderr(`${heapRecycleStderrLine(record)}\n`);
898
+ }
899
+ catch {
900
+ /* stream closed — the stderr line is lost, but the exit still happens */
901
+ }
902
+ io.exit(exitCode);
903
+ }
430
904
  const defaultScheduler = {
431
905
  setInterval: (handler, ms) => setInterval(handler, ms),
432
906
  clearInterval: (handle) => clearInterval(handle),
@@ -443,14 +917,27 @@ function warn(log, err) {
443
917
  export const HEAP_SAMPLE_INTERVAL_ENV = "HQ_SYNC_RUNNER_HEAP_SAMPLE_MS";
444
918
  export const HEAP_PRESSURE_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_PRESSURE_FRACTION";
445
919
  export const HEAP_RECYCLE_ENABLED_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_ENABLED";
446
- export const HEAP_RECYCLE_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_FRACTION";
447
- export const HEAP_RECYCLE_SUSTAINED_SAMPLES_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_SUSTAINED_SAMPLES";
920
+ export const HEAP_RECYCLE_LIVE_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_LIVE_FRACTION";
921
+ export const HEAP_RECYCLE_SUSTAIN_WINDOW_MS_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_SUSTAIN_WINDOW_MS";
922
+ export const HEAP_RECYCLE_EMERGENCY_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_EMERGENCY_FRACTION";
448
923
  export const HEAP_RECYCLE_MIN_UPTIME_MS_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_MIN_UPTIME_MS";
924
+ export const HEAP_CHECKPOINT_MS_ENV = "HQ_SYNC_RUNNER_HEAP_CHECKPOINT_MS";
925
+ export const HEAP_PINNED_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_PINNED_FRACTION";
926
+ export const HEAP_PINNED_SUSTAIN_MS_ENV = "HQ_SYNC_RUNNER_HEAP_PINNED_SUSTAIN_MS";
927
+ export const HEAP_HARD_EXIT_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_HARD_EXIT_FRACTION";
928
+ export const HEAP_RECYCLE_LATCH_HARD_EXIT_MS_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_LATCH_HARD_EXIT_MS";
449
929
  export const DEFAULT_HEAP_SAMPLE_INTERVAL_MS = DEFAULT_SAMPLE_INTERVAL_MS;
450
930
  export const DEFAULT_HEAP_PRESSURE_FRACTION = DEFAULT_PRESSURE_THRESHOLD;
451
- export const DEFAULT_HEAP_RECYCLE_FRACTION = DEFAULT_RECYCLE_FRACTION;
452
- export const DEFAULT_HEAP_RECYCLE_SUSTAINED_SAMPLES = DEFAULT_RECYCLE_SUSTAINED_SAMPLES;
931
+ export const DEFAULT_HEAP_RECYCLE_LIVE_FRACTION = DEFAULT_RECYCLE_LIVE_FRACTION;
932
+ export const DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS = DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
933
+ export const DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION = DEFAULT_RECYCLE_EMERGENCY_FRACTION;
453
934
  export const DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS = DEFAULT_RECYCLE_MIN_UPTIME_MS;
935
+ export const DEFAULT_HEAP_CHECKPOINT_MS = DEFAULT_CHECKPOINT_MS;
936
+ export const DEFAULT_HEAP_PINNED_FRACTION = DEFAULT_RECYCLE_PINNED_FRACTION;
937
+ export const DEFAULT_HEAP_PINNED_SUSTAIN_MS = DEFAULT_RECYCLE_PINNED_SUSTAIN_WINDOW_MS;
938
+ export const DEFAULT_HEAP_HARD_EXIT_FRACTION = DEFAULT_HARD_EXIT_FRACTION;
939
+ export const DEFAULT_HEAP_RECYCLE_LATCH_HARD_EXIT_MS = DEFAULT_LATCH_HARD_EXIT_MS;
940
+ export const DEFAULT_HEAP_RECYCLE_LATCH_HARD_EXIT_OBSERVATIONS = DEFAULT_LATCH_HARD_EXIT_OBSERVATIONS;
454
941
  /**
455
942
  * Parse the sampling-interval env knob (whole milliseconds). Rejects the ENTIRE
456
943
  * string: a value like "60s" must fall back to the default, not silently parse
@@ -493,39 +980,53 @@ export function resolveHeapRecycleEnabled(raw) {
493
980
  return !(trimmed === "0" || trimmed === "false" || trimmed === "no" || trimmed === "off");
494
981
  }
495
982
  /**
496
- * Parse the recycle-fraction knob; falls back to the default (0.78) on garbage
497
- * or out-of-range input. Rejects the entire string (e.g. "0.8x" falls back)
498
- * rather than accepting a numeric prefix.
983
+ * Parse the primary live-fraction knob; falls back to the default (0.60) on
984
+ * garbage or out-of-range input. Rejects the entire string (e.g. "0.6x" falls
985
+ * back) rather than accepting a numeric prefix.
499
986
  */
500
- export function resolveHeapRecycleFraction(raw) {
987
+ export function resolveHeapRecycleLiveFraction(raw) {
501
988
  if (raw === undefined)
502
- return DEFAULT_RECYCLE_FRACTION;
989
+ return DEFAULT_RECYCLE_LIVE_FRACTION;
503
990
  const trimmed = raw.trim();
504
991
  if (!/^\d*\.?\d+$/.test(trimmed))
505
- return DEFAULT_RECYCLE_FRACTION;
992
+ return DEFAULT_RECYCLE_LIVE_FRACTION;
506
993
  const value = Number.parseFloat(trimmed);
507
- return Number.isFinite(value) && value > 0 && value <= 1 ? value : DEFAULT_RECYCLE_FRACTION;
994
+ return Number.isFinite(value) && value > 0 && value <= 1 ? value : DEFAULT_RECYCLE_LIVE_FRACTION;
508
995
  }
509
996
  /**
510
- * Parse the sustained-samples knob (a whole count >= 1). Falls back to the
511
- * default (2) on garbage, 0, or a non-integer — a degenerate 0 would let a
512
- * single spike recycle, exactly what the sustained requirement exists to
513
- * prevent, so it is refused rather than honoured.
997
+ * Parse the emergency-fraction knob; falls back to the default (0.90) on garbage
998
+ * or out-of-range input. Rejects the entire string rather than a numeric prefix.
514
999
  */
515
- export function resolveHeapRecycleSustainedSamples(raw) {
1000
+ export function resolveHeapRecycleEmergencyFraction(raw) {
516
1001
  if (raw === undefined)
517
- return DEFAULT_RECYCLE_SUSTAINED_SAMPLES;
1002
+ return DEFAULT_RECYCLE_EMERGENCY_FRACTION;
1003
+ const trimmed = raw.trim();
1004
+ if (!/^\d*\.?\d+$/.test(trimmed))
1005
+ return DEFAULT_RECYCLE_EMERGENCY_FRACTION;
1006
+ const value = Number.parseFloat(trimmed);
1007
+ return Number.isFinite(value) && value > 0 && value <= 1
1008
+ ? value
1009
+ : DEFAULT_RECYCLE_EMERGENCY_FRACTION;
1010
+ }
1011
+ /**
1012
+ * Parse the sustain-window knob (whole milliseconds, >= 0). 0 is a VALID value
1013
+ * here — it removes the window so two qualifying observations at the same uptime
1014
+ * suffice, which the deterministic tests rely on — so only a non-integer or
1015
+ * negative value falls back to the 120-second default.
1016
+ */
1017
+ export function resolveHeapRecycleSustainWindowMs(raw) {
1018
+ if (raw === undefined)
1019
+ return DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
518
1020
  const trimmed = raw.trim();
519
1021
  if (!/^\d+$/.test(trimmed))
520
- return DEFAULT_RECYCLE_SUSTAINED_SAMPLES;
1022
+ return DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
521
1023
  const value = Number.parseInt(trimmed, 10);
522
- return Number.isFinite(value) && value >= 1 ? value : DEFAULT_RECYCLE_SUSTAINED_SAMPLES;
1024
+ return Number.isFinite(value) && value >= 0 ? value : DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
523
1025
  }
524
1026
  /**
525
- * Parse the minimum-uptime-floor knob (whole milliseconds, >= 0). Unlike the
526
- * sample-interval knob, 0 is a VALID value here — it disables the floor, which
527
- * the deterministic tests rely on — so only a non-integer or negative value
528
- * falls back to the 30-minute default.
1027
+ * Parse the minimum-uptime-floor knob (whole milliseconds, >= 0). 0 is a VALID
1028
+ * value here — it disables the floor, which the deterministic tests rely on — so
1029
+ * only a non-integer or negative value falls back to the 35-minute default.
529
1030
  */
530
1031
  export function resolveHeapRecycleMinUptimeMs(raw) {
531
1032
  if (raw === undefined)
@@ -536,6 +1037,148 @@ export function resolveHeapRecycleMinUptimeMs(raw) {
536
1037
  const value = Number.parseInt(trimmed, 10);
537
1038
  return Number.isFinite(value) && value >= 0 ? value : DEFAULT_RECYCLE_MIN_UPTIME_MS;
538
1039
  }
1040
+ /**
1041
+ * Parse the checkpoint-gate knob (whole milliseconds, >= 0). 0 is a VALID value
1042
+ * — it disables the work-driven checkpoints, leaving the interval timer and the
1043
+ * loop-top backstop as the only observers (exact base behaviour) — so only a
1044
+ * non-integer or negative value falls back to the 250 ms default.
1045
+ */
1046
+ export function resolveHeapCheckpointMs(raw) {
1047
+ if (raw === undefined)
1048
+ return DEFAULT_CHECKPOINT_MS;
1049
+ const trimmed = raw.trim();
1050
+ if (!/^\d+$/.test(trimmed))
1051
+ return DEFAULT_CHECKPOINT_MS;
1052
+ const value = Number.parseInt(trimmed, 10);
1053
+ return Number.isFinite(value) && value >= 0 ? value : DEFAULT_CHECKPOINT_MS;
1054
+ }
1055
+ /**
1056
+ * Parse the pinned-fraction knob; falls back to the default (0.84) on garbage or
1057
+ * out-of-range input. Rejects the entire string rather than a numeric prefix.
1058
+ */
1059
+ export function resolveHeapPinnedFraction(raw) {
1060
+ if (raw === undefined)
1061
+ return DEFAULT_RECYCLE_PINNED_FRACTION;
1062
+ const trimmed = raw.trim();
1063
+ if (!/^\d*\.?\d+$/.test(trimmed))
1064
+ return DEFAULT_RECYCLE_PINNED_FRACTION;
1065
+ const value = Number.parseFloat(trimmed);
1066
+ return Number.isFinite(value) && value > 0 && value <= 1
1067
+ ? value
1068
+ : DEFAULT_RECYCLE_PINNED_FRACTION;
1069
+ }
1070
+ /**
1071
+ * Parse the pinned-sustain-window knob (whole milliseconds, >= 0). 0 is a VALID
1072
+ * value — it removes the window so two qualifying observations at the same uptime
1073
+ * suffice, which the deterministic tests rely on — so only a non-integer or
1074
+ * negative value falls back to the 20-second default.
1075
+ */
1076
+ export function resolveHeapPinnedSustainMs(raw) {
1077
+ if (raw === undefined)
1078
+ return DEFAULT_RECYCLE_PINNED_SUSTAIN_WINDOW_MS;
1079
+ const trimmed = raw.trim();
1080
+ if (!/^\d+$/.test(trimmed))
1081
+ return DEFAULT_RECYCLE_PINNED_SUSTAIN_WINDOW_MS;
1082
+ const value = Number.parseInt(trimmed, 10);
1083
+ return Number.isFinite(value) && value >= 0
1084
+ ? value
1085
+ : DEFAULT_RECYCLE_PINNED_SUSTAIN_WINDOW_MS;
1086
+ }
1087
+ /**
1088
+ * Parse the hard-exit-fraction knob; falls back to the default (0.955) on garbage
1089
+ * or out-of-range input. Rejects the entire string rather than a numeric prefix.
1090
+ */
1091
+ export function resolveHeapHardExitFraction(raw) {
1092
+ if (raw === undefined)
1093
+ return DEFAULT_HARD_EXIT_FRACTION;
1094
+ const trimmed = raw.trim();
1095
+ if (!/^\d*\.?\d+$/.test(trimmed))
1096
+ return DEFAULT_HARD_EXIT_FRACTION;
1097
+ const value = Number.parseFloat(trimmed);
1098
+ return Number.isFinite(value) && value > 0 && value <= 1 ? value : DEFAULT_HARD_EXIT_FRACTION;
1099
+ }
1100
+ /**
1101
+ * Parse the latch-hard-exit knob (whole milliseconds, >= 0). 0 is a VALID value
1102
+ * — it makes the first post-latch observation hard-exit — so only a non-integer
1103
+ * or negative value falls back to the 60-second default.
1104
+ */
1105
+ export function resolveHeapRecycleLatchHardExitMs(raw) {
1106
+ if (raw === undefined)
1107
+ return DEFAULT_LATCH_HARD_EXIT_MS;
1108
+ const trimmed = raw.trim();
1109
+ if (!/^\d+$/.test(trimmed))
1110
+ return DEFAULT_LATCH_HARD_EXIT_MS;
1111
+ const value = Number.parseInt(trimmed, 10);
1112
+ return Number.isFinite(value) && value >= 0 ? value : DEFAULT_LATCH_HARD_EXIT_MS;
1113
+ }
1114
+ const MAX_OLD_SPACE_FLAG = /^--max[-_]old[-_]space[-_]size$/;
1115
+ /** A whole positive integer of MB → bytes, else null (whole-string only). */
1116
+ function wholeMbToBytes(raw) {
1117
+ const trimmed = raw.trim();
1118
+ if (!/^\d+$/.test(trimmed))
1119
+ return null;
1120
+ const mb = Number.parseInt(trimmed, 10);
1121
+ return Number.isFinite(mb) && mb > 0 ? mb * 1024 * 1024 : null;
1122
+ }
1123
+ /** Strip exactly one pair of surrounding single/double quotes, if present. */
1124
+ function unquoteOptionValue(value) {
1125
+ if (value.length >= 2) {
1126
+ const first = value[0];
1127
+ const last = value[value.length - 1];
1128
+ if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
1129
+ return value.slice(1, -1);
1130
+ }
1131
+ }
1132
+ return value;
1133
+ }
1134
+ /** Last valid `--max-old-space-size` value in a token list, or null. */
1135
+ function declaredCeilingFromTokens(tokens) {
1136
+ let last = null;
1137
+ for (let i = 0; i < tokens.length; i += 1) {
1138
+ const token = tokens[i];
1139
+ const eq = token.indexOf("=");
1140
+ if (eq >= 0) {
1141
+ if (MAX_OLD_SPACE_FLAG.test(token.slice(0, eq))) {
1142
+ const bytes = wholeMbToBytes(unquoteOptionValue(token.slice(eq + 1)));
1143
+ if (bytes !== null)
1144
+ last = bytes;
1145
+ }
1146
+ }
1147
+ else if (MAX_OLD_SPACE_FLAG.test(token)) {
1148
+ const next = tokens[i + 1];
1149
+ if (next !== undefined) {
1150
+ const bytes = wholeMbToBytes(unquoteOptionValue(next));
1151
+ if (bytes !== null)
1152
+ last = bytes;
1153
+ }
1154
+ }
1155
+ }
1156
+ return last;
1157
+ }
1158
+ /**
1159
+ * Resolve the declared V8 old-space ceiling (bytes) the runner is actually
1160
+ * running under, mirroring how hq-desktop-app hands it to the npx-spawned
1161
+ * runner: a `--max-old-space-size` flag on `process.execArgv` wins over one in
1162
+ * `NODE_OPTIONS`, and within each source the LAST occurrence wins (V8's own
1163
+ * precedence). Both the `=value` and adjacent-arg forms are accepted, values may
1164
+ * carry one pair of surrounding quotes, and only a whole positive integer of MB
1165
+ * is honoured — anything else yields null, so the governor falls back to V8's
1166
+ * reported `heap_size_limit`. Never throws.
1167
+ */
1168
+ export function resolveDeclaredHeapCeilingBytes(execArgv, nodeOptions) {
1169
+ try {
1170
+ const fromArgv = declaredCeilingFromTokens(execArgv ?? []);
1171
+ if (fromArgv !== null)
1172
+ return fromArgv;
1173
+ if (nodeOptions === undefined)
1174
+ return null;
1175
+ const tokens = nodeOptions.trim().split(/\s+/).filter(Boolean);
1176
+ return declaredCeilingFromTokens(tokens);
1177
+ }
1178
+ catch {
1179
+ return null;
1180
+ }
1181
+ }
539
1182
  /**
540
1183
  * Production readers over the real runtime. Kept here so the wiring site stays
541
1184
  * a one-liner and the `v8` import lives in exactly one place.