@indigoai-us/hq-cloud 6.16.2 → 6.16.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/dist/bin/sync-runner-events.test.js +22 -12
  2. package/dist/bin/sync-runner-events.test.js.map +1 -1
  3. package/dist/bin/sync-runner-heap-telemetry.d.ts +238 -84
  4. package/dist/bin/sync-runner-heap-telemetry.d.ts.map +1 -1
  5. package/dist/bin/sync-runner-heap-telemetry.js +385 -131
  6. package/dist/bin/sync-runner-heap-telemetry.js.map +1 -1
  7. package/dist/bin/sync-runner-heap-telemetry.test.js +539 -204
  8. package/dist/bin/sync-runner-heap-telemetry.test.js.map +1 -1
  9. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  10. package/dist/bin/sync-runner-watch-loop.js +65 -15
  11. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  12. package/dist/bin/sync-runner-watch-loop.test.js +47 -9
  13. package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
  14. package/dist/bin/sync-runner.d.ts +32 -14
  15. package/dist/bin/sync-runner.d.ts.map +1 -1
  16. package/dist/bin/sync-runner.js.map +1 -1
  17. package/dist/cli/reindex.d.ts.map +1 -1
  18. package/dist/cli/reindex.js +2 -253
  19. package/dist/cli/reindex.js.map +1 -1
  20. package/dist/cli/sync.d.ts.map +1 -1
  21. package/dist/cli/sync.js +10 -0
  22. package/dist/cli/sync.js.map +1 -1
  23. package/dist/cli/sync.test.js +67 -1
  24. package/dist/cli/sync.test.js.map +1 -1
  25. package/dist/object-io.d.ts +2 -0
  26. package/dist/object-io.d.ts.map +1 -1
  27. package/dist/object-io.js +2 -0
  28. package/dist/object-io.js.map +1 -1
  29. package/dist/object-io.test.js +5 -1
  30. package/dist/object-io.test.js.map +1 -1
  31. package/dist/personal-vault.d.ts.map +1 -1
  32. package/dist/personal-vault.js +3 -1
  33. package/dist/personal-vault.js.map +1 -1
  34. package/dist/personal-vault.test.js +6 -0
  35. package/dist/personal-vault.test.js.map +1 -1
  36. package/dist/s3.d.ts +2 -0
  37. package/dist/s3.d.ts.map +1 -1
  38. package/dist/s3.js +1 -0
  39. package/dist/s3.js.map +1 -1
  40. package/dist/s3.test.js +22 -0
  41. package/dist/s3.test.js.map +1 -1
  42. package/dist/session-log-capture.d.ts +20 -0
  43. package/dist/session-log-capture.d.ts.map +1 -0
  44. package/dist/session-log-capture.js +648 -0
  45. package/dist/session-log-capture.js.map +1 -0
  46. package/dist/session-log-capture.test.d.ts +2 -0
  47. package/dist/session-log-capture.test.d.ts.map +1 -0
  48. package/dist/session-log-capture.test.js +276 -0
  49. package/dist/session-log-capture.test.js.map +1 -0
  50. package/dist/sync/state-store.js +5 -1
  51. package/dist/sync/state-store.js.map +1 -1
  52. package/dist/vault-client.d.ts +5 -0
  53. package/dist/vault-client.d.ts.map +1 -1
  54. package/dist/vault-client.js +1 -0
  55. package/dist/vault-client.js.map +1 -1
  56. package/dist/vault-client.test.js +15 -0
  57. package/dist/vault-client.test.js.map +1 -1
  58. package/package.json +1 -1
@@ -7,31 +7,45 @@
7
7
  * runner shipped no memory telemetry at all, so nothing in the abort evidence
8
8
  * could name which structure held the memory. This module closes that gap: it
9
9
  * samples the process heap on a bounded cadence, attributes live entry counts
10
- * to a fixed set of long-lived structures (the "census"), and — the moment the
11
- * used heap first crosses a configurable fraction of the ceiling — emits one
12
- * escalation record ranking those structures by growth since the session began.
10
+ * to a fixed set of long-lived structures (the "census"), and — before V8
11
+ * aborts — recycles the process so the desktop supervisor respawns a fresh one.
12
+ *
13
+ * The governor keys on the POST-GC LIVE HEAP, not the instantaneous
14
+ * `used_heap_size`. Between major collections `used_heap_size` is live-set plus
15
+ * not-yet-collected garbage, so it sawtooths by a large fraction of the ceiling
16
+ * from minute to minute; a rule that watches it can sit at a sub-threshold
17
+ * sample forever while the true live set climbs to the abort point (the
18
+ * 2026-09-01 regression: a full production day produced ZERO recycle verdicts
19
+ * against 1462 samples while the runner kept aborting). The monotonic quantity
20
+ * is the live set that survives a major GC — exactly what V8 uses for its own
21
+ * OOM verdict — so the primary signal is read on each observed MAJOR GC. A
22
+ * single instantaneous backstop remains for the case where the live signal is
23
+ * unavailable and the heap is already minutes from aborting.
13
24
  *
14
25
  * The whole module is numeric-only and allocation-light by construction:
15
26
  *
16
27
  * - Every emitted value is a number, and every census site is a FIXED
17
- * identifier token, never a path/filename/company/message. Records
18
- * therefore pass the existing telemetry sanitizer unchanged.
28
+ * identifier token, never a path/filename/company/message. Property keys are
29
+ * additionally shaped to survive the vault's ingest contract: they contain
30
+ * none of its denied key substrings and end in `_count`/`_bytes`/`_ms`, so
31
+ * the census attribution is actually stored rather than silently dropped.
19
32
  * - The controller retains only the FIRST sample (the growth baseline) and
20
33
  * the LATEST sample. It never accumulates history — a memory diagnostic
21
34
  * that leaks would regress the very defect it measures.
22
- * - All readers, timers, and emit callbacks are injected, so the module is
23
- * driven deterministically by tests with no real allocation, no real heap
24
- * growth, no wall-clock timing, and no network.
35
+ * - All readers, timers, the GC observer, and emit callbacks are injected, so
36
+ * the module is driven deterministically by tests with no real allocation,
37
+ * no real heap growth, no wall-clock timing, and no network.
25
38
  *
26
39
  * Diagnostics must never affect sync: a throwing reader, a faulty census
27
- * entry, or a rejecting emit callback degrades to silence and never disturbs a
28
- * watch cycle.
40
+ * entry, an unavailable GC entry API, or a rejecting emit callback degrades to
41
+ * silence and never disturbs a watch cycle.
29
42
  */
43
+ import * as perfHooks from "node:perf_hooks";
30
44
  /**
31
45
  * Census tokens are fixed identifiers, never user-derived strings. The pattern
32
46
  * is deliberately stricter than the telemetry sanitizer's SAFE_KEY so a token
33
- * can always be prefixed (`census_`, `count_`, `growth_`) and still fit inside
34
- * the 64-character key budget.
47
+ * can always be prefixed (`census_`) and suffixed (`_count`, `_growth_count`)
48
+ * and still fit inside the 64-character key budget.
35
49
  */
36
50
  const CENSUS_TOKEN = /^[A-Za-z][A-Za-z0-9_]{0,48}$/;
37
51
  /**
@@ -93,11 +107,18 @@ export function sampleRunnerHeap(deps) {
93
107
  function finiteNumber(value) {
94
108
  return typeof value === "number" && Number.isFinite(value) ? value : 0;
95
109
  }
110
+ /** Parts-per-thousand, the integer spelling every fraction is emitted as. */
111
+ function toPermille(fraction) {
112
+ return Math.round(fraction * 1000);
113
+ }
96
114
  /**
97
115
  * Flatten a sample into numeric telemetry properties. Raw heap figures carry a
98
- * `heap_`/`rss_`/`external_` prefix; each census site becomes `census_<token>`.
99
- * Every value is a number, so the record survives the telemetry sanitizer with
100
- * no weakening of its path/name/message rejection.
116
+ * `heap_`/`rss_`/`external_` prefix; each census site becomes
117
+ * `census_<token>_count`. Every value is a number AND every key ends in
118
+ * `_count`/`_bytes`/`_ms` and contains none of the vault ingest contract's
119
+ * denied substrings, so the record is both accepted by the local sanitizer and
120
+ * actually STORED by ingest (unlike the pre-fix `census_<token>` spelling,
121
+ * which ingest silently dropped, so the attribution never reached the store).
101
122
  */
102
123
  export function heapSampleCloudProperties(sample) {
103
124
  const props = {
@@ -109,18 +130,18 @@ export function heapSampleCloudProperties(sample) {
109
130
  external_bytes: sample.externalBytes,
110
131
  array_buffers_bytes: sample.arrayBuffersBytes,
111
132
  uptime_ms: sample.uptimeMs,
112
- census_total: sample.censusTotal,
133
+ census_total_count: sample.censusTotal,
113
134
  };
114
135
  for (const { site, count } of sample.census) {
115
- props[`census_${site}`] = count;
136
+ props[`census_${site}_count`] = count;
116
137
  }
117
138
  return props;
118
139
  }
119
140
  /** Flatten a pressure record into numeric-only telemetry properties. */
120
141
  export function heapPressureCloudProperties(record) {
121
142
  const props = {
122
- pressure_used_fraction_milli: Math.round(record.usedFraction * 1000),
123
- pressure_threshold_milli: Math.round(record.thresholdFraction * 1000),
143
+ pressure_used_permille_count: toPermille(record.usedFraction),
144
+ pressure_threshold_permille_count: toPermille(record.thresholdFraction),
124
145
  heap_used_bytes: record.usedHeapBytes,
125
146
  heap_limit_bytes: record.heapLimitBytes,
126
147
  rss_bytes: record.rssBytes,
@@ -128,8 +149,8 @@ export function heapPressureCloudProperties(record) {
128
149
  uptime_ms: record.uptimeMs,
129
150
  };
130
151
  for (const { site, count, growth } of record.topSites) {
131
- props[`count_${site}`] = count;
132
- props[`growth_${site}`] = growth;
152
+ props[`${site}_count`] = count;
153
+ props[`${site}_growth_count`] = growth;
133
154
  }
134
155
  return props;
135
156
  }
@@ -140,26 +161,35 @@ export function heapPressureStderrLine(record) {
140
161
  .join(",");
141
162
  return ("hq-sync-runner: heap-pressure " +
142
163
  `used=${record.usedHeapBytes} limit=${record.heapLimitBytes} ` +
143
- `frac_milli=${Math.round(record.usedFraction * 1000)} rss=${record.rssBytes} ` +
164
+ `permille=${toPermille(record.usedFraction)} rss=${record.rssBytes} ` +
144
165
  `uptime_ms=${record.uptimeMs} top=${top}`);
145
166
  }
146
167
  /** Flatten a recycle record into numeric-only telemetry properties. */
147
168
  export function heapRecycleCloudProperties(record) {
148
169
  const props = {
149
- recycle_used_fraction_milli: Math.round(record.usedFraction * 1000),
150
- recycle_fraction_milli: Math.round(record.recycleFraction * 1000),
151
- recycle_sustained_samples: record.sustainedSamples,
152
- recycle_observed_sustained_samples: record.observedSustainedSamples,
170
+ recycle_used_permille_count: toPermille(record.usedFraction),
171
+ recycle_live_permille_count: toPermille(record.liveFraction),
172
+ recycle_live_floor_permille_count: toPermille(record.recycleLiveFraction),
173
+ recycle_emergency_floor_permille_count: toPermille(record.emergencyFraction),
174
+ recycle_emergency_trigger_count: record.trigger === "emergency" ? 1 : 0,
175
+ recycle_sustain_window_ms: record.sustainWindowMs,
176
+ recycle_observed_sustain_ms: record.observedSustainMs,
153
177
  recycle_min_uptime_ms: record.minUptimeMs,
154
178
  heap_used_bytes: record.usedHeapBytes,
179
+ heap_live_bytes: record.liveHeapBytes,
155
180
  heap_limit_bytes: record.heapLimitBytes,
156
181
  rss_bytes: record.rssBytes,
157
182
  external_bytes: record.externalBytes,
183
+ array_buffers_bytes: record.arrayBuffersBytes,
184
+ rss_growth_bytes: record.rssGrowthBytes,
185
+ external_growth_bytes: record.externalGrowthBytes,
186
+ array_buffers_growth_bytes: record.arrayBuffersGrowthBytes,
158
187
  uptime_ms: record.uptimeMs,
188
+ census_total_count: record.censusTotal,
159
189
  };
160
190
  for (const { site, count, growth } of record.topSites) {
161
- props[`count_${site}`] = count;
162
- props[`growth_${site}`] = growth;
191
+ props[`${site}_count`] = count;
192
+ props[`${site}_growth_count`] = growth;
163
193
  }
164
194
  return props;
165
195
  }
@@ -177,20 +207,28 @@ export function heapRecycleStderrLine(record) {
177
207
  .map((entry) => `${entry.site}:${entry.growth}`)
178
208
  .join(",");
179
209
  return ("hq-sync-runner: heap-recycle " +
180
- `used=${record.usedHeapBytes} limit=${record.heapLimitBytes} ` +
181
- `frac_milli=${Math.round(record.usedFraction * 1000)} rss=${record.rssBytes} ` +
182
- `uptime_ms=${record.uptimeMs} sustained=${record.observedSustainedSamples} top=${top}`);
210
+ `trigger=${record.trigger} ` +
211
+ `used=${record.usedHeapBytes} live=${record.liveHeapBytes} limit=${record.heapLimitBytes} ` +
212
+ `live_permille=${toPermille(record.liveFraction)} rss=${record.rssBytes} ` +
213
+ `uptime_ms=${record.uptimeMs} sustain_ms=${record.observedSustainMs} top=${top}`);
183
214
  }
184
215
  const DEFAULT_PRESSURE_THRESHOLD = 0.75;
185
216
  const DEFAULT_TOP_SITE_COUNT = 8;
186
- const DEFAULT_RECYCLE_FRACTION = 0.78;
187
- const DEFAULT_RECYCLE_SUSTAINED_SAMPLES = 2;
188
- const DEFAULT_RECYCLE_MIN_UPTIME_MS = 30 * 60_000;
217
+ const DEFAULT_RECYCLE_LIVE_FRACTION = 0.6;
218
+ const DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS = 120_000;
219
+ const DEFAULT_RECYCLE_EMERGENCY_FRACTION = 0.9;
220
+ const DEFAULT_RECYCLE_MIN_UPTIME_MS = 35 * 60_000;
189
221
  /**
190
- * Owns the sampling state machine: it keeps the baseline and latest sample, and
191
- * decides — once per rising crossing of the threshold — when to escalate. The
192
- * threshold is derived from the injected `heap_size_limit`, so it self-calibrates
193
- * to whatever `--max-old-space-size` the desktop supplied.
222
+ * Owns the sampling state machine. It keeps the baseline and latest sample, the
223
+ * pressure escalation, and the two recycle paths:
224
+ * - {@link observeMajorGc}, the PRIMARY path, driven by the GC observer: each
225
+ * (throttled) call reads the post-GC live heap and tracks the sustain
226
+ * window that a live recycle requires.
227
+ * - {@link evaluate} and the emergency arm of {@link sample}, the instantaneous
228
+ * BACKSTOP: they recycle on a single sample at/above the emergency fraction
229
+ * even if the GC signal never arrives or the sampler timer is starved.
230
+ * Every fraction is derived from the injected `heap_size_limit`, so the governor
231
+ * self-calibrates to whatever `--max-old-space-size` the desktop supplied.
194
232
  */
195
233
  export class RunnerHeapTelemetry {
196
234
  readers;
@@ -199,14 +237,17 @@ export class RunnerHeapTelemetry {
199
237
  threshold;
200
238
  topSiteCount;
201
239
  recycleEnabled;
202
- recycleFraction;
203
- recycleSustainedSamples;
240
+ recycleLiveFraction;
241
+ recycleSustainWindowMs;
242
+ recycleEmergencyFraction;
204
243
  recycleMinUptimeMs;
205
244
  baseline = null;
206
245
  latest = null;
207
246
  pressureSignalled = false;
208
- recycleSustainedCount = 0;
209
247
  recycleSignalled = false;
248
+ /** Uptime (ms) of the first live observation at/above the floor in the
249
+ * current consecutive run; null when the last live reading dipped below it. */
250
+ liveAnchorUptimeMs = null;
210
251
  constructor(options) {
211
252
  this.readers = options.readers;
212
253
  this.census = options.census;
@@ -219,16 +260,11 @@ export class RunnerHeapTelemetry {
219
260
  const count = options.topSiteCount ?? DEFAULT_TOP_SITE_COUNT;
220
261
  this.topSiteCount = Number.isFinite(count) && count > 0 ? Math.trunc(count) : DEFAULT_TOP_SITE_COUNT;
221
262
  this.recycleEnabled = options.recycleEnabled ?? true;
222
- const recycleFraction = options.recycleFraction ?? DEFAULT_RECYCLE_FRACTION;
223
- this.recycleFraction =
224
- Number.isFinite(recycleFraction) && recycleFraction > 0 && recycleFraction <= 1
225
- ? recycleFraction
226
- : DEFAULT_RECYCLE_FRACTION;
227
- const sustained = options.recycleSustainedSamples ?? DEFAULT_RECYCLE_SUSTAINED_SAMPLES;
228
- this.recycleSustainedSamples =
229
- Number.isFinite(sustained) && sustained >= 1
230
- ? Math.trunc(sustained)
231
- : DEFAULT_RECYCLE_SUSTAINED_SAMPLES;
263
+ this.recycleLiveFraction = fractionOrDefault(options.recycleLiveFraction, DEFAULT_RECYCLE_LIVE_FRACTION);
264
+ const window = options.recycleSustainWindowMs ?? DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
265
+ this.recycleSustainWindowMs =
266
+ Number.isFinite(window) && window >= 0 ? Math.trunc(window) : DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
267
+ this.recycleEmergencyFraction = fractionOrDefault(options.recycleEmergencyFraction, DEFAULT_RECYCLE_EMERGENCY_FRACTION);
232
268
  const minUptime = options.recycleMinUptimeMs ?? DEFAULT_RECYCLE_MIN_UPTIME_MS;
233
269
  this.recycleMinUptimeMs =
234
270
  Number.isFinite(minUptime) && minUptime >= 0
@@ -237,78 +273,151 @@ export class RunnerHeapTelemetry {
237
273
  }
238
274
  /**
239
275
  * Capture the growth baseline when sampling starts, before the first interval,
240
- * WITHOUT emitting or arming the pressure signal. Without this, a runner that
241
- * is already above the threshold at its first scheduled sample would make that
242
- * same high-water sample its own baseline, so the single crossing record it
243
- * emits would report zero growth for every site and lose the attribution.
276
+ * WITHOUT emitting or arming any signal. Without this, a runner that is already
277
+ * above the threshold at its first scheduled sample would make that same high
278
+ * sample its own baseline, so a crossing record would report zero growth for
279
+ * every site and lose the attribution.
244
280
  */
245
281
  primeBaseline() {
246
- const sample = sampleRunnerHeap({
247
- readers: this.readers,
248
- census: this.census,
249
- sessionStartedAtMs: this.sessionStartedAtMs,
250
- });
251
- this.baseline ??= sample;
252
- this.latest = sample;
253
- return sample;
282
+ return this.buildSample();
254
283
  }
284
+ /**
285
+ * The interval sampler tick. Emits a sample and (at most once per rising
286
+ * crossing) a pressure record. Its recycle output is the EMERGENCY backstop
287
+ * only — the instantaneous sawtooth is deliberately NOT the primary verdict.
288
+ */
255
289
  sample() {
256
- const sample = sampleRunnerHeap({
257
- readers: this.readers,
258
- census: this.census,
259
- sessionStartedAtMs: this.sessionStartedAtMs,
260
- });
261
- // Retain ONLY the baseline (first sample) and the latest. Growth is the
262
- // difference between the two; no per-sample history is ever kept.
263
- this.baseline ??= sample;
264
- this.latest = sample;
290
+ const sample = this.buildSample();
265
291
  return {
266
292
  sample,
267
293
  pressure: this.detectPressure(sample),
268
- recycle: this.detectRecycle(sample),
294
+ recycle: this.detectEmergency(sample),
269
295
  };
270
296
  }
271
297
  /**
272
- * The governor decision. Returns a record at most once per session, and only
273
- * when the heap has stayed at or above the recycle fraction for a sustained
274
- * run of consecutive samples AND the process has outlived the uptime floor.
275
- * The sustained counter is maintained on every sample so the streak is real
276
- * consecutive pressure, but the verdict itself is gated on the floor, so a
277
- * recycle can never fire early however high the heap climbs.
298
+ * The PRIMARY governor path, driven by an observed MAJOR GC. The caller
299
+ * ({@link nodeMajorGcObserver}) only invokes this for a GC whose reading it
300
+ * could capture PROMPTLY — a callback delayed by a blocked event loop is
301
+ * skipped upstream, because `getHeapStatistics()` read late would be an
302
+ * instantaneous sawtooth value (live set plus everything allocated since the
303
+ * GC), not the post-GC live floor.
304
+ *
305
+ * The anchor is maintained on EVERY observed major GC via a cheap
306
+ * `getHeapStatistics()` read (no throttle), so a post-GC reading that dips
307
+ * below the floor ALWAYS invalidates the sustain run — a throttle that skipped
308
+ * such a reading would let a stale anchor fake a continuous run. The fuller
309
+ * sample (census snapshot + memory deltas) is taken only at the verdict, so a
310
+ * GC storm never costs more than the O(1) statistics read here.
278
311
  */
279
- detectRecycle(sample) {
280
- if (!this.recycleEnabled)
312
+ observeMajorGc() {
313
+ if (!this.recycleEnabled || this.recycleSignalled)
281
314
  return null;
282
- if (sample.heapLimitBytes <= 0)
315
+ const stats = this.readers.getHeapStatistics();
316
+ const heapLimitBytes = finiteNumber(stats.heap_size_limit);
317
+ if (heapLimitBytes <= 0)
318
+ return null;
319
+ const liveFraction = finiteNumber(stats.used_heap_size) / heapLimitBytes;
320
+ const uptimeMs = Math.max(0, Math.trunc(this.readers.monotonicNowMs() - this.sessionStartedAtMs));
321
+ // A genuine post-GC reading already abort-adjacent: recycle now, bypassing
322
+ // both the sustain window and the uptime floor.
323
+ if (liveFraction >= this.recycleEmergencyFraction) {
324
+ return this.produceRecycleFromCurrent("emergency", 0);
325
+ }
326
+ if (liveFraction < this.recycleLiveFraction) {
327
+ // The live floor dipped: reset the anchor so the window only ever measures
328
+ // a genuinely continuous run at/above the floor.
329
+ this.liveAnchorUptimeMs = null;
283
330
  return null;
284
- const usedFraction = sample.usedHeapBytes / sample.heapLimitBytes;
285
- if (usedFraction >= this.recycleFraction) {
286
- this.recycleSustainedCount += 1;
287
331
  }
288
- else {
289
- // A dip below the fraction breaks the streak: a single spike can never
290
- // accumulate the sustained run the verdict requires.
291
- this.recycleSustainedCount = 0;
332
+ if (this.liveAnchorUptimeMs === null) {
333
+ // First qualifying observation. A second, at least a window later, confirms.
334
+ this.liveAnchorUptimeMs = uptimeMs;
292
335
  return null;
293
336
  }
294
- if (this.recycleSignalled)
337
+ const observedSustainMs = Math.max(0, uptimeMs - this.liveAnchorUptimeMs);
338
+ if (observedSustainMs < this.recycleSustainWindowMs)
295
339
  return null;
296
- if (sample.uptimeMs < this.recycleMinUptimeMs)
340
+ if (uptimeMs < this.recycleMinUptimeMs)
297
341
  return null;
298
- if (this.recycleSustainedCount < this.recycleSustainedSamples)
342
+ return this.produceRecycleFromCurrent("live", observedSustainMs);
343
+ }
344
+ /**
345
+ * The loop-top BACKSTOP against timer starvation (the observed 11-minute
346
+ * sampler hole). Reads the instantaneous heap and runs the emergency check
347
+ * only — an instantaneous reading can never confirm the post-GC live floor,
348
+ * so it must not feed the primary path's sustain window. Cheap: an O(1)
349
+ * statistics read, sampling the census only if it actually fires.
350
+ */
351
+ evaluate() {
352
+ if (!this.recycleEnabled || this.recycleSignalled)
299
353
  return null;
354
+ const stats = this.readers.getHeapStatistics();
355
+ const heapLimitBytes = finiteNumber(stats.heap_size_limit);
356
+ if (heapLimitBytes <= 0)
357
+ return null;
358
+ const usedFraction = finiteNumber(stats.used_heap_size) / heapLimitBytes;
359
+ if (usedFraction < this.recycleEmergencyFraction)
360
+ return null;
361
+ return this.produceRecycleFromCurrent("emergency", 0);
362
+ }
363
+ buildSample() {
364
+ const sample = sampleRunnerHeap({
365
+ readers: this.readers,
366
+ census: this.census,
367
+ sessionStartedAtMs: this.sessionStartedAtMs,
368
+ });
369
+ // Retain ONLY the baseline (first sample) and the latest. Growth is the
370
+ // difference between the two; no per-sample history is ever kept.
371
+ this.baseline ??= sample;
372
+ this.latest = sample;
373
+ return sample;
374
+ }
375
+ /** The instantaneous emergency backstop: a single sample at/above the
376
+ * emergency fraction, bypassing the sustain window and the uptime floor. */
377
+ detectEmergency(sample) {
378
+ if (!this.recycleEnabled || this.recycleSignalled)
379
+ return null;
380
+ if (sample.heapLimitBytes <= 0)
381
+ return null;
382
+ const usedFraction = sample.usedHeapBytes / sample.heapLimitBytes;
383
+ if (usedFraction < this.recycleEmergencyFraction)
384
+ return null;
385
+ return this.produceRecycle(sample, "emergency", usedFraction, 0);
386
+ }
387
+ /**
388
+ * Produce a verdict from a freshly-taken sample. Called only once the cheap
389
+ * per-GC (or loop-top) path has decided a recycle is due, so the census
390
+ * snapshot and memory deltas are computed exactly once per session.
391
+ */
392
+ produceRecycleFromCurrent(trigger, observedSustainMs) {
393
+ const sample = this.buildSample();
394
+ const liveFraction = sample.heapLimitBytes > 0 ? sample.usedHeapBytes / sample.heapLimitBytes : 0;
395
+ return this.produceRecycle(sample, trigger, liveFraction, observedSustainMs);
396
+ }
397
+ produceRecycle(sample, trigger, liveFraction, observedSustainMs) {
300
398
  this.recycleSignalled = true;
399
+ const usedFraction = sample.heapLimitBytes > 0 ? sample.usedHeapBytes / sample.heapLimitBytes : 0;
400
+ const baseline = this.baseline;
301
401
  return {
402
+ trigger,
302
403
  usedHeapBytes: sample.usedHeapBytes,
404
+ liveHeapBytes: sample.usedHeapBytes,
303
405
  heapLimitBytes: sample.heapLimitBytes,
304
406
  usedFraction,
305
- recycleFraction: this.recycleFraction,
306
- sustainedSamples: this.recycleSustainedSamples,
307
- observedSustainedSamples: this.recycleSustainedCount,
407
+ liveFraction,
408
+ recycleLiveFraction: this.recycleLiveFraction,
409
+ emergencyFraction: this.recycleEmergencyFraction,
410
+ sustainWindowMs: this.recycleSustainWindowMs,
411
+ observedSustainMs,
308
412
  minUptimeMs: this.recycleMinUptimeMs,
413
+ uptimeMs: sample.uptimeMs,
309
414
  rssBytes: sample.rssBytes,
310
415
  externalBytes: sample.externalBytes,
311
- uptimeMs: sample.uptimeMs,
416
+ arrayBuffersBytes: sample.arrayBuffersBytes,
417
+ rssGrowthBytes: sample.rssBytes - (baseline?.rssBytes ?? sample.rssBytes),
418
+ externalGrowthBytes: sample.externalBytes - (baseline?.externalBytes ?? sample.externalBytes),
419
+ arrayBuffersGrowthBytes: sample.arrayBuffersBytes - (baseline?.arrayBuffersBytes ?? sample.arrayBuffersBytes),
420
+ censusTotal: sample.censusTotal,
312
421
  topSites: this.topSitesByGrowth(sample),
313
422
  };
314
423
  }
@@ -355,18 +464,104 @@ export class RunnerHeapTelemetry {
355
464
  return this.latest;
356
465
  }
357
466
  }
467
+ function fractionOrDefault(value, fallback) {
468
+ return typeof value === "number" && Number.isFinite(value) && value > 0 && value <= 1
469
+ ? value
470
+ : fallback;
471
+ }
472
+ /**
473
+ * The maximum age (ms) of a major-GC PerformanceEntry, at the moment its
474
+ * observer callback runs, for the post-GC heap read to be trustworthy. GC
475
+ * entries are delivered ASYNCHRONOUSLY, so when the event loop was blocked the
476
+ * callback fires long after the collection and a `getHeapStatistics()` read then
477
+ * reflects the live set PLUS everything allocated since — an instantaneous
478
+ * sawtooth value, not the post-GC live floor. A reading older than this is
479
+ * skipped rather than fed to the sustain window, so only promptly-captured
480
+ * (genuinely post-GC) readings can arm a live recycle.
481
+ */
482
+ export const MAX_FRESH_GC_READING_MS = 250;
483
+ /**
484
+ * Whether a major-GC reading taken at `nowMs` is fresh enough to treat as a
485
+ * post-GC live-heap snapshot, given the entry's end time (`entry.startTime +
486
+ * entry.duration`). Exported for deterministic testing of the staleness gate.
487
+ */
488
+ export function isFreshGcReading(gcEndMs, nowMs, maxAgeMs = MAX_FRESH_GC_READING_MS) {
489
+ if (!Number.isFinite(gcEndMs) || !Number.isFinite(nowMs))
490
+ return false;
491
+ return nowMs - gcEndMs <= maxAgeMs;
492
+ }
493
+ /**
494
+ * Production major-GC observer over `perf_hooks`. Everything is wrapped so a
495
+ * runtime without the GC PerformanceEntry API, or one whose entries omit
496
+ * `detail.kind`, degrades to a silent no-op rather than throwing into the
497
+ * runner — in that degraded case the instantaneous emergency backstop and the
498
+ * pressure diagnostics remain active, so the governor is never worse than base.
499
+ *
500
+ * It only invokes the callback for a major GC whose reading it can capture
501
+ * PROMPTLY ({@link isFreshGcReading}): a callback delayed by a blocked event
502
+ * loop would read an inflated instantaneous heap rather than the post-GC live
503
+ * floor, so it is skipped.
504
+ */
505
+ export function nodeMajorGcObserver() {
506
+ let observer = null;
507
+ return {
508
+ start(onMajorGc) {
509
+ try {
510
+ const PerformanceObserverCtor = perfHooks.PerformanceObserver;
511
+ const majorKind = perfHooks.constants?.NODE_PERFORMANCE_GC_MAJOR;
512
+ if (typeof PerformanceObserverCtor !== "function" || typeof majorKind !== "number") {
513
+ // No usable major-GC signal on this runtime: stay silent.
514
+ observer = null;
515
+ return;
516
+ }
517
+ observer = new PerformanceObserverCtor((list) => {
518
+ try {
519
+ const nowMs = perfHooks.performance.now();
520
+ for (const entry of list.getEntries()) {
521
+ const kind = entry.detail?.kind;
522
+ if (typeof kind !== "number" || kind !== majorKind)
523
+ continue;
524
+ const gcEndMs = entry.startTime + (entry.duration ?? 0);
525
+ if (!isFreshGcReading(gcEndMs, nowMs))
526
+ continue;
527
+ onMajorGc();
528
+ }
529
+ }
530
+ catch {
531
+ /* a mis-shaped entry or a throwing callback must never surface */
532
+ }
533
+ });
534
+ observer.observe({ entryTypes: ["gc"] });
535
+ }
536
+ catch {
537
+ observer = null;
538
+ }
539
+ },
540
+ stop() {
541
+ try {
542
+ observer?.disconnect();
543
+ }
544
+ catch {
545
+ /* disconnecting is best-effort during shutdown */
546
+ }
547
+ observer = null;
548
+ },
549
+ };
550
+ }
358
551
  const DEFAULT_SAMPLE_INTERVAL_MS = 60_000;
359
552
  /**
360
- * Start periodic sampling. The timer is `unref`'d so it can never hold the
361
- * process open, and `stop()` clears it on every shutdown path. Each tick — and
362
- * every emit callback it drives — is wrapped so a fault degrades to silence and
363
- * never affects the watch cycle.
553
+ * Start periodic sampling and, when a GC observer is supplied, the primary
554
+ * live-heap path. The interval timer is `unref`'d so it can never hold the
555
+ * process open, and `stop()` clears it and disconnects the GC observer on every
556
+ * shutdown path. Each tick, GC callback, and emit — and the loop-top
557
+ * `evaluate()` — is wrapped so a fault degrades to silence and never affects the
558
+ * watch cycle.
364
559
  */
365
560
  export function startRunnerHeapSampling(options) {
366
561
  const telemetry = new RunnerHeapTelemetry(options);
367
562
  // Capture the baseline immediately, before the first interval, so the first
368
- // pressure crossing attributes growth against session start rather than
369
- // against a first sample that may already sit at the ceiling.
563
+ // pressure crossing (and every growth delta) attributes against session start
564
+ // rather than against a first sample that may already sit at the ceiling.
370
565
  try {
371
566
  telemetry.primeBaseline();
372
567
  }
@@ -378,6 +573,16 @@ export function startRunnerHeapSampling(options) {
378
573
  ? Math.trunc(options.intervalMs)
379
574
  : DEFAULT_SAMPLE_INTERVAL_MS;
380
575
  let stopped = false;
576
+ // One recycle verdict per session, from whichever path reaches it first. The
577
+ // telemetry latches internally, so this callback fires at most once.
578
+ const fireRecycle = (record) => {
579
+ try {
580
+ options.onRecycle?.(record);
581
+ }
582
+ catch (err) {
583
+ warn(options.log, err);
584
+ }
585
+ };
381
586
  const tick = () => {
382
587
  if (stopped)
383
588
  return;
@@ -397,23 +602,50 @@ export function startRunnerHeapSampling(options) {
397
602
  warn(options.log, err);
398
603
  }
399
604
  }
400
- if (recycle) {
605
+ if (recycle)
606
+ fireRecycle(recycle);
607
+ }
608
+ catch (err) {
609
+ warn(options.log, err);
610
+ }
611
+ };
612
+ const handle = scheduler.setInterval(tick, intervalMs);
613
+ handle.unref?.();
614
+ // The primary path: a major GC drives one (throttled) post-GC live evaluation.
615
+ const gcObserver = options.gcObserver;
616
+ if (gcObserver) {
617
+ try {
618
+ gcObserver.start(() => {
619
+ if (stopped)
620
+ return;
401
621
  try {
402
- options.onRecycle?.(recycle);
622
+ const recycle = telemetry.observeMajorGc();
623
+ if (recycle)
624
+ fireRecycle(recycle);
403
625
  }
404
626
  catch (err) {
405
627
  warn(options.log, err);
406
628
  }
407
- }
629
+ });
408
630
  }
409
631
  catch (err) {
410
632
  warn(options.log, err);
411
633
  }
412
- };
413
- const handle = scheduler.setInterval(tick, intervalMs);
414
- handle.unref?.();
634
+ }
415
635
  return {
416
636
  telemetry,
637
+ evaluate: () => {
638
+ if (stopped)
639
+ return;
640
+ try {
641
+ const recycle = telemetry.evaluate();
642
+ if (recycle)
643
+ fireRecycle(recycle);
644
+ }
645
+ catch (err) {
646
+ warn(options.log, err);
647
+ }
648
+ },
417
649
  stop: () => {
418
650
  if (stopped)
419
651
  return;
@@ -424,6 +656,12 @@ export function startRunnerHeapSampling(options) {
424
656
  catch {
425
657
  /* clearing is best-effort during shutdown */
426
658
  }
659
+ try {
660
+ gcObserver?.stop();
661
+ }
662
+ catch {
663
+ /* disconnecting is best-effort during shutdown */
664
+ }
427
665
  },
428
666
  };
429
667
  }
@@ -443,13 +681,15 @@ function warn(log, err) {
443
681
  export const HEAP_SAMPLE_INTERVAL_ENV = "HQ_SYNC_RUNNER_HEAP_SAMPLE_MS";
444
682
  export const HEAP_PRESSURE_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_PRESSURE_FRACTION";
445
683
  export const HEAP_RECYCLE_ENABLED_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_ENABLED";
446
- export const HEAP_RECYCLE_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_FRACTION";
447
- export const HEAP_RECYCLE_SUSTAINED_SAMPLES_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_SUSTAINED_SAMPLES";
684
+ export const HEAP_RECYCLE_LIVE_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_LIVE_FRACTION";
685
+ export const HEAP_RECYCLE_SUSTAIN_WINDOW_MS_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_SUSTAIN_WINDOW_MS";
686
+ export const HEAP_RECYCLE_EMERGENCY_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_EMERGENCY_FRACTION";
448
687
  export const HEAP_RECYCLE_MIN_UPTIME_MS_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_MIN_UPTIME_MS";
449
688
  export const DEFAULT_HEAP_SAMPLE_INTERVAL_MS = DEFAULT_SAMPLE_INTERVAL_MS;
450
689
  export const DEFAULT_HEAP_PRESSURE_FRACTION = DEFAULT_PRESSURE_THRESHOLD;
451
- export const DEFAULT_HEAP_RECYCLE_FRACTION = DEFAULT_RECYCLE_FRACTION;
452
- export const DEFAULT_HEAP_RECYCLE_SUSTAINED_SAMPLES = DEFAULT_RECYCLE_SUSTAINED_SAMPLES;
690
+ export const DEFAULT_HEAP_RECYCLE_LIVE_FRACTION = DEFAULT_RECYCLE_LIVE_FRACTION;
691
+ export const DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS = DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
692
+ export const DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION = DEFAULT_RECYCLE_EMERGENCY_FRACTION;
453
693
  export const DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS = DEFAULT_RECYCLE_MIN_UPTIME_MS;
454
694
  /**
455
695
  * Parse the sampling-interval env knob (whole milliseconds). Rejects the ENTIRE
@@ -493,39 +733,53 @@ export function resolveHeapRecycleEnabled(raw) {
493
733
  return !(trimmed === "0" || trimmed === "false" || trimmed === "no" || trimmed === "off");
494
734
  }
495
735
  /**
496
- * Parse the recycle-fraction knob; falls back to the default (0.78) on garbage
497
- * or out-of-range input. Rejects the entire string (e.g. "0.8x" falls back)
498
- * rather than accepting a numeric prefix.
736
+ * Parse the primary live-fraction knob; falls back to the default (0.60) on
737
+ * garbage or out-of-range input. Rejects the entire string (e.g. "0.6x" falls
738
+ * back) rather than accepting a numeric prefix.
739
+ */
740
+ export function resolveHeapRecycleLiveFraction(raw) {
741
+ if (raw === undefined)
742
+ return DEFAULT_RECYCLE_LIVE_FRACTION;
743
+ const trimmed = raw.trim();
744
+ if (!/^\d*\.?\d+$/.test(trimmed))
745
+ return DEFAULT_RECYCLE_LIVE_FRACTION;
746
+ const value = Number.parseFloat(trimmed);
747
+ return Number.isFinite(value) && value > 0 && value <= 1 ? value : DEFAULT_RECYCLE_LIVE_FRACTION;
748
+ }
749
+ /**
750
+ * Parse the emergency-fraction knob; falls back to the default (0.90) on garbage
751
+ * or out-of-range input. Rejects the entire string rather than a numeric prefix.
499
752
  */
500
- export function resolveHeapRecycleFraction(raw) {
753
+ export function resolveHeapRecycleEmergencyFraction(raw) {
501
754
  if (raw === undefined)
502
- return DEFAULT_RECYCLE_FRACTION;
755
+ return DEFAULT_RECYCLE_EMERGENCY_FRACTION;
503
756
  const trimmed = raw.trim();
504
757
  if (!/^\d*\.?\d+$/.test(trimmed))
505
- return DEFAULT_RECYCLE_FRACTION;
758
+ return DEFAULT_RECYCLE_EMERGENCY_FRACTION;
506
759
  const value = Number.parseFloat(trimmed);
507
- return Number.isFinite(value) && value > 0 && value <= 1 ? value : DEFAULT_RECYCLE_FRACTION;
760
+ return Number.isFinite(value) && value > 0 && value <= 1
761
+ ? value
762
+ : DEFAULT_RECYCLE_EMERGENCY_FRACTION;
508
763
  }
509
764
  /**
510
- * Parse the sustained-samples knob (a whole count >= 1). Falls back to the
511
- * default (2) on garbage, 0, or a non-integer — a degenerate 0 would let a
512
- * single spike recycle, exactly what the sustained requirement exists to
513
- * prevent, so it is refused rather than honoured.
765
+ * Parse the sustain-window knob (whole milliseconds, >= 0). 0 is a VALID value
766
+ * here — it removes the window so two qualifying observations at the same uptime
767
+ * suffice, which the deterministic tests rely on — so only a non-integer or
768
+ * negative value falls back to the 120-second default.
514
769
  */
515
- export function resolveHeapRecycleSustainedSamples(raw) {
770
+ export function resolveHeapRecycleSustainWindowMs(raw) {
516
771
  if (raw === undefined)
517
- return DEFAULT_RECYCLE_SUSTAINED_SAMPLES;
772
+ return DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
518
773
  const trimmed = raw.trim();
519
774
  if (!/^\d+$/.test(trimmed))
520
- return DEFAULT_RECYCLE_SUSTAINED_SAMPLES;
775
+ return DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
521
776
  const value = Number.parseInt(trimmed, 10);
522
- return Number.isFinite(value) && value >= 1 ? value : DEFAULT_RECYCLE_SUSTAINED_SAMPLES;
777
+ return Number.isFinite(value) && value >= 0 ? value : DEFAULT_RECYCLE_SUSTAIN_WINDOW_MS;
523
778
  }
524
779
  /**
525
- * Parse the minimum-uptime-floor knob (whole milliseconds, >= 0). Unlike the
526
- * sample-interval knob, 0 is a VALID value here — it disables the floor, which
527
- * the deterministic tests rely on — so only a non-integer or negative value
528
- * falls back to the 30-minute default.
780
+ * Parse the minimum-uptime-floor knob (whole milliseconds, >= 0). 0 is a VALID
781
+ * value here — it disables the floor, which the deterministic tests rely on — so
782
+ * only a non-integer or negative value falls back to the 35-minute default.
529
783
  */
530
784
  export function resolveHeapRecycleMinUptimeMs(raw) {
531
785
  if (raw === undefined)