@indigoai-us/hq-cloud 6.16.3 → 6.16.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,25 +7,38 @@
7
7
  * runner shipped no memory telemetry at all, so nothing in the abort evidence
8
8
  * could name which structure held the memory. This module closes that gap: it
9
9
  * samples the process heap on a bounded cadence, attributes live entry counts
10
- * to a fixed set of long-lived structures (the "census"), and — the moment the
11
- * used heap first crosses a configurable fraction of the ceiling — emits one
12
- * escalation record ranking those structures by growth since the session began.
10
+ * to a fixed set of long-lived structures (the "census"), and — before V8
11
+ * aborts — recycles the process so the desktop supervisor respawns a fresh one.
12
+ *
13
+ * The governor keys on the POST-GC LIVE HEAP, not the instantaneous
14
+ * `used_heap_size`. Between major collections `used_heap_size` is live-set plus
15
+ * not-yet-collected garbage, so it sawtooths by a large fraction of the ceiling
16
+ * from minute to minute; a rule that watches it can sit at a sub-threshold
17
+ * sample forever while the true live set climbs to the abort point (the
18
+ * 2026-09-01 regression: a full production day produced ZERO recycle verdicts
19
+ * against 1462 samples while the runner kept aborting). The monotonic quantity
20
+ * is the live set that survives a major GC — exactly what V8 uses for its own
21
+ * OOM verdict — so the primary signal is read on each observed MAJOR GC. A
22
+ * single instantaneous backstop remains for the case where the live signal is
23
+ * unavailable and the heap is already minutes from aborting.
13
24
  *
14
25
  * The whole module is numeric-only and allocation-light by construction:
15
26
  *
16
27
  * - Every emitted value is a number, and every census site is a FIXED
17
- * identifier token, never a path/filename/company/message. Records
18
- * therefore pass the existing telemetry sanitizer unchanged.
28
+ * identifier token, never a path/filename/company/message. Property keys are
29
+ * additionally shaped to survive the vault's ingest contract: they contain
30
+ * none of its denied key substrings and end in `_count`/`_bytes`/`_ms`, so
31
+ * the census attribution is actually stored rather than silently dropped.
19
32
  * - The controller retains only the FIRST sample (the growth baseline) and
20
33
  * the LATEST sample. It never accumulates history — a memory diagnostic
21
34
  * that leaks would regress the very defect it measures.
22
- * - All readers, timers, and emit callbacks are injected, so the module is
23
- * driven deterministically by tests with no real allocation, no real heap
24
- * growth, no wall-clock timing, and no network.
35
+ * - All readers, timers, the GC observer, and emit callbacks are injected, so
36
+ * the module is driven deterministically by tests with no real allocation,
37
+ * no real heap growth, no wall-clock timing, and no network.
25
38
  *
26
39
  * Diagnostics must never affect sync: a throwing reader, a faulty census
27
- * entry, or a rejecting emit callback degrades to silence and never disturbs a
28
- * watch cycle.
40
+ * entry, an unavailable GC entry API, or a rejecting emit callback degrades to
41
+ * silence and never disturbs a watch cycle.
29
42
  */
30
43
  /** The subset of `v8.getHeapStatistics()` this module reads. */
31
44
  export interface HeapStatisticsReading {
@@ -91,13 +104,34 @@ export declare function sampleRunnerHeap(deps: {
91
104
  census: HeapCensus;
92
105
  sessionStartedAtMs: number;
93
106
  }): RunnerHeapSample;
107
+ /**
108
+ * The observation-cadence counters carried by every emitted cloud record. They
109
+ * measure whether the work-driven checkpoint seam is keeping the governor fed
110
+ * while the 60 s interval timer is starved by a long synchronous pass. Every key
111
+ * already ends in `_count`/`_ms`, so all four survive the ingestion redaction,
112
+ * and they are always inserted BEFORE the per-census-site keys so the
113
+ * 40-property insertion-order cap can never evict them.
114
+ */
115
+ export interface ObservationCounters {
116
+ /** Total observations from every source (interval ticks + checkpoints). */
117
+ heapObservationsCount: number;
118
+ /** Observations driven by the 60 s idle interval timer. */
119
+ heapIntervalObservationsCount: number;
120
+ /** Observations driven by a work-driven checkpoint (emitter / pass / wait). */
121
+ heapCheckpointObservationsCount: number;
122
+ /** Monotonic gap since the previous observation from any source. */
123
+ msSincePreviousObservationMs: number;
124
+ }
94
125
  /**
95
126
  * Flatten a sample into numeric telemetry properties. Raw heap figures carry a
96
- * `heap_`/`rss_`/`external_` prefix; each census site becomes `census_<token>`.
97
- * Every value is a number, so the record survives the telemetry sanitizer with
98
- * no weakening of its path/name/message rejection.
127
+ * `heap_`/`rss_`/`external_` prefix; each census site becomes
128
+ * `census_<token>_count`. Every value is a number AND every key ends in
129
+ * `_count`/`_bytes`/`_ms` and contains none of the vault ingest contract's
130
+ * denied substrings, so the record is both accepted by the local sanitizer and
131
+ * actually STORED by ingest (unlike the pre-fix `census_<token>` spelling,
132
+ * which ingest silently dropped, so the attribution never reached the store).
99
133
  */
100
- export declare function heapSampleCloudProperties(sample: RunnerHeapSample): Record<string, number>;
134
+ export declare function heapSampleCloudProperties(sample: RunnerHeapSample, counters?: ObservationCounters): Record<string, number>;
101
135
  export interface HeapPressureSite {
102
136
  site: string;
103
137
  count: number;
@@ -106,7 +140,20 @@ export interface HeapPressureSite {
106
140
  /** The escalation record emitted once when the heap crosses the threshold. */
107
141
  export interface HeapPressureRecord {
108
142
  usedHeapBytes: number;
143
+ /** V8's reported `heap_size_limit`, kept for provenance. */
109
144
  heapLimitBytes: number;
145
+ /**
146
+ * The denominator every fraction is computed against:
147
+ * `min(declaredCeilingBytes, heap_size_limit)` — or the reported limit when no
148
+ * `--max-old-space-size` was declared. V8's reported limit sits ABOVE the
149
+ * declared ceiling on Node 25, so a fraction of the reported limit is
150
+ * unreachable near the top; dividing by the effective limit is what makes the
151
+ * trigger real. Equal to `heapLimitBytes` when nothing was declared.
152
+ */
153
+ effectiveLimitBytes: number;
154
+ /** The declared `--max-old-space-size` in bytes, or 0 when none was resolved. */
155
+ declaredCeilingBytes: number;
156
+ /** `usedHeapBytes / effectiveLimitBytes`. */
110
157
  usedFraction: number;
111
158
  thresholdFraction: number;
112
159
  rssBytes: number;
@@ -116,41 +163,104 @@ export interface HeapPressureRecord {
116
163
  topSites: HeapPressureSite[];
117
164
  }
118
165
  /** Flatten a pressure record into numeric-only telemetry properties. */
119
- export declare function heapPressureCloudProperties(record: HeapPressureRecord): Record<string, number>;
166
+ export declare function heapPressureCloudProperties(record: HeapPressureRecord, counters?: ObservationCounters): Record<string, number>;
120
167
  /** A compact, numeric-only stderr line that survives when the vault is down. */
121
168
  export declare function heapPressureStderrLine(record: HeapPressureRecord): string;
169
+ /**
170
+ * Which detection tier produced a recycle verdict:
171
+ * - `"live"` — the primary post-GC live path (fraction held across the
172
+ * sustain window past the uptime floor). Preserved from the shipped governor.
173
+ * - `"pinned"` — a NEW instantaneous tier: the used fraction pinned at or
174
+ * above the pinned floor across the pinned sustain window, for the starved
175
+ * synchronous stretch where no post-GC reading is ever taken.
176
+ * - `"emergency"` — a single instantaneous observation at or above the
177
+ * emergency fraction, abort-adjacent, bypassing the window and the floor.
178
+ */
179
+ export type HeapRecycleTrigger = "live" | "pinned" | "emergency";
180
+ /**
181
+ * How a verdict is actuated:
182
+ * - `"cooperative"` — the design-centre exit. The verdict is LATCHED and the
183
+ * watch loop honours it at the next pass boundary (the in-flight pass runs to
184
+ * completion, then the loop returns the benign recycle code). Every `"live"`,
185
+ * `"pinned"`, and `"emergency"` verdict is cooperative first.
186
+ * - `"latch_timeout"` — a cooperative latch the loop never honoured (the
187
+ * in-flight pass never returned) aged past the latch deadline, so a later
188
+ * observation hard-exits (writeSync-then-exit) instead of waiting for a pass
189
+ * boundary that will never arrive.
190
+ * - `"hard_fraction"` — a single instantaneous observation at or above the
191
+ * hard-exit fraction (above emergency), too close to the abort line to risk
192
+ * waiting for a pass boundary, so it hard-exits immediately.
193
+ */
194
+ export type HeapRecycleReason = "cooperative" | "latch_timeout" | "hard_fraction";
122
195
  /**
123
196
  * The governor verdict, emitted at most once per session. Where a pressure
124
197
  * record is a diagnostic breadcrumb, a recycle is ACTED ON: the watch loop
125
198
  * stops admitting new passes, lets the in-flight pass finish, and exits so the
126
199
  * desktop supervisor respawns a fresh runner BEFORE V8 aborts the current one.
127
200
  *
128
- * It fires only when ALL of these hold, so a single spike, a young process, or
129
- * a smaller ceiling can never trip it:
130
- * - `usedFraction >= recycleFraction` (default 0.78 of the reported heap
131
- * limit, self-calibrating to whatever `--max-old-space-size` was supplied);
132
- * - that has held for `sustainedSamples` consecutive samples (default 2);
133
- * - `uptimeMs >= minUptimeMs` (default 30 min).
201
+ * It fires on one of two triggers:
202
+ * - `"live"` — the primary path. Read on an observed MAJOR GC, the post-GC
203
+ * live fraction has held at or above `recycleLiveFraction` (default 0.60)
204
+ * across at least two observations spanning `sustainWindowMs` (default
205
+ * 120 s), AND the process has outlived `minUptimeMs` (default 35 min, above
206
+ * hq-desktop-app's 30-minute HEALTHY_RUN_WINDOW so a recycle clears the
207
+ * crash streak rather than pacing respawn backoff).
208
+ * - `"emergency"` — an instantaneous `used_heap_size` sample at or above
209
+ * `emergencyFraction` (default 0.90). At that reading V8 is minutes from
210
+ * aborting, so this path bypasses the sustain window AND the uptime floor:
211
+ * a clean exit-1 before the floor is still strictly better than a SIGABRT,
212
+ * which counts the same crash episode and files a Sentry capture.
134
213
  */
135
214
  export interface HeapRecycleRecord {
215
+ /** The detection tier that produced this verdict. */
216
+ trigger: HeapRecycleTrigger;
217
+ /** How the verdict is actuated (cooperative / latch_timeout / hard_fraction). */
218
+ reason: HeapRecycleReason;
219
+ /** Instantaneous `used_heap_size` at the verdict. */
136
220
  usedHeapBytes: number;
221
+ /** The governing heap figure: the post-GC live set for `"live"`, else used. */
222
+ liveHeapBytes: number;
223
+ /** V8's reported `heap_size_limit`, kept for provenance. */
137
224
  heapLimitBytes: number;
225
+ /** `min(declaredCeilingBytes, heap_size_limit)` — the fraction denominator. */
226
+ effectiveLimitBytes: number;
227
+ /** The declared `--max-old-space-size` in bytes, or 0 when none was resolved. */
228
+ declaredCeilingBytes: number;
229
+ /** `usedHeapBytes / effectiveLimitBytes`. */
138
230
  usedFraction: number;
139
- recycleFraction: number;
140
- /** Configured number of consecutive over-fraction samples required. */
141
- sustainedSamples: number;
142
- /** How many consecutive over-fraction samples were actually observed. */
143
- observedSustainedSamples: number;
144
- /** Configured uptime floor (ms) the recycle could not fire before. */
231
+ /** The governing fraction: `liveHeapBytes / effectiveLimitBytes`. */
232
+ liveFraction: number;
233
+ /** Configured post-GC live floor a `"live"` recycle requires. */
234
+ recycleLiveFraction: number;
235
+ /** Configured instantaneous floor the NEW `"pinned"` tier requires. */
236
+ pinnedFraction: number;
237
+ /** Configured instantaneous fraction the `"emergency"` tier requires. */
238
+ emergencyFraction: number;
239
+ /** Configured instantaneous fraction the `hard_fraction` backstop requires. */
240
+ hardExitFraction: number;
241
+ /** Configured minimum span between the two qualifying live observations. */
242
+ sustainWindowMs: number;
243
+ /** Actual span observed across the sustain window (0 for `"emergency"`). */
244
+ observedSustainMs: number;
245
+ /** Configured uptime floor (ms) a floored tier could not fire before. */
145
246
  minUptimeMs: number;
247
+ /** Observations since the cooperative latch armed; 0 unless `latch_timeout`. */
248
+ postLatchObservations: number;
249
+ uptimeMs: number;
146
250
  rssBytes: number;
147
251
  externalBytes: number;
148
- uptimeMs: number;
252
+ arrayBuffersBytes: number;
253
+ /** RSS/external/arrayBuffers growth since the session's first sample, so the
254
+ * next occurrence distinguishes V8 object retention from native retention. */
255
+ rssGrowthBytes: number;
256
+ externalGrowthBytes: number;
257
+ arrayBuffersGrowthBytes: number;
258
+ censusTotal: number;
149
259
  /** Census sites ranked by growth since the session's first sample. */
150
260
  topSites: HeapPressureSite[];
151
261
  }
152
262
  /** Flatten a recycle record into numeric-only telemetry properties. */
153
- export declare function heapRecycleCloudProperties(record: HeapRecycleRecord): Record<string, number>;
263
+ export declare function heapRecycleCloudProperties(record: HeapRecycleRecord, counters?: ObservationCounters): Record<string, number>;
154
264
  /**
155
265
  * A compact, numeric-only stderr line for the recycle verdict. It is worded so
156
266
  * it matches NONE of hq-desktop-app's fatal stderr classifiers (no "javascript
@@ -176,35 +286,111 @@ export interface RunnerHeapTelemetryOptions {
176
286
  */
177
287
  recycleEnabled?: boolean;
178
288
  /**
179
- * Fraction of `heap_size_limit` at or above which a recycle can arm. Default
180
- * 0.78. Chosen from the measured abort ratio (both production aborts landed
181
- * at used/declared-ceiling = 0.873): 0.78 x 2048 MB = 1597 MB, plus the
182
- * worst-case growth accrued while the sustained window confirms (2 samples x
183
- * 60 s x ~32 MB/min ~= 64 MB) = ~1661 MB, comfortably below the 1788 MB abort
184
- * point. Fractions are taken against V8's reported `heap_size_limit`, which
185
- * sits slightly ABOVE the declared `--max-old-space-size`, so the real
186
- * trigger is if anything lower than this arithmetic — the headroom errs safe.
289
+ * Post-GC live fraction of `heap_size_limit` at or above which the PRIMARY
290
+ * recycle arms. Default 0.60. Derived from the recorded aborting session: its
291
+ * post-GC live floor was <= 0.51 at 36.7 min and V8 aborted at 0.90 of the
292
+ * reported limit at 55.3 min, so 0.60 trips roughly 16 minutes ahead of the
293
+ * abort while every healthy runner in the same day's telemetry sampled below
294
+ * 0.06. Taken against V8's reported `heap_size_limit`, so it self-calibrates
295
+ * to whatever `--max-old-space-size` the desktop supplied.
187
296
  */
188
- recycleFraction?: number;
297
+ recycleLiveFraction?: number;
189
298
  /**
190
- * Consecutive over-fraction samples required before a recycle arms. Default
191
- * 2, so a single spike can never recycle.
299
+ * Minimum span (ms) the live fraction must hold at or above the floor, across
300
+ * at least two major-GC observations, before the primary recycle arms.
301
+ * Default 120 s, so a single post-GC reading taken mid-allocation cannot trip
302
+ * it.
192
303
  */
193
- recycleSustainedSamples?: number;
304
+ recycleSustainWindowMs?: number;
194
305
  /**
195
- * Process-uptime floor (ms) below which a recycle can never fire, however
196
- * high the heap climbs. Default 30 min — equal to hq-desktop-app's
197
- * HEALTHY_RUN_WINDOW — so a recycle can never advance the desktop's 60 s
198
- * fast-fail counter or its 30-minute slow-death episode streak.
306
+ * Instantaneous `used_heap_size` fraction at or above which the EMERGENCY
307
+ * backstop recycles on a single sample, bypassing the sustain window and the
308
+ * uptime floor. Default 0.90 — at that reading V8 is minutes from aborting
309
+ * and the anti-spike guard is actively harmful.
310
+ */
311
+ recycleEmergencyFraction?: number;
312
+ /**
313
+ * Process-uptime floor (ms) below which a PRIMARY (live) recycle can never
314
+ * fire, however high the heap climbs. Default 35 min — strictly above
315
+ * hq-desktop-app's 30-minute HEALTHY_RUN_WINDOW — so an ordinary recycle
316
+ * always clears the crash streak via should_reset_after_recovery instead of
317
+ * pacing respawn backoff. The emergency backstop deliberately ignores it.
199
318
  */
200
319
  recycleMinUptimeMs?: number;
320
+ /**
321
+ * The declared `--max-old-space-size` in bytes, or null/undefined when none
322
+ * was supplied. Every fraction — pressure, live, pinned, emergency, hard —
323
+ * divides `usedHeapBytes` by `effectiveLimitBytes = min(declaredCeilingBytes,
324
+ * heap_size_limit)`. On Node 25 V8's reported `heap_size_limit` (~2233 MB)
325
+ * sits ABOVE the desktop-declared 2048 MB ceiling the aborts actually respect,
326
+ * so dividing by the reported limit put every real trigger ~9% higher in bytes
327
+ * than its designed value and pushed the 0.90 emergency line onto the 1995-2064
328
+ * MB observed abort band. When absent (dev / bare node), the effective limit IS
329
+ * the reported limit and every fraction keeps its exact prior meaning.
330
+ */
331
+ declaredCeilingBytes?: number | null;
332
+ /**
333
+ * Instantaneous fraction of `effectiveLimitBytes` at or above which the NEW
334
+ * pinned tier arms once it has held across {@link recyclePinnedSustainWindowMs}.
335
+ * Default 0.84 — above the max healthy instantaneous evidence (~0.816 of
336
+ * declared) and below the legacy abort floor (1788/2048 = 0.873, ~68 MB
337
+ * margin). Unlike the live tier this reads INSTANTANEOUS used, so it observes
338
+ * the starved synchronous stretch where no post-GC reading is ever taken; it
339
+ * never feeds the live tier's post-GC sustain window.
340
+ */
341
+ recyclePinnedFraction?: number;
342
+ /**
343
+ * Minimum span (ms) the instantaneous fraction must stay at or above the pinned
344
+ * floor, across at least two gated observations, before the pinned tier arms.
345
+ * Default 20 s. Any single below-floor observation resets the anchor, so a
346
+ * healthy sawtooth (a functioning GC collapses used within seconds) never
347
+ * sustains it.
348
+ */
349
+ recyclePinnedSustainWindowMs?: number;
350
+ /**
351
+ * Instantaneous fraction of `effectiveLimitBytes` at or above which a SINGLE
352
+ * observation HARD-EXITS immediately (reason `hard_fraction`), for a climb too
353
+ * fast for any cooperative pass boundary. Default 0.955 (1956 MB on the fatal
354
+ * host) — above the 0.90 emergency line and below the post-#471 observed abort
355
+ * floor (1995/2048 = 0.974, ~39 MB margin). Bypasses the sustain window and the
356
+ * uptime floor, like a SIGABRT it preempts.
357
+ */
358
+ hardExitFraction?: number;
359
+ /**
360
+ * Monotonic ms after an unhonoured cooperative latch before the runner
361
+ * HARD-EXITS (reason `latch_timeout`). Default 60 s (~= one desktop
362
+ * respawn-backoff window and one sampler interval). Bounds actuation when an
363
+ * in-flight pass never returns to the loop to honour the latch.
364
+ */
365
+ latchHardExitMs?: number;
366
+ /**
367
+ * Post-latch observation count after which the runner HARD-EXITS even before
368
+ * the time deadline. Default 240 (= 60 s at the 250 ms checkpoint cadence, so
369
+ * it is coherent with {@link latchHardExitMs} in production). A backstop for the
370
+ * case where observations arrive faster than wall-clock; the deterministic
371
+ * tests set it small with a large time deadline to age the latch by count.
372
+ */
373
+ latchHardExitObservations?: number;
201
374
  }
202
375
  /**
203
- * Owns the sampling state machine: it keeps the baseline and latest sample, and
204
- * decides — once per rising crossing of the threshold — when to escalate. The
205
- * threshold is derived from the injected `heap_size_limit`, so it self-calibrates
206
- * to whatever `--max-old-space-size` the desktop supplied.
376
+ * Owns the sampling state machine. It keeps the baseline and latest sample, the
377
+ * pressure escalation, and the two recycle paths:
378
+ * - {@link observeMajorGc}, the PRIMARY path, driven by the GC observer: each
379
+ * (throttled) call reads the post-GC live heap and tracks the sustain
380
+ * window that a live recycle requires.
381
+ * - {@link evaluate} and the emergency arm of {@link sample}, the instantaneous
382
+ * BACKSTOP: they recycle on a single sample at/above the emergency fraction
383
+ * even if the GC signal never arrives or the sampler timer is starved.
384
+ * Every fraction is derived from the injected `heap_size_limit`, so the governor
385
+ * self-calibrates to whatever `--max-old-space-size` the desktop supplied.
207
386
  */
387
+ /** The two verdict channels an observation can produce, at most one non-null. */
388
+ export interface GovernorVerdict {
389
+ /** A cooperative recycle the loop honours at the next pass boundary. */
390
+ recycle: HeapRecycleRecord | null;
391
+ /** A hard exit (writeSync-then-exit) when cooperation cannot be honoured. */
392
+ hardExit: HeapRecycleRecord | null;
393
+ }
208
394
  export declare class RunnerHeapTelemetry {
209
395
  private readonly readers;
210
396
  private readonly census;
@@ -212,37 +398,117 @@ export declare class RunnerHeapTelemetry {
212
398
  private readonly threshold;
213
399
  private readonly topSiteCount;
214
400
  private readonly recycleEnabled;
215
- private readonly recycleFraction;
216
- private readonly recycleSustainedSamples;
401
+ private readonly declaredCeilingBytes;
402
+ private readonly recycleLiveFraction;
403
+ private readonly recycleSustainWindowMs;
404
+ private readonly recycleEmergencyFraction;
217
405
  private readonly recycleMinUptimeMs;
406
+ private readonly pinnedFraction;
407
+ private readonly pinnedSustainWindowMs;
408
+ private readonly hardExitFraction;
409
+ private readonly latchHardExitMs;
410
+ private readonly latchHardExitObservations;
218
411
  private baseline;
219
412
  private latest;
220
413
  private pressureSignalled;
221
- private recycleSustainedCount;
222
414
  private recycleSignalled;
415
+ private hardExitSignalled;
416
+ /** Uptime (ms) of the first live observation at/above the floor in the
417
+ * current consecutive run; null when the last live reading dipped below it. */
418
+ private liveAnchorUptimeMs;
419
+ /** Uptime (ms) of the first instantaneous observation at/above the pinned
420
+ * floor in the current consecutive run; null when a reading dipped below it. */
421
+ private pinnedAnchorUptimeMs;
422
+ /** Consecutive instantaneous observations at/above the pinned floor. */
423
+ private pinnedObservationCount;
424
+ /** Uptime (ms) at which the FIRST cooperative latch (any tier) armed. */
425
+ private latchedAtUptimeMs;
426
+ /** The tier that produced the cooperative latch, carried onto a hard exit. */
427
+ private latchedTrigger;
428
+ /** Observations seen since the cooperative latch armed (drives latch_timeout). */
429
+ private postLatchObservations;
223
430
  constructor(options: RunnerHeapTelemetryOptions);
431
+ /**
432
+ * The one denominator every fraction is computed against: the declared
433
+ * `--max-old-space-size` when known (it is the real V8 ceiling the aborts
434
+ * respect), else V8's reported `heap_size_limit`. Never larger than reported.
435
+ */
436
+ private effectiveLimit;
437
+ private uptimeNow;
224
438
  /**
225
439
  * Capture the growth baseline when sampling starts, before the first interval,
226
- * WITHOUT emitting or arming the pressure signal. Without this, a runner that
227
- * is already above the threshold at its first scheduled sample would make that
228
- * same high-water sample its own baseline, so the single crossing record it
229
- * emits would report zero growth for every site and lose the attribution.
440
+ * WITHOUT emitting or arming any signal. Without this, a runner that is already
441
+ * above the threshold at its first scheduled sample would make that same high
442
+ * sample its own baseline, so a crossing record would report zero growth for
443
+ * every site and lose the attribution.
230
444
  */
231
445
  primeBaseline(): RunnerHeapSample;
446
+ /**
447
+ * The interval sampler tick. Emits a sample and (at most once per rising
448
+ * crossing) a pressure record, then runs the instantaneous tiers (pinned,
449
+ * emergency, hard). The primary post-GC live verdict comes from
450
+ * {@link observeMajorGc}; this instantaneous path catches the starved
451
+ * synchronous stretch where no post-GC reading is ever taken.
452
+ */
232
453
  sample(): {
233
454
  sample: RunnerHeapSample;
234
455
  pressure: HeapPressureRecord | null;
235
456
  recycle: HeapRecycleRecord | null;
457
+ hardExit: HeapRecycleRecord | null;
236
458
  };
237
459
  /**
238
- * The governor decision. Returns a record at most once per session, and only
239
- * when the heap has stayed at or above the recycle fraction for a sustained
240
- * run of consecutive samples AND the process has outlived the uptime floor.
241
- * The sustained counter is maintained on every sample so the streak is real
242
- * consecutive pressure, but the verdict itself is gated on the floor, so a
243
- * recycle can never fire early however high the heap climbs.
460
+ * The PRIMARY governor path, driven by an observed MAJOR GC. The caller
461
+ * ({@link nodeMajorGcObserver}) only invokes this for a GC whose reading it
462
+ * could capture PROMPTLY — a callback delayed by a blocked event loop is
463
+ * skipped upstream, because `getHeapStatistics()` read late would be an
464
+ * instantaneous sawtooth value (live set plus everything allocated since the
465
+ * GC), not the post-GC live floor.
466
+ *
467
+ * The anchor is maintained on EVERY observed major GC via a cheap
468
+ * `getHeapStatistics()` read (no throttle), so a post-GC reading that dips
469
+ * below the floor ALWAYS invalidates the sustain run — a throttle that skipped
470
+ * such a reading would let a stale anchor fake a continuous run. The fuller
471
+ * sample (census snapshot + memory deltas) is taken only at the verdict, so a
472
+ * GC storm never costs more than the O(1) statistics read here. The fraction
473
+ * is denominated in the EFFECTIVE limit (min(declared, reported)) — the only
474
+ * change from the shipped live-tier logic.
475
+ */
476
+ observeMajorGc(): GovernorVerdict;
477
+ /**
478
+ * The loop-top / work-driven BACKSTOP against timer starvation (the observed
479
+ * 11-minute sampler hole). Reads the instantaneous heap and runs the
480
+ * instantaneous tiers (hard, emergency, pinned). An instantaneous reading can
481
+ * never confirm the post-GC live floor, so it never feeds the live tier's
482
+ * sustain window. Cheap: an O(1) statistics read, sampling the census only if
483
+ * a verdict actually fires.
484
+ */
485
+ evaluate(): GovernorVerdict;
486
+ /**
487
+ * The shared instantaneous tier pipeline (interval tick, loop-top, and the
488
+ * work-driven checkpoint all reach it). Order: T4 hard (hard-fraction, then an
489
+ * aged-out latch) → T3 emergency (single observation, bypasses window+floor) →
490
+ * T2 pinned (instantaneous fraction sustained across the pinned window past the
491
+ * uptime floor, with any dip resetting the anchor). The full sample is built
492
+ * only at a verdict, via `buildVerdictSample`.
493
+ */
494
+ private observeInstant;
495
+ /**
496
+ * T4 hard-exit evaluation, run at EVERY observation point. Fires at most once
497
+ * per session (independent of the cooperative latch), on either arm:
498
+ * (a) `hard_fraction` — the instantaneous fraction is at/above the hard line;
499
+ * (b) `latch_timeout` — a cooperative latch aged past its time OR observation
500
+ * deadline without the loop honouring it.
501
+ * The `latch_timeout` observation counter advances on every later observation,
502
+ * never on the observation that armed the latch (that call sees a still-false
503
+ * `recycleSignalled` before the cooperative tier runs).
244
504
  */
245
- private detectRecycle;
505
+ private checkHardExit;
506
+ private buildSample;
507
+ /** Arm the one cooperative latch (from any tier) and build its verdict. */
508
+ private armCooperativeLatch;
509
+ /** Arm the one hard exit and build its verdict (`hard_fraction`/`latch_timeout`). */
510
+ private armHardExit;
511
+ private buildRecord;
246
512
  private detectPressure;
247
513
  private topSitesByGrowth;
248
514
  /** Test accessor: the retained growth baseline. */
@@ -258,42 +524,153 @@ export interface IntervalScheduler {
258
524
  export interface IntervalHandle {
259
525
  unref?: () => void;
260
526
  }
527
+ /**
528
+ * Major-GC observer seam. Production wires {@link nodeMajorGcObserver}; tests
529
+ * inject a manual observer they fire by hand. `start` registers a callback
530
+ * invoked once per observed MAJOR GC; `stop` disconnects it. Both are wrapped
531
+ * so an unavailable or mis-shaped GC entry API degrades to silence and can
532
+ * never throw into the watch loop.
533
+ */
534
+ export interface GcObserver {
535
+ start(onMajorGc: () => void): void;
536
+ stop(): void;
537
+ }
538
+ /**
539
+ * The maximum age (ms) of a major-GC PerformanceEntry, at the moment its
540
+ * observer callback runs, for the post-GC heap read to be trustworthy. GC
541
+ * entries are delivered ASYNCHRONOUSLY, so when the event loop was blocked the
542
+ * callback fires long after the collection and a `getHeapStatistics()` read then
543
+ * reflects the live set PLUS everything allocated since — an instantaneous
544
+ * sawtooth value, not the post-GC live floor. A reading older than this is
545
+ * skipped rather than fed to the sustain window, so only promptly-captured
546
+ * (genuinely post-GC) readings can arm a live recycle.
547
+ */
548
+ export declare const MAX_FRESH_GC_READING_MS = 250;
549
+ /**
550
+ * Whether a major-GC reading taken at `nowMs` is fresh enough to treat as a
551
+ * post-GC live-heap snapshot, given the entry's end time (`entry.startTime +
552
+ * entry.duration`). Exported for deterministic testing of the staleness gate.
553
+ */
554
+ export declare function isFreshGcReading(gcEndMs: number, nowMs: number, maxAgeMs?: number): boolean;
555
+ /**
556
+ * Production major-GC observer over `perf_hooks`. Everything is wrapped so a
557
+ * runtime without the GC PerformanceEntry API, or one whose entries omit
558
+ * `detail.kind`, degrades to a silent no-op rather than throwing into the
559
+ * runner — in that degraded case the instantaneous emergency backstop and the
560
+ * pressure diagnostics remain active, so the governor is never worse than base.
561
+ *
562
+ * It only invokes the callback for a major GC whose reading it can capture
563
+ * PROMPTLY ({@link isFreshGcReading}): a callback delayed by a blocked event
564
+ * loop would read an inflated instantaneous heap rather than the post-GC live
565
+ * floor, so it is skipped.
566
+ */
567
+ export declare function nodeMajorGcObserver(): GcObserver;
261
568
  export interface RunnerHeapSamplingOptions extends RunnerHeapTelemetryOptions {
262
569
  intervalMs: number;
263
- onSample: (sample: RunnerHeapSample) => void;
264
- onPressure: (record: HeapPressureRecord) => void;
265
570
  /**
266
- * Called at most once, when the governor decides the process should recycle.
267
- * Optional so the diagnostics-only wiring (and every existing caller) needs
268
- * no change; a faulty callback is swallowed exactly like the others.
571
+ * Minimum monotonic ms between two full observations driven by the work-driven
572
+ * checkpoint seam. A checkpoint that lands closer than this to the previous
573
+ * observation FROM ANY SOURCE is a single clock compare and returns. `0`
574
+ * disables checkpoints entirely — the interval timer and the loop-top
575
+ * `evaluate()` are then the only observers, i.e. exact base behaviour. Default
576
+ * 250.
577
+ */
578
+ checkpointMs?: number;
579
+ onSample: (sample: RunnerHeapSample, counters: ObservationCounters) => void;
580
+ onPressure: (record: HeapPressureRecord, counters: ObservationCounters) => void;
581
+ /**
582
+ * Called at most once, when a cooperative tier (live/pinned/emergency) decides
583
+ * the process should recycle at the next pass boundary. Optional so the
584
+ * diagnostics-only wiring (and every existing caller) needs no change; a faulty
585
+ * callback is swallowed exactly like the others.
586
+ */
587
+ onRecycle?: (record: HeapRecycleRecord, counters: ObservationCounters) => void;
588
+ /**
589
+ * Called at most once, when a hard-exit tier (reason `hard_fraction` or
590
+ * `latch_timeout`) fires. The production wiring writes the NDJSON + stderr
591
+ * records synchronously and exits the process; a test captures the record
592
+ * instead. When a hard exit and a cooperative recycle would fire on the same
593
+ * observation, ONLY the hard exit is delivered — the process is leaving now, so
594
+ * the cooperative latch would be moot. A faulty callback is swallowed.
269
595
  */
270
- onRecycle?: (record: HeapRecycleRecord) => void;
596
+ onHardExit?: (record: HeapRecycleRecord, counters: ObservationCounters) => void;
271
597
  scheduler?: IntervalScheduler;
598
+ /**
599
+ * Major-GC observer that drives the primary live-heap path. Optional: when
600
+ * absent, only the instantaneous tiers and pressure diagnostics run. Production
601
+ * wires {@link nodeMajorGcObserver}; tests inject a manual one.
602
+ */
603
+ gcObserver?: GcObserver;
272
604
  log?: (message: string) => void;
273
605
  }
274
606
  export interface RunnerHeapSamplingHandle {
275
607
  stop: () => void;
608
+ /**
609
+ * The loop-top backstop: run one ungated instantaneous check and fire
610
+ * `onRecycle`/`onHardExit` if it trips. Cheap and safe to call every watch-loop
611
+ * iteration; a fault degrades to silence. Runs regardless of `checkpointMs`.
612
+ */
613
+ evaluate: () => void;
614
+ /**
615
+ * The work-driven checkpoint seam. The watch loop wires it to the protocol-
616
+ * event emitter, pass boundaries, and wait resume so the governor keeps
617
+ * observing while a long synchronous pass starves the interval timer. Gated to
618
+ * at most one full observation per `checkpointMs`; below the gate it is a
619
+ * single monotonic clock compare. Never emits a per-observation cloud sample.
620
+ */
621
+ maybeObserve: () => void;
276
622
  readonly telemetry: RunnerHeapTelemetry;
277
623
  }
278
624
  /**
279
- * Start periodic sampling. The timer is `unref`'d so it can never hold the
280
- * process open, and `stop()` clears it on every shutdown path. Each tick — and
281
- * every emit callback it drives — is wrapped so a fault degrades to silence and
282
- * never affects the watch cycle.
625
+ * Start periodic sampling and, when a GC observer is supplied, the primary
626
+ * live-heap path. The interval timer is `unref`'d so it can never hold the
627
+ * process open, and `stop()` clears it and disconnects the GC observer on every
628
+ * shutdown path. The interval tick, the work-driven checkpoint, and the loop-top
629
+ * `evaluate()` share ONE observation pipeline, one last-observation timestamp,
630
+ * and one set of cadence counters. Each observation — and every emit callback it
631
+ * drives — is wrapped so a fault degrades to silence and never affects the watch
632
+ * cycle.
283
633
  */
284
634
  export declare function startRunnerHeapSampling(options: RunnerHeapSamplingOptions): RunnerHeapSamplingHandle;
635
+ export interface HeapHardExitIo {
636
+ writeStdout: (line: string) => void;
637
+ writeStderr: (line: string) => void;
638
+ exit: (code: number) => void;
639
+ }
640
+ /**
641
+ * Perform a deliberate hard exit for a `hard_fraction`/`latch_timeout` verdict.
642
+ * Writes the additive `heap-recycle` NDJSON record to stdout and the compact
643
+ * numeric stderr line — through SYNCHRONOUS injected writers, so both records
644
+ * land even when the streams are buffered or blocked — and only THEN exits. The
645
+ * exit code and the writers are injected so the exact write-before-exit ordering
646
+ * is unit-testable and the code stays the runner's single benign-exit constant.
647
+ */
648
+ export declare function performHeapHardExit(record: HeapRecycleRecord, io: HeapHardExitIo, exitCode: number): void;
285
649
  /** Env knob names, exported so the runner and its tests agree on the strings. */
286
650
  export declare const HEAP_SAMPLE_INTERVAL_ENV = "HQ_SYNC_RUNNER_HEAP_SAMPLE_MS";
287
651
  export declare const HEAP_PRESSURE_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_PRESSURE_FRACTION";
288
652
  export declare const HEAP_RECYCLE_ENABLED_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_ENABLED";
289
- export declare const HEAP_RECYCLE_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_FRACTION";
290
- export declare const HEAP_RECYCLE_SUSTAINED_SAMPLES_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_SUSTAINED_SAMPLES";
653
+ export declare const HEAP_RECYCLE_LIVE_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_LIVE_FRACTION";
654
+ export declare const HEAP_RECYCLE_SUSTAIN_WINDOW_MS_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_SUSTAIN_WINDOW_MS";
655
+ export declare const HEAP_RECYCLE_EMERGENCY_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_EMERGENCY_FRACTION";
291
656
  export declare const HEAP_RECYCLE_MIN_UPTIME_MS_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_MIN_UPTIME_MS";
657
+ export declare const HEAP_CHECKPOINT_MS_ENV = "HQ_SYNC_RUNNER_HEAP_CHECKPOINT_MS";
658
+ export declare const HEAP_PINNED_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_PINNED_FRACTION";
659
+ export declare const HEAP_PINNED_SUSTAIN_MS_ENV = "HQ_SYNC_RUNNER_HEAP_PINNED_SUSTAIN_MS";
660
+ export declare const HEAP_HARD_EXIT_FRACTION_ENV = "HQ_SYNC_RUNNER_HEAP_HARD_EXIT_FRACTION";
661
+ export declare const HEAP_RECYCLE_LATCH_HARD_EXIT_MS_ENV = "HQ_SYNC_RUNNER_HEAP_RECYCLE_LATCH_HARD_EXIT_MS";
292
662
  export declare const DEFAULT_HEAP_SAMPLE_INTERVAL_MS = 60000;
293
663
  export declare const DEFAULT_HEAP_PRESSURE_FRACTION = 0.75;
294
- export declare const DEFAULT_HEAP_RECYCLE_FRACTION = 0.78;
295
- export declare const DEFAULT_HEAP_RECYCLE_SUSTAINED_SAMPLES = 2;
664
+ export declare const DEFAULT_HEAP_RECYCLE_LIVE_FRACTION = 0.6;
665
+ export declare const DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS = 120000;
666
+ export declare const DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION = 0.9;
296
667
  export declare const DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS: number;
668
+ export declare const DEFAULT_HEAP_CHECKPOINT_MS = 250;
669
+ export declare const DEFAULT_HEAP_PINNED_FRACTION = 0.84;
670
+ export declare const DEFAULT_HEAP_PINNED_SUSTAIN_MS = 20000;
671
+ export declare const DEFAULT_HEAP_HARD_EXIT_FRACTION = 0.955;
672
+ export declare const DEFAULT_HEAP_RECYCLE_LATCH_HARD_EXIT_MS = 60000;
673
+ export declare const DEFAULT_HEAP_RECYCLE_LATCH_HARD_EXIT_OBSERVATIONS = 240;
297
674
  /**
298
675
  * Parse the sampling-interval env knob (whole milliseconds). Rejects the ENTIRE
299
676
  * string: a value like "60s" must fall back to the default, not silently parse
@@ -315,25 +692,70 @@ export declare function resolveHeapPressureFraction(raw: string | undefined): nu
315
692
  */
316
693
  export declare function resolveHeapRecycleEnabled(raw: string | undefined): boolean;
317
694
  /**
318
- * Parse the recycle-fraction knob; falls back to the default (0.78) on garbage
319
- * or out-of-range input. Rejects the entire string (e.g. "0.8x" falls back)
320
- * rather than accepting a numeric prefix.
695
+ * Parse the primary live-fraction knob; falls back to the default (0.60) on
696
+ * garbage or out-of-range input. Rejects the entire string (e.g. "0.6x" falls
697
+ * back) rather than accepting a numeric prefix.
321
698
  */
322
- export declare function resolveHeapRecycleFraction(raw: string | undefined): number;
699
+ export declare function resolveHeapRecycleLiveFraction(raw: string | undefined): number;
323
700
  /**
324
- * Parse the sustained-samples knob (a whole count >= 1). Falls back to the
325
- * default (2) on garbage, 0, or a non-integer — a degenerate 0 would let a
326
- * single spike recycle, exactly what the sustained requirement exists to
327
- * prevent, so it is refused rather than honoured.
701
+ * Parse the emergency-fraction knob; falls back to the default (0.90) on garbage
702
+ * or out-of-range input. Rejects the entire string rather than a numeric prefix.
328
703
  */
329
- export declare function resolveHeapRecycleSustainedSamples(raw: string | undefined): number;
704
+ export declare function resolveHeapRecycleEmergencyFraction(raw: string | undefined): number;
330
705
  /**
331
- * Parse the minimum-uptime-floor knob (whole milliseconds, >= 0). Unlike the
332
- * sample-interval knob, 0 is a VALID value here — it disables the floor, which
333
- * the deterministic tests rely on — so only a non-integer or negative value
334
- * falls back to the 30-minute default.
706
+ * Parse the sustain-window knob (whole milliseconds, >= 0). 0 is a VALID value
707
+ * here — it removes the window so two qualifying observations at the same uptime
708
+ * suffice, which the deterministic tests rely on — so only a non-integer or
709
+ * negative value falls back to the 120-second default.
710
+ */
711
+ export declare function resolveHeapRecycleSustainWindowMs(raw: string | undefined): number;
712
+ /**
713
+ * Parse the minimum-uptime-floor knob (whole milliseconds, >= 0). 0 is a VALID
714
+ * value here — it disables the floor, which the deterministic tests rely on — so
715
+ * only a non-integer or negative value falls back to the 35-minute default.
335
716
  */
336
717
  export declare function resolveHeapRecycleMinUptimeMs(raw: string | undefined): number;
718
+ /**
719
+ * Parse the checkpoint-gate knob (whole milliseconds, >= 0). 0 is a VALID value
720
+ * — it disables the work-driven checkpoints, leaving the interval timer and the
721
+ * loop-top backstop as the only observers (exact base behaviour) — so only a
722
+ * non-integer or negative value falls back to the 250 ms default.
723
+ */
724
+ export declare function resolveHeapCheckpointMs(raw: string | undefined): number;
725
+ /**
726
+ * Parse the pinned-fraction knob; falls back to the default (0.84) on garbage or
727
+ * out-of-range input. Rejects the entire string rather than a numeric prefix.
728
+ */
729
+ export declare function resolveHeapPinnedFraction(raw: string | undefined): number;
730
+ /**
731
+ * Parse the pinned-sustain-window knob (whole milliseconds, >= 0). 0 is a VALID
732
+ * value — it removes the window so two qualifying observations at the same uptime
733
+ * suffice, which the deterministic tests rely on — so only a non-integer or
734
+ * negative value falls back to the 20-second default.
735
+ */
736
+ export declare function resolveHeapPinnedSustainMs(raw: string | undefined): number;
737
+ /**
738
+ * Parse the hard-exit-fraction knob; falls back to the default (0.955) on garbage
739
+ * or out-of-range input. Rejects the entire string rather than a numeric prefix.
740
+ */
741
+ export declare function resolveHeapHardExitFraction(raw: string | undefined): number;
742
+ /**
743
+ * Parse the latch-hard-exit knob (whole milliseconds, >= 0). 0 is a VALID value
744
+ * — it makes the first post-latch observation hard-exit — so only a non-integer
745
+ * or negative value falls back to the 60-second default.
746
+ */
747
+ export declare function resolveHeapRecycleLatchHardExitMs(raw: string | undefined): number;
748
+ /**
749
+ * Resolve the declared V8 old-space ceiling (bytes) the runner is actually
750
+ * running under, mirroring how hq-desktop-app hands it to the npx-spawned
751
+ * runner: a `--max-old-space-size` flag on `process.execArgv` wins over one in
752
+ * `NODE_OPTIONS`, and within each source the LAST occurrence wins (V8's own
753
+ * precedence). Both the `=value` and adjacent-arg forms are accepted, values may
754
+ * carry one pair of surrounding quotes, and only a whole positive integer of MB
755
+ * is honoured — anything else yields null, so the governor falls back to V8's
756
+ * reported `heap_size_limit`. Never throws.
757
+ */
758
+ export declare function resolveDeclaredHeapCeilingBytes(execArgv: readonly string[] | undefined, nodeOptions: string | undefined): number | null;
337
759
  /**
338
760
  * Production readers over the real runtime. Kept here so the wiring site stays
339
761
  * a one-liner and the `v8` import lives in exactly one place.