@indigoai-us/hq-cloud 6.16.3 → 6.16.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-events.d.ts +10 -0
- package/dist/bin/sync-runner-events.d.ts.map +1 -1
- package/dist/bin/sync-runner-events.js +34 -0
- package/dist/bin/sync-runner-events.js.map +1 -1
- package/dist/bin/sync-runner-events.test.js +87 -13
- package/dist/bin/sync-runner-events.test.js.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.d.ts +512 -90
- package/dist/bin/sync-runner-heap-telemetry.d.ts.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.js +794 -151
- package/dist/bin/sync-runner-heap-telemetry.js.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.test.js +964 -204
- package/dist/bin/sync-runner-heap-telemetry.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +132 -22
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +60 -10
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +58 -14
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +10 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +7 -1
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +54 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, expect, it, vi } from "vitest";
|
|
2
|
-
import { HeapCensus, RunnerHeapTelemetry, sampleRunnerHeap, heapSampleCloudProperties, heapPressureStderrLine, heapRecycleStderrLine, startRunnerHeapSampling, resolveHeapPressureFraction, resolveHeapSampleIntervalMs, resolveHeapRecycleEnabled,
|
|
2
|
+
import { HeapCensus, RunnerHeapTelemetry, sampleRunnerHeap, heapSampleCloudProperties, heapPressureCloudProperties, heapRecycleCloudProperties, heapPressureStderrLine, heapRecycleStderrLine, performHeapHardExit, startRunnerHeapSampling, nodeMajorGcObserver, isFreshGcReading, MAX_FRESH_GC_READING_MS, resolveHeapPressureFraction, resolveHeapSampleIntervalMs, resolveHeapRecycleEnabled, resolveHeapRecycleLiveFraction, resolveHeapRecycleEmergencyFraction, resolveHeapRecycleSustainWindowMs, resolveHeapRecycleMinUptimeMs, resolveHeapCheckpointMs, resolveHeapPinnedFraction, resolveHeapPinnedSustainMs, resolveHeapHardExitFraction, resolveHeapRecycleLatchHardExitMs, resolveDeclaredHeapCeilingBytes, DEFAULT_HEAP_RECYCLE_LIVE_FRACTION, DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION, DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS, DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS, DEFAULT_HEAP_CHECKPOINT_MS, DEFAULT_HEAP_PINNED_FRACTION, DEFAULT_HEAP_PINNED_SUSTAIN_MS, DEFAULT_HEAP_HARD_EXIT_FRACTION, DEFAULT_HEAP_RECYCLE_LATCH_HARD_EXIT_MS, } from "./sync-runner-heap-telemetry.js";
|
|
3
3
|
import { emitCloudTelemetry } from "../telemetry-events.js";
|
|
4
4
|
function mutableReaders() {
|
|
5
5
|
const state = {
|
|
@@ -55,6 +55,28 @@ function fakeScheduler() {
|
|
|
55
55
|
unrefCalls: () => unrefCalls,
|
|
56
56
|
};
|
|
57
57
|
}
|
|
58
|
+
/** A manual GC observer whose registered callback the test fires by hand. */
|
|
59
|
+
function fakeGcObserver() {
|
|
60
|
+
let cb = null;
|
|
61
|
+
let started = false;
|
|
62
|
+
let stopped = false;
|
|
63
|
+
const observer = {
|
|
64
|
+
start: (onMajorGc) => {
|
|
65
|
+
cb = onMajorGc;
|
|
66
|
+
started = true;
|
|
67
|
+
},
|
|
68
|
+
stop: () => {
|
|
69
|
+
stopped = true;
|
|
70
|
+
cb = null;
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
return {
|
|
74
|
+
observer,
|
|
75
|
+
fire: () => cb?.(),
|
|
76
|
+
started: () => started,
|
|
77
|
+
stopped: () => stopped,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
58
80
|
describe("sampleRunnerHeap", () => {
|
|
59
81
|
it("maps injected v8/process readings to a flat numeric record", () => {
|
|
60
82
|
const census = new HeapCensus();
|
|
@@ -88,11 +110,17 @@ describe("sampleRunnerHeap", () => {
|
|
|
88
110
|
{ site: "setB", count: 5 },
|
|
89
111
|
]);
|
|
90
112
|
});
|
|
91
|
-
it("emits only numeric properties that survive the
|
|
113
|
+
it("emits only numeric properties that survive the LOCAL telemetry sanitizer", async () => {
|
|
114
|
+
// NOTE: this exercises hq-cloud's own emit-side sanitizer, which keys on the
|
|
115
|
+
// VALUE type — a numeric value survives regardless of key. It is a DIFFERENT
|
|
116
|
+
// and more permissive layer than hq-pro's ingest contract (the vault's
|
|
117
|
+
// redactProperties, which drops any key not ending in _count/_bytes/_ms or
|
|
118
|
+
// containing a denied substring). The regression this module fixes is that
|
|
119
|
+
// the old `census_<token>` spelling passed HERE but was silently dropped at
|
|
120
|
+
// ingest; the deny-safe `_count` spelling — pinned by the ingest-key-contract
|
|
121
|
+
// test below — is what makes the attribution actually reach the store.
|
|
92
122
|
const census = new HeapCensus();
|
|
93
|
-
|
|
94
|
-
// "content": it must still survive BECAUSE its value is numeric.
|
|
95
|
-
census.register("publishedContentHashGenerations", () => 12345);
|
|
123
|
+
census.register("publishedDigestGenerations", () => 12345);
|
|
96
124
|
census.register("knownKinds", () => 10);
|
|
97
125
|
const { state, readers } = mutableReaders();
|
|
98
126
|
Object.assign(state, {
|
|
@@ -123,7 +151,7 @@ describe("sampleRunnerHeap", () => {
|
|
|
123
151
|
properties: props,
|
|
124
152
|
});
|
|
125
153
|
const survived = captured[0]?.properties ?? {};
|
|
126
|
-
expect(survived.
|
|
154
|
+
expect(survived.census_publishedDigestGenerations_count).toBe(12345);
|
|
127
155
|
expect(survived.heap_used_bytes).toBe(1);
|
|
128
156
|
// No property was dropped by the sanitizer.
|
|
129
157
|
expect(Object.keys(survived).length).toBe(Object.keys(props).length);
|
|
@@ -339,6 +367,8 @@ describe("heapPressureStderrLine", () => {
|
|
|
339
367
|
const line = heapPressureStderrLine({
|
|
340
368
|
usedHeapBytes: 100,
|
|
341
369
|
heapLimitBytes: 200,
|
|
370
|
+
effectiveLimitBytes: 200,
|
|
371
|
+
declaredCeilingBytes: 0,
|
|
342
372
|
usedFraction: 0.5,
|
|
343
373
|
thresholdFraction: 0.75,
|
|
344
374
|
rssBytes: 300,
|
|
@@ -352,166 +382,280 @@ describe("heapPressureStderrLine", () => {
|
|
|
352
382
|
expect(line).toContain("top=knownKinds:3");
|
|
353
383
|
});
|
|
354
384
|
});
|
|
355
|
-
// --- Self-recycling governor
|
|
385
|
+
// --- Self-recycling governor (GC-anchored post-GC live floor) -----------------
|
|
356
386
|
const MB = 1024 * 1024;
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
function runToRecycle(opts) {
|
|
364
|
-
const census = opts.census ?? new HeapCensus();
|
|
365
|
-
if (!opts.census)
|
|
387
|
+
// Comfortably past the 35-minute uptime floor, so the floor never gates a test
|
|
388
|
+
// that is exercising a different property.
|
|
389
|
+
const PAST_FLOOR_MS = DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS + 5 * 60_000;
|
|
390
|
+
function governor(overrides = {}) {
|
|
391
|
+
const census = overrides.census ?? new HeapCensus();
|
|
392
|
+
if (!overrides.census)
|
|
366
393
|
census.register("knownKinds", () => 1);
|
|
367
394
|
const { state, readers } = mutableReaders();
|
|
368
|
-
state.limit =
|
|
369
|
-
state.used = opts.usedBytes;
|
|
370
|
-
state.now = opts.uptimeMs ?? ONE_HOUR_MS;
|
|
395
|
+
state.limit = overrides.limitBytes ?? 2048 * MB;
|
|
371
396
|
const controller = new RunnerHeapTelemetry({
|
|
372
397
|
readers,
|
|
373
398
|
census,
|
|
374
399
|
sessionStartedAtMs: 0,
|
|
375
|
-
pressureThresholdFraction:
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
400
|
+
pressureThresholdFraction: 0.75,
|
|
401
|
+
recycleLiveFraction: overrides.liveFraction ?? 0.6,
|
|
402
|
+
recycleSustainWindowMs: overrides.windowMs ?? 120_000,
|
|
403
|
+
recycleEmergencyFraction: overrides.emergencyFraction ?? 0.9,
|
|
404
|
+
recycleMinUptimeMs: overrides.minUptimeMs ?? DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS,
|
|
405
|
+
recycleEnabled: overrides.recycleEnabled ?? true,
|
|
406
|
+
...(overrides.declaredCeilingBytes !== undefined
|
|
407
|
+
? { declaredCeilingBytes: overrides.declaredCeilingBytes }
|
|
408
|
+
: {}),
|
|
409
|
+
...(overrides.pinnedFraction !== undefined
|
|
410
|
+
? { recyclePinnedFraction: overrides.pinnedFraction }
|
|
411
|
+
: {}),
|
|
412
|
+
...(overrides.pinnedSustainWindowMs !== undefined
|
|
413
|
+
? { recyclePinnedSustainWindowMs: overrides.pinnedSustainWindowMs }
|
|
414
|
+
: {}),
|
|
415
|
+
...(overrides.hardExitFraction !== undefined
|
|
416
|
+
? { hardExitFraction: overrides.hardExitFraction }
|
|
417
|
+
: {}),
|
|
418
|
+
...(overrides.latchHardExitMs !== undefined
|
|
419
|
+
? { latchHardExitMs: overrides.latchHardExitMs }
|
|
420
|
+
: {}),
|
|
421
|
+
...(overrides.latchHardExitObservations !== undefined
|
|
422
|
+
? { latchHardExitObservations: overrides.latchHardExitObservations }
|
|
423
|
+
: {}),
|
|
424
|
+
});
|
|
425
|
+
const setAt = (fraction, nowMs) => {
|
|
426
|
+
state.used = Math.round(state.limit * fraction);
|
|
427
|
+
state.now = nowMs;
|
|
428
|
+
};
|
|
429
|
+
// The cooperative recycle channel (backward-compatible with the shipped tests).
|
|
430
|
+
const observeAt = (fraction, nowMs) => {
|
|
431
|
+
setAt(fraction, nowMs);
|
|
432
|
+
return controller.observeMajorGc().recycle;
|
|
433
|
+
};
|
|
434
|
+
const sampleAt = (fraction, nowMs) => {
|
|
435
|
+
setAt(fraction, nowMs);
|
|
436
|
+
return controller.sample();
|
|
437
|
+
};
|
|
438
|
+
const evaluateAt = (fraction, nowMs) => {
|
|
439
|
+
setAt(fraction, nowMs);
|
|
440
|
+
return controller.evaluate().recycle;
|
|
441
|
+
};
|
|
442
|
+
// The full verdict channel (both cooperative recycle AND hard exit).
|
|
443
|
+
const observeFullAt = (fraction, nowMs) => {
|
|
444
|
+
setAt(fraction, nowMs);
|
|
445
|
+
return controller.observeMajorGc();
|
|
446
|
+
};
|
|
447
|
+
const evaluateFullAt = (fraction, nowMs) => {
|
|
448
|
+
setAt(fraction, nowMs);
|
|
449
|
+
return controller.evaluate();
|
|
450
|
+
};
|
|
451
|
+
return {
|
|
452
|
+
state,
|
|
453
|
+
readers,
|
|
454
|
+
census,
|
|
455
|
+
controller,
|
|
456
|
+
setAt,
|
|
457
|
+
observeAt,
|
|
458
|
+
sampleAt,
|
|
459
|
+
evaluateAt,
|
|
460
|
+
observeFullAt,
|
|
461
|
+
evaluateFullAt,
|
|
462
|
+
};
|
|
391
463
|
}
|
|
392
|
-
describe("RunnerHeapTelemetry — recycle governor", () => {
|
|
393
|
-
it("
|
|
394
|
-
const
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
recycleMinUptimeMs: 30 * 60_000,
|
|
407
|
-
});
|
|
408
|
-
// One spike above the fraction, then a dip below: the streak resets, so no
|
|
409
|
-
// recycle can ever accumulate from a single excursion.
|
|
410
|
-
state.used = Math.round(2048 * MB * 0.9);
|
|
411
|
-
expect(controller.sample().recycle).toBeNull(); // 1 consecutive
|
|
412
|
-
state.used = Math.round(2048 * MB * 0.5);
|
|
413
|
-
expect(controller.sample().recycle).toBeNull(); // dip -> streak reset
|
|
414
|
-
state.used = Math.round(2048 * MB * 0.9);
|
|
415
|
-
expect(controller.sample().recycle).toBeNull(); // 1 consecutive again
|
|
416
|
-
// Now a genuine sustained run of two consecutive over-fraction samples.
|
|
417
|
-
const second = controller.sample().recycle; // 2 consecutive -> recycle
|
|
418
|
-
expect(second).not.toBeNull();
|
|
419
|
-
expect(second?.observedSustainedSamples).toBe(2);
|
|
420
|
-
expect(second?.sustainedSamples).toBe(2);
|
|
421
|
-
});
|
|
422
|
-
it("never recycles before the uptime floor, however high the heap climbs", () => {
|
|
423
|
-
// Heap pinned at 0.99 of the ceiling for many samples, but uptime is below
|
|
424
|
-
// the 30-minute floor: the desktop's fast-fail (60s) and slow-death
|
|
425
|
-
// (30-min) counters can never be advanced by a recycle.
|
|
426
|
-
const belowFloor = runToRecycle({
|
|
427
|
-
limitBytes: 2048 * MB,
|
|
428
|
-
usedBytes: Math.round(2048 * MB * 0.99),
|
|
429
|
-
uptimeMs: 30 * 60_000 - 1, // one ms below the floor
|
|
430
|
-
samples: 12,
|
|
431
|
-
});
|
|
432
|
-
expect(belowFloor.recycle).toBeNull();
|
|
433
|
-
expect(belowFloor.recycleCount).toBe(0);
|
|
434
|
-
// At the floor, the sustained streak (already satisfied) lets it recycle.
|
|
435
|
-
const atFloor = runToRecycle({
|
|
436
|
-
limitBytes: 2048 * MB,
|
|
437
|
-
usedBytes: Math.round(2048 * MB * 0.99),
|
|
438
|
-
uptimeMs: 30 * 60_000,
|
|
439
|
-
samples: 12,
|
|
440
|
-
});
|
|
441
|
-
expect(atFloor.recycle).not.toBeNull();
|
|
464
|
+
describe("RunnerHeapTelemetry — recycle governor (primary post-GC live path)", () => {
|
|
465
|
+
it("recycles on a sustained post-GC live floor across the window, past the uptime floor", () => {
|
|
466
|
+
const g = governor();
|
|
467
|
+
// First qualifying post-GC observation at/above the live floor: anchors only.
|
|
468
|
+
expect(g.observeAt(0.62, PAST_FLOOR_MS)).toBeNull();
|
|
469
|
+
// Too soon (less than the 120 s window since the anchor): still no verdict.
|
|
470
|
+
expect(g.observeAt(0.65, PAST_FLOOR_MS + 119_000)).toBeNull();
|
|
471
|
+
// A window later, still at/above the floor: recycle, trigger "live".
|
|
472
|
+
const recycle = g.observeAt(0.66, PAST_FLOOR_MS + 121_000);
|
|
473
|
+
expect(recycle).not.toBeNull();
|
|
474
|
+
expect(recycle?.trigger).toBe("live");
|
|
475
|
+
expect(recycle?.liveFraction).toBeCloseTo(0.66, 2);
|
|
476
|
+
expect(recycle?.observedSustainMs).toBe(121_000);
|
|
477
|
+
expect(recycle?.recycleLiveFraction).toBe(0.6);
|
|
442
478
|
});
|
|
443
|
-
it("
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
//
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
expect(
|
|
468
|
-
expect(
|
|
469
|
-
expect(
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
//
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
const
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
479
|
+
it("is immune to an instantaneous sawtooth: neither the sample path nor a below-floor live reading ever recycles", () => {
|
|
480
|
+
// The exact defect: instantaneous used_heap_size oscillates high (0.60..0.89,
|
|
481
|
+
// never at the 0.90 emergency line) while the POST-GC live floor stays low
|
|
482
|
+
// (0.30). The old rule watched the instantaneous series and would have tripped;
|
|
483
|
+
// the GC-anchored rule watches only the post-GC floor and never does.
|
|
484
|
+
const g = governor();
|
|
485
|
+
const inst = [0.62, 0.30, 0.88, 0.35, 0.89, 0.28, 0.87, 0.40];
|
|
486
|
+
let t = PAST_FLOOR_MS;
|
|
487
|
+
for (const f of inst) {
|
|
488
|
+
// The interval sampler sees the instantaneous sawtooth — emergency only,
|
|
489
|
+
// and 0.89 < 0.90, so it never fires.
|
|
490
|
+
expect(g.sampleAt(f, t).recycle).toBeNull();
|
|
491
|
+
// The post-GC live floor is flat at 0.30, well below the 0.60 floor.
|
|
492
|
+
expect(g.observeAt(0.3, t)).toBeNull();
|
|
493
|
+
t += 60_000;
|
|
494
|
+
}
|
|
495
|
+
});
|
|
496
|
+
it("requires the window between the TWO qualifying observations, and resets the anchor on a dip", () => {
|
|
497
|
+
const g = governor();
|
|
498
|
+
// Two qualifying observations less than a window apart never recycle.
|
|
499
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS)).toBeNull();
|
|
500
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS + 60_000)).toBeNull(); // 60 s < 120 s
|
|
501
|
+
// A dip below the floor resets the anchor: the clock since the FIRST
|
|
502
|
+
// observation no longer counts.
|
|
503
|
+
expect(g.observeAt(0.4, PAST_FLOOR_MS + 90_000)).toBeNull(); // dip -> anchor cleared
|
|
504
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS + 100_000)).toBeNull(); // re-anchor here
|
|
505
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS + 180_000)).toBeNull(); // only 80 s since re-anchor
|
|
506
|
+
const recycle = g.observeAt(0.7, PAST_FLOOR_MS + 221_000); // 121 s since re-anchor
|
|
507
|
+
expect(recycle).not.toBeNull();
|
|
508
|
+
// The sustain span is measured from the RE-ANCHOR, not the first observation.
|
|
509
|
+
expect(recycle?.observedSustainMs).toBe(121_000);
|
|
510
|
+
});
|
|
511
|
+
it("never recycles (live path) before the uptime floor, however high the live floor climbs", () => {
|
|
512
|
+
const g = governor();
|
|
513
|
+
// Live floor pinned at 0.80 for many observations, but every one is below the
|
|
514
|
+
// 35-minute floor: no verdict, so the desktop's crash streak is never touched.
|
|
515
|
+
let t = 60_000;
|
|
516
|
+
for (let i = 0; i < 40; i += 1) {
|
|
517
|
+
expect(g.observeAt(0.8, t)).toBeNull();
|
|
518
|
+
t += 30_000; // stays under DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS (35 min)
|
|
519
|
+
}
|
|
520
|
+
// Cross the floor with the window already satisfied -> recycle.
|
|
521
|
+
const recycle = g.observeAt(0.8, DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS + 1);
|
|
522
|
+
expect(recycle).not.toBeNull();
|
|
523
|
+
expect(recycle?.trigger).toBe("live");
|
|
524
|
+
});
|
|
525
|
+
it("keeps the uptime floor strictly above hq-desktop-app's 30-minute HEALTHY_RUN_WINDOW", () => {
|
|
526
|
+
// A recycle must OUTLIVE the desktop's 30-minute window so should_reset_after_recovery
|
|
527
|
+
// clears the crash streak instead of pacing respawn backoff. The literal is
|
|
528
|
+
// quoted so a future change that would drop the floor to/under it fails here.
|
|
529
|
+
const DESKTOP_HEALTHY_RUN_WINDOW_MS = 1_800_000;
|
|
530
|
+
expect(DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS).toBe(2_100_000); // 35 min
|
|
531
|
+
expect(DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS).toBeGreaterThan(DESKTOP_HEALTHY_RUN_WINDOW_MS);
|
|
532
|
+
});
|
|
533
|
+
it("scales the live floor with the reported heap limit, so a larger ceiling never recycles prematurely", () => {
|
|
534
|
+
// 1700 MB is 0.83 of a 2048 MB ceiling (recycles) but only 0.415 of a 4096 MB
|
|
535
|
+
// ceiling (below the 0.60 live floor -> never), because the floor is a
|
|
536
|
+
// fraction of the injected heap_size_limit, not an absolute figure.
|
|
537
|
+
const small = governor({ limitBytes: 2048 * MB });
|
|
538
|
+
small.state.used = 1700 * MB;
|
|
539
|
+
small.state.now = PAST_FLOOR_MS;
|
|
540
|
+
expect(small.controller.observeMajorGc().recycle).toBeNull(); // anchor
|
|
541
|
+
small.state.now = PAST_FLOOR_MS + 121_000;
|
|
542
|
+
expect(small.controller.observeMajorGc().recycle).not.toBeNull();
|
|
543
|
+
const large = governor({ limitBytes: 4096 * MB });
|
|
544
|
+
let t = PAST_FLOOR_MS;
|
|
545
|
+
for (let i = 0; i < 10; i += 1) {
|
|
546
|
+
large.state.used = 1700 * MB; // 0.415 of 4096 MB
|
|
547
|
+
large.state.now = t;
|
|
548
|
+
expect(large.controller.observeMajorGc().recycle).toBeNull();
|
|
549
|
+
t += 121_000;
|
|
550
|
+
}
|
|
551
|
+
});
|
|
552
|
+
it("maintains the anchor on every major GC, so a close below-floor recovery resets the sustain run", () => {
|
|
553
|
+
// The anchor is updated on EVERY observed major GC — there is no throttle
|
|
554
|
+
// that could skip a below-floor recovery and leave a stale anchor faking a
|
|
555
|
+
// continuous run (the reviewed defect).
|
|
556
|
+
const g = governor();
|
|
557
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS)).toBeNull(); // anchor
|
|
558
|
+
expect(g.observeAt(0.4, PAST_FLOOR_MS + 1_000)).toBeNull(); // 1 s later, below floor -> reset
|
|
559
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS + 2_000)).toBeNull(); // re-anchor here
|
|
560
|
+
// 121 s past the FIRST anchor but only 119 s past the RE-anchor: no recycle.
|
|
561
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS + 121_000)).toBeNull();
|
|
562
|
+
// A window past the re-anchor -> recycle, sustain measured from the re-anchor.
|
|
563
|
+
const recycle = g.observeAt(0.7, PAST_FLOOR_MS + 123_000);
|
|
564
|
+
expect(recycle).not.toBeNull();
|
|
565
|
+
expect(recycle?.observedSustainMs).toBe(121_000);
|
|
566
|
+
});
|
|
567
|
+
it("attaches census growth and rss/external/arrayBuffers deltas to the recycle record", () => {
|
|
568
|
+
let entries = 100;
|
|
487
569
|
const census = new HeapCensus();
|
|
488
|
-
census.register("
|
|
489
|
-
const {
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
570
|
+
census.register("watcherJournalStores", () => entries);
|
|
571
|
+
const g = governor({ census });
|
|
572
|
+
// Prime the growth baseline at session start (entries=100, rss=0), exactly as
|
|
573
|
+
// startRunnerHeapSampling does before the first interval.
|
|
574
|
+
g.controller.primeBaseline();
|
|
575
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS)).toBeNull(); // anchor
|
|
576
|
+
entries = 900;
|
|
577
|
+
g.state.rss = 1_500 * MB;
|
|
578
|
+
g.state.external = 40 * MB;
|
|
579
|
+
g.state.arrayBuffers = 12 * MB;
|
|
580
|
+
const recycle = g.observeAt(0.7, PAST_FLOOR_MS + 121_000);
|
|
581
|
+
expect(recycle).not.toBeNull();
|
|
582
|
+
expect(recycle?.topSites[0]).toMatchObject({ site: "watcherJournalStores", growth: 800 });
|
|
583
|
+
expect(recycle?.rssGrowthBytes).toBe(1_500 * MB);
|
|
584
|
+
expect(recycle?.externalGrowthBytes).toBe(40 * MB);
|
|
585
|
+
expect(recycle?.arrayBuffersGrowthBytes).toBe(12 * MB);
|
|
586
|
+
expect(recycle?.censusTotal).toBe(900);
|
|
587
|
+
});
|
|
588
|
+
});
|
|
589
|
+
describe("RunnerHeapTelemetry — emergency backstop", () => {
|
|
590
|
+
it("recycles on a single instantaneous sample at/above the emergency fraction, even below the uptime floor", () => {
|
|
591
|
+
const g = governor();
|
|
592
|
+
// Well below the uptime floor, but a single instantaneous sample above the
|
|
593
|
+
// 0.90 emergency line: at that reading V8 is minutes from aborting, recycle now.
|
|
594
|
+
const { recycle } = g.sampleAt(0.92, 30_000);
|
|
595
|
+
expect(recycle).not.toBeNull();
|
|
596
|
+
expect(recycle?.trigger).toBe("emergency");
|
|
597
|
+
expect(recycle?.observedSustainMs).toBe(0);
|
|
598
|
+
});
|
|
599
|
+
it("does not fire the emergency backstop below the emergency fraction", () => {
|
|
600
|
+
const g = governor();
|
|
601
|
+
// 0.89 instantaneous, sustained across many samples below the floor: the
|
|
602
|
+
// emergency arm never fires (and the live path is off — no GC observations).
|
|
603
|
+
let t = 30_000;
|
|
604
|
+
for (let i = 0; i < 20; i += 1) {
|
|
605
|
+
expect(g.sampleAt(0.89, t).recycle).toBeNull();
|
|
606
|
+
t += 60_000;
|
|
607
|
+
}
|
|
608
|
+
});
|
|
609
|
+
it("fires the emergency path even when a post-GC observation is already abort-adjacent (bypassing the window)", () => {
|
|
610
|
+
const g = governor();
|
|
611
|
+
// A post-GC live reading already at 0.95 recycles immediately, without the
|
|
612
|
+
// sustain window and without a prior anchor.
|
|
613
|
+
const recycle = g.observeAt(0.95, 20_000);
|
|
614
|
+
expect(recycle).not.toBeNull();
|
|
615
|
+
expect(recycle?.trigger).toBe("emergency");
|
|
616
|
+
});
|
|
617
|
+
it("evaluate() is a loop-top backstop that runs the emergency check off the timer path", () => {
|
|
618
|
+
const g = governor();
|
|
619
|
+
// The interval sampler never ticked, but a loop-top evaluate() catches an
|
|
620
|
+
// abort-adjacent instantaneous reading — the starved-timer safety net.
|
|
621
|
+
expect(g.evaluateAt(0.6, 30_000)).toBeNull(); // below emergency, no window seen
|
|
622
|
+
const recycle = g.evaluateAt(0.92, 60_000);
|
|
623
|
+
expect(recycle).not.toBeNull();
|
|
624
|
+
expect(recycle?.trigger).toBe("emergency");
|
|
625
|
+
});
|
|
626
|
+
});
|
|
627
|
+
describe("RunnerHeapTelemetry — recycle governor invariants", () => {
|
|
628
|
+
it("emits at most one recycle verdict per session, across every path", () => {
|
|
629
|
+
const g = governor();
|
|
630
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS)).toBeNull();
|
|
631
|
+
const first = g.observeAt(0.7, PAST_FLOOR_MS + 121_000);
|
|
632
|
+
expect(first).not.toBeNull();
|
|
633
|
+
// No second verdict from any path.
|
|
634
|
+
expect(g.observeAt(0.95, PAST_FLOOR_MS + 200_000)).toBeNull();
|
|
635
|
+
expect(g.sampleAt(0.99, PAST_FLOOR_MS + 260_000).recycle).toBeNull();
|
|
636
|
+
expect(g.evaluateAt(0.99, PAST_FLOOR_MS + 320_000)).toBeNull();
|
|
637
|
+
});
|
|
638
|
+
it("the kill switch disables every recycle path; pressure still behaves as base", () => {
|
|
639
|
+
const g = governor({ recycleEnabled: false });
|
|
501
640
|
let pressures = 0;
|
|
641
|
+
let recycles = 0;
|
|
642
|
+
let t = PAST_FLOOR_MS;
|
|
502
643
|
for (let i = 0; i < 20; i += 1) {
|
|
503
|
-
const { pressure, recycle } =
|
|
504
|
-
if (recycle)
|
|
505
|
-
recycles += 1;
|
|
644
|
+
const { pressure, recycle } = g.sampleAt(0.99, t); // far above any fraction
|
|
506
645
|
if (pressure)
|
|
507
646
|
pressures += 1;
|
|
647
|
+
if (recycle)
|
|
648
|
+
recycles += 1;
|
|
649
|
+
if (g.observeAt(0.99, t))
|
|
650
|
+
recycles += 1;
|
|
651
|
+
if (g.evaluateAt(0.99, t))
|
|
652
|
+
recycles += 1;
|
|
653
|
+
t += 60_000;
|
|
508
654
|
}
|
|
509
|
-
expect(recycles).toBe(0); // governor off — never recycles
|
|
655
|
+
expect(recycles).toBe(0); // governor off — never recycles on ANY path
|
|
510
656
|
expect(pressures).toBe(1); // pressure record still fires once per crossing
|
|
511
657
|
});
|
|
512
658
|
it("a recycle record and stderr line carry only fixed tokens and integers — never a runner byte", () => {
|
|
513
|
-
// A census whose backing structure is keyed by REAL-looking absolute paths,
|
|
514
|
-
// a company slug, and a filename. Only its numeric COUNT may ever be emitted.
|
|
515
659
|
const SECRET_PATH = "/Users/hassaan/hq/companies/acme-corp/vault/report.md";
|
|
516
660
|
const SLUG = "acme-corp";
|
|
517
661
|
const FILENAME = "report.md";
|
|
@@ -521,12 +665,9 @@ describe("RunnerHeapTelemetry — recycle governor", () => {
|
|
|
521
665
|
backing.set(FILENAME, 3);
|
|
522
666
|
const census = new HeapCensus();
|
|
523
667
|
census.register("watcherJournalStores", () => backing.size);
|
|
524
|
-
census.register("
|
|
525
|
-
const
|
|
526
|
-
|
|
527
|
-
usedBytes: Math.round(2048 * MB * 0.95),
|
|
528
|
-
census,
|
|
529
|
-
});
|
|
668
|
+
census.register("publishedDigestGenerations", () => backing.size);
|
|
669
|
+
const g = governor({ census });
|
|
670
|
+
const recycle = g.sampleAt(0.95, PAST_FLOOR_MS).recycle; // emergency path
|
|
530
671
|
expect(recycle).not.toBeNull();
|
|
531
672
|
const line = heapRecycleStderrLine(recycle);
|
|
532
673
|
const json = JSON.stringify(recycle);
|
|
@@ -534,12 +675,9 @@ describe("RunnerHeapTelemetry — recycle governor", () => {
|
|
|
534
675
|
expect(line).not.toContain(secret);
|
|
535
676
|
expect(json).not.toContain(secret);
|
|
536
677
|
}
|
|
537
|
-
// The FIXED census tokens are allowed; the runner bytes are not.
|
|
538
678
|
expect(json).toContain("watcherJournalStores");
|
|
539
679
|
});
|
|
540
680
|
it("the recycle stderr line matches NONE of hq-desktop-app's fatal stderr markers", () => {
|
|
541
|
-
// The exact marker vocabulary of classify_runner_fatal_class /
|
|
542
|
-
// is_disk_exhaustion_failure in hq-desktop-app crates/hq-desktop-core.
|
|
543
681
|
const DESKTOP_FATAL_MARKERS = [
|
|
544
682
|
"javascript heap out of memory",
|
|
545
683
|
"ineffective mark-compacts",
|
|
@@ -556,53 +694,199 @@ describe("RunnerHeapTelemetry — recycle governor", () => {
|
|
|
556
694
|
"command not found",
|
|
557
695
|
];
|
|
558
696
|
const census = new HeapCensus();
|
|
559
|
-
// Register the real production census tokens so the `top=` list is realistic.
|
|
560
697
|
for (const token of [
|
|
561
698
|
"pendingAppliedUpserts",
|
|
562
699
|
"watcherJournalStores",
|
|
563
|
-
"
|
|
700
|
+
"publishedDigestGenerations",
|
|
564
701
|
"knownKinds",
|
|
565
702
|
"deleteSnapshotEpochs",
|
|
566
703
|
]) {
|
|
567
704
|
census.register(token, () => 7);
|
|
568
705
|
}
|
|
569
|
-
const
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
census,
|
|
573
|
-
});
|
|
706
|
+
const g = governor({ census });
|
|
707
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS)).toBeNull();
|
|
708
|
+
const recycle = g.observeAt(0.7, PAST_FLOOR_MS + 121_000);
|
|
574
709
|
const line = heapRecycleStderrLine(recycle);
|
|
575
710
|
const lowered = line.toLowerCase();
|
|
576
711
|
for (const marker of DESKTOP_FATAL_MARKERS) {
|
|
577
712
|
expect(lowered).not.toContain(marker);
|
|
578
713
|
}
|
|
579
|
-
// NodeCheckAbort keys on a leading '#'; the recycle line never starts with one.
|
|
580
714
|
expect(line.trimStart().startsWith("#")).toBe(false);
|
|
581
|
-
// Sanity: the line still carries its own recognizable, benign shape.
|
|
582
715
|
expect(line).toContain("hq-sync-runner: heap-recycle");
|
|
583
|
-
expect(line).toContain("
|
|
716
|
+
expect(line).toContain("trigger=live");
|
|
717
|
+
expect(line).toContain("sustain_ms=");
|
|
718
|
+
});
|
|
719
|
+
});
|
|
720
|
+
describe("RunnerHeapTelemetry — recorded production series replay (regression reopen)", () => {
|
|
721
|
+
// The aborting generation's own recorded telemetry (person#prs_01KQ2TZQMA...,
|
|
722
|
+
// 2026-09-01 18:57..19:48Z). heap_size_limit was 2348810240 B; the SIGABRT
|
|
723
|
+
// landed at uptime 55m16s = 3_316_000 ms.
|
|
724
|
+
const RECORDED_LIMIT_BYTES = 2_348_810_240;
|
|
725
|
+
const ABORT_UPTIME_MS = 3_316_000;
|
|
726
|
+
// The recorded per-sample INSTANTANEOUS used/limit fractions. Max is 0.741 —
|
|
727
|
+
// it never reaches the old 0.78 rule and never two-in-a-row, and never the
|
|
728
|
+
// 0.90 emergency line, so the instantaneous path is inert (the base failure).
|
|
729
|
+
const INSTANTANEOUS = [
|
|
730
|
+
0.072, 0.106, 0.109, 0.458, 0.546, 0.251, 0.592, 0.564, 0.559, 0.427, 0.457,
|
|
731
|
+
0.564, 0.397, 0.584, 0.438, 0.523, 0.64, 0.548, 0.66, 0.664, 0.512, 0.513,
|
|
732
|
+
0.69, 0.703, 0.741,
|
|
733
|
+
];
|
|
734
|
+
it("base instantaneous logic is inert: no sample-path recycle across the entire recorded series", () => {
|
|
735
|
+
const g = governor({ limitBytes: RECORDED_LIMIT_BYTES });
|
|
736
|
+
let recycles = 0;
|
|
737
|
+
let t = 4.24 * 60_000;
|
|
738
|
+
for (const f of INSTANTANEOUS) {
|
|
739
|
+
if (g.sampleAt(f, t).recycle)
|
|
740
|
+
recycles += 1;
|
|
741
|
+
t += 2 * 60_000;
|
|
742
|
+
}
|
|
743
|
+
expect(recycles).toBe(0); // the exact inertness the regression exhibited
|
|
744
|
+
});
|
|
745
|
+
it("GC-anchored logic produces exactly one live recycle before the recorded 55m16s abort", () => {
|
|
746
|
+
// The implied POST-GC live floor: monotonic, crossing 0.60 between ~36.7 and
|
|
747
|
+
// ~39 min (the lower envelope of the instantaneous series). One major-GC
|
|
748
|
+
// observation per minute.
|
|
749
|
+
const LIVE_FLOOR = [
|
|
750
|
+
{ uptimeMin: 4, fraction: 0.07 },
|
|
751
|
+
{ uptimeMin: 12, fraction: 0.11 },
|
|
752
|
+
{ uptimeMin: 20, fraction: 0.25 },
|
|
753
|
+
{ uptimeMin: 28, fraction: 0.42 },
|
|
754
|
+
{ uptimeMin: 34, fraction: 0.51 },
|
|
755
|
+
{ uptimeMin: 37, fraction: 0.6 }, // crosses the floor here -> anchor
|
|
756
|
+
{ uptimeMin: 39, fraction: 0.63 }, // >= 120 s later -> recycle
|
|
757
|
+
{ uptimeMin: 46, fraction: 0.69 },
|
|
758
|
+
{ uptimeMin: 52, fraction: 0.74 },
|
|
759
|
+
];
|
|
760
|
+
const g = governor({ limitBytes: RECORDED_LIMIT_BYTES });
|
|
761
|
+
const verdicts = [];
|
|
762
|
+
for (const point of LIVE_FLOOR) {
|
|
763
|
+
const recycle = g.observeAt(point.fraction, point.uptimeMin * 60_000);
|
|
764
|
+
if (recycle)
|
|
765
|
+
verdicts.push(recycle);
|
|
766
|
+
}
|
|
767
|
+
expect(verdicts).toHaveLength(1);
|
|
768
|
+
expect(verdicts[0].trigger).toBe("live");
|
|
769
|
+
expect(verdicts[0].uptimeMs).toBeLessThan(ABORT_UPTIME_MS); // wins the race to V8
|
|
770
|
+
expect(verdicts[0].uptimeMs).toBeGreaterThanOrEqual(DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS);
|
|
771
|
+
// ~16 minutes of headroom before the recorded abort.
|
|
772
|
+
expect(ABORT_UPTIME_MS - verdicts[0].uptimeMs).toBeGreaterThan(10 * 60_000);
|
|
773
|
+
});
|
|
774
|
+
});
|
|
775
|
+
describe("ingest key contract — every emitted property key survives hq-pro redactProperties", () => {
|
|
776
|
+
// hq-pro's raw-telemetry-envelope ingest keeps a numeric property ONLY when its
|
|
777
|
+
// key matches this shape, ends in _count/_bytes/_ms, and contains none of these
|
|
778
|
+
// denied substrings. Quoted here as literals: a future key that violates the
|
|
779
|
+
// contract (the exact defect that dropped every census field to date) fails.
|
|
780
|
+
const KEY_SHAPE = /^[A-Za-z][A-Za-z0-9_]{0,63}$/;
|
|
781
|
+
const DENY_PARTS = [
|
|
782
|
+
"authorization",
|
|
783
|
+
"body",
|
|
784
|
+
"content",
|
|
785
|
+
"cookie",
|
|
786
|
+
"email",
|
|
787
|
+
"file",
|
|
788
|
+
"message",
|
|
789
|
+
"password",
|
|
790
|
+
"path",
|
|
791
|
+
"prompt",
|
|
792
|
+
"query",
|
|
793
|
+
"secret",
|
|
794
|
+
"text",
|
|
795
|
+
"token",
|
|
796
|
+
"transcript",
|
|
797
|
+
];
|
|
798
|
+
// Every FIXED census token the production runner registers (watch-loop ownMaps
|
|
799
|
+
// + the decoupled emitter/watcher tokens). If a new token is added, add it here.
|
|
800
|
+
const PRODUCTION_CENSUS_TOKENS = [
|
|
801
|
+
"pendingAppliedUpserts",
|
|
802
|
+
"pendingWatcherEntries",
|
|
803
|
+
"pendingWatcherChanges",
|
|
804
|
+
"pendingWatcherDroppedRouteHints",
|
|
805
|
+
"quarantinedDeleteRoots",
|
|
806
|
+
"watcherJournalStores",
|
|
807
|
+
"watcherJournalRows",
|
|
808
|
+
"targetedJournalSessions",
|
|
809
|
+
"targetedJournalRows",
|
|
810
|
+
"cachedAreaJournalMigrations",
|
|
811
|
+
"pendingSlowPasses",
|
|
812
|
+
"pendingFastPasses",
|
|
813
|
+
"lastPublishedDigests",
|
|
814
|
+
"publishedDigestGenerations",
|
|
815
|
+
"deferredPublishes",
|
|
816
|
+
"durablePublishedHashRetries",
|
|
817
|
+
"initialPublishedDigests",
|
|
818
|
+
"knownKinds",
|
|
819
|
+
"deleteSnapshotEpochs",
|
|
820
|
+
"ignoreFilterMemo",
|
|
821
|
+
];
|
|
822
|
+
function assertIngestSafe(props) {
|
|
823
|
+
for (const key of Object.keys(props)) {
|
|
824
|
+
const lower = key.toLowerCase();
|
|
825
|
+
expect(KEY_SHAPE.test(key), `key shape: ${key}`).toBe(true);
|
|
826
|
+
expect(lower.endsWith("count") || lower.endsWith("bytes") || lower.endsWith("ms"), `key suffix: ${key}`).toBe(true);
|
|
827
|
+
for (const part of DENY_PARTS) {
|
|
828
|
+
expect(lower.includes(part), `deny part "${part}" in ${key}`).toBe(false);
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
function fullCensus() {
|
|
833
|
+
const census = new HeapCensus();
|
|
834
|
+
for (const token of PRODUCTION_CENSUS_TOKENS)
|
|
835
|
+
census.register(token, () => 7);
|
|
836
|
+
return census;
|
|
837
|
+
}
|
|
838
|
+
it("heapSampleCloudProperties keys are all ingest-safe", () => {
|
|
839
|
+
const { readers } = mutableReaders();
|
|
840
|
+
const sample = sampleRunnerHeap({ readers, census: fullCensus(), sessionStartedAtMs: 0 });
|
|
841
|
+
assertIngestSafe(heapSampleCloudProperties(sample));
|
|
842
|
+
});
|
|
843
|
+
it("heapPressureCloudProperties keys are all ingest-safe", () => {
|
|
844
|
+
const census = fullCensus();
|
|
845
|
+
const { state, readers } = mutableReaders();
|
|
846
|
+
state.limit = 2048 * MB;
|
|
847
|
+
state.used = Math.round(2048 * MB * 0.8);
|
|
848
|
+
const controller = new RunnerHeapTelemetry({
|
|
849
|
+
readers,
|
|
850
|
+
census,
|
|
851
|
+
sessionStartedAtMs: 0,
|
|
852
|
+
pressureThresholdFraction: 0.75,
|
|
853
|
+
});
|
|
854
|
+
const { pressure } = controller.sample();
|
|
855
|
+
expect(pressure).not.toBeNull();
|
|
856
|
+
assertIngestSafe(heapPressureCloudProperties(pressure));
|
|
857
|
+
});
|
|
858
|
+
it("heapRecycleCloudProperties keys are all ingest-safe (both triggers)", () => {
|
|
859
|
+
const emergency = governor({ census: fullCensus() }).sampleAt(0.95, PAST_FLOOR_MS).recycle;
|
|
860
|
+
expect(emergency).not.toBeNull();
|
|
861
|
+
assertIngestSafe(heapRecycleCloudProperties(emergency));
|
|
862
|
+
const g = governor({ census: fullCensus() });
|
|
863
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS)).toBeNull();
|
|
864
|
+
const live = g.observeAt(0.7, PAST_FLOOR_MS + 121_000);
|
|
865
|
+
expect(live).not.toBeNull();
|
|
866
|
+
assertIngestSafe(heapRecycleCloudProperties(live));
|
|
584
867
|
});
|
|
585
868
|
});
|
|
586
|
-
describe("startRunnerHeapSampling — recycle callback", () => {
|
|
587
|
-
it("drives the governor to a recycle and
|
|
869
|
+
describe("startRunnerHeapSampling — GC observer + recycle callback", () => {
|
|
870
|
+
it("drives the governor to a LIVE recycle via the injected GC observer and swallows a throwing onRecycle", () => {
|
|
588
871
|
const census = new HeapCensus();
|
|
589
872
|
census.register("x", () => 1);
|
|
590
873
|
const { state, readers } = mutableReaders();
|
|
591
|
-
state.limit =
|
|
592
|
-
state.used = 950; // 0.95 -> above both the 0.75 pressure and 0.78 recycle fractions
|
|
593
|
-
state.now = ONE_HOUR_MS; // past the floor
|
|
874
|
+
state.limit = 2048 * MB;
|
|
594
875
|
const fake = fakeScheduler();
|
|
876
|
+
const gc = fakeGcObserver();
|
|
595
877
|
const recycles = [];
|
|
596
|
-
startRunnerHeapSampling({
|
|
878
|
+
const handle = startRunnerHeapSampling({
|
|
597
879
|
readers,
|
|
598
880
|
census,
|
|
599
881
|
sessionStartedAtMs: 0,
|
|
600
882
|
pressureThresholdFraction: 0.75,
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
883
|
+
recycleLiveFraction: 0.6,
|
|
884
|
+
recycleSustainWindowMs: 120_000,
|
|
885
|
+
recycleEmergencyFraction: 0.9,
|
|
886
|
+
recycleMinUptimeMs: DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS,
|
|
604
887
|
intervalMs: 1,
|
|
605
888
|
scheduler: fake.scheduler,
|
|
889
|
+
gcObserver: gc.observer,
|
|
606
890
|
onSample: () => { },
|
|
607
891
|
onPressure: () => { },
|
|
608
892
|
onRecycle: (record) => {
|
|
@@ -611,15 +895,51 @@ describe("startRunnerHeapSampling — recycle callback", () => {
|
|
|
611
895
|
},
|
|
612
896
|
log: () => { },
|
|
613
897
|
});
|
|
614
|
-
|
|
615
|
-
|
|
898
|
+
expect(gc.started()).toBe(true);
|
|
899
|
+
// First major GC at/above the live floor: anchors, no verdict.
|
|
900
|
+
state.used = Math.round(state.limit * 0.7);
|
|
901
|
+
state.now = PAST_FLOOR_MS;
|
|
902
|
+
expect(() => gc.fire()).not.toThrow();
|
|
616
903
|
expect(recycles).toHaveLength(0);
|
|
617
|
-
//
|
|
618
|
-
|
|
904
|
+
// A window later: the second major GC produces the live recycle (throw swallowed).
|
|
905
|
+
state.now = PAST_FLOOR_MS + 121_000;
|
|
906
|
+
expect(() => gc.fire()).not.toThrow();
|
|
619
907
|
expect(recycles).toHaveLength(1);
|
|
620
|
-
|
|
621
|
-
|
|
908
|
+
expect(recycles[0].trigger).toBe("live");
|
|
909
|
+
// At most once per session, even as GCs continue.
|
|
910
|
+
state.now = PAST_FLOOR_MS + 300_000;
|
|
911
|
+
expect(() => gc.fire()).not.toThrow();
|
|
912
|
+
expect(recycles).toHaveLength(1);
|
|
913
|
+
handle.stop();
|
|
914
|
+
expect(gc.stopped()).toBe(true);
|
|
915
|
+
});
|
|
916
|
+
it("the loop-top evaluate() fires the emergency backstop when the interval timer never ticks", () => {
|
|
917
|
+
const census = new HeapCensus();
|
|
918
|
+
census.register("x", () => 1);
|
|
919
|
+
const { state, readers } = mutableReaders();
|
|
920
|
+
state.limit = 2048 * MB;
|
|
921
|
+
const fake = fakeScheduler();
|
|
922
|
+
const recycles = [];
|
|
923
|
+
const handle = startRunnerHeapSampling({
|
|
924
|
+
readers,
|
|
925
|
+
census,
|
|
926
|
+
sessionStartedAtMs: 0,
|
|
927
|
+
pressureThresholdFraction: 0.75,
|
|
928
|
+
recycleEmergencyFraction: 0.9,
|
|
929
|
+
intervalMs: 60_000,
|
|
930
|
+
scheduler: fake.scheduler,
|
|
931
|
+
onSample: () => { },
|
|
932
|
+
onPressure: () => { },
|
|
933
|
+
onRecycle: (record) => recycles.push(record),
|
|
934
|
+
log: () => { },
|
|
935
|
+
});
|
|
936
|
+
// The timer never ticked. An abort-adjacent instantaneous heap: the loop-top
|
|
937
|
+
// evaluate() catches it anyway.
|
|
938
|
+
state.used = Math.round(state.limit * 0.93);
|
|
939
|
+
state.now = 30_000;
|
|
940
|
+
handle.evaluate();
|
|
622
941
|
expect(recycles).toHaveLength(1);
|
|
942
|
+
expect(recycles[0].trigger).toBe("emergency");
|
|
623
943
|
});
|
|
624
944
|
it("produces no recycle when the governor is disabled by option", () => {
|
|
625
945
|
const census = new HeapCensus();
|
|
@@ -627,10 +947,11 @@ describe("startRunnerHeapSampling — recycle callback", () => {
|
|
|
627
947
|
const { state, readers } = mutableReaders();
|
|
628
948
|
state.limit = 1000;
|
|
629
949
|
state.used = 999;
|
|
630
|
-
state.now =
|
|
950
|
+
state.now = PAST_FLOOR_MS;
|
|
631
951
|
const fake = fakeScheduler();
|
|
952
|
+
const gc = fakeGcObserver();
|
|
632
953
|
const recycles = [];
|
|
633
|
-
startRunnerHeapSampling({
|
|
954
|
+
const handle = startRunnerHeapSampling({
|
|
634
955
|
readers,
|
|
635
956
|
census,
|
|
636
957
|
sessionStartedAtMs: 0,
|
|
@@ -639,16 +960,62 @@ describe("startRunnerHeapSampling — recycle callback", () => {
|
|
|
639
960
|
recycleMinUptimeMs: 0,
|
|
640
961
|
intervalMs: 1,
|
|
641
962
|
scheduler: fake.scheduler,
|
|
963
|
+
gcObserver: gc.observer,
|
|
642
964
|
onSample: () => { },
|
|
643
965
|
onPressure: () => { },
|
|
644
966
|
onRecycle: (record) => recycles.push(record),
|
|
645
967
|
log: () => { },
|
|
646
968
|
});
|
|
647
|
-
for (let i = 0; i < 10; i += 1)
|
|
969
|
+
for (let i = 0; i < 10; i += 1) {
|
|
648
970
|
fake.tick();
|
|
971
|
+
gc.fire();
|
|
972
|
+
handle.evaluate();
|
|
973
|
+
}
|
|
649
974
|
expect(recycles).toHaveLength(0);
|
|
650
975
|
});
|
|
651
976
|
});
|
|
977
|
+
describe("isFreshGcReading", () => {
|
|
978
|
+
it("accepts a promptly-captured reading and rejects a stale (blocked-loop) one", () => {
|
|
979
|
+
expect(isFreshGcReading(1000, 1000)).toBe(true); // captured at the GC end
|
|
980
|
+
expect(isFreshGcReading(1000, 1000 + MAX_FRESH_GC_READING_MS)).toBe(true); // exactly at the bound
|
|
981
|
+
expect(isFreshGcReading(1000, 1000 + MAX_FRESH_GC_READING_MS + 1)).toBe(false); // just over
|
|
982
|
+
expect(isFreshGcReading(1000, 5000)).toBe(false); // blocked loop: 4 s stale reading
|
|
983
|
+
expect(isFreshGcReading(1000, 900)).toBe(true); // reading at/just before the GC end
|
|
984
|
+
// Missing entry timing degrades to "not fresh" (skip), never a false accept.
|
|
985
|
+
expect(isFreshGcReading(Number.NaN, 1000)).toBe(false);
|
|
986
|
+
expect(isFreshGcReading(1000, Number.NaN)).toBe(false);
|
|
987
|
+
expect(isFreshGcReading(1000, 1400, 500)).toBe(true); // custom threshold honored
|
|
988
|
+
});
|
|
989
|
+
});
|
|
990
|
+
describe("nodeMajorGcObserver", () => {
|
|
991
|
+
it("constructs, observes major GCs, and disconnects without throwing", () => {
|
|
992
|
+
const observer = nodeMajorGcObserver();
|
|
993
|
+
let fires = 0;
|
|
994
|
+
expect(() => observer.start(() => (fires += 1))).not.toThrow();
|
|
995
|
+
// Force a real major GC if the runtime exposes it; otherwise this is a no-op
|
|
996
|
+
// and the observer simply never fires — both are acceptable (degrade to
|
|
997
|
+
// silence). The contract under test is that neither start nor stop throws.
|
|
998
|
+
const globalGc = globalThis.gc;
|
|
999
|
+
if (typeof globalGc === "function") {
|
|
1000
|
+
try {
|
|
1001
|
+
globalGc({ type: "major" });
|
|
1002
|
+
}
|
|
1003
|
+
catch {
|
|
1004
|
+
globalGc();
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
expect(() => observer.stop()).not.toThrow();
|
|
1008
|
+
expect(fires).toBeGreaterThanOrEqual(0);
|
|
1009
|
+
});
|
|
1010
|
+
it("start() twice and stop() before start() never throw (degrade to silence)", () => {
|
|
1011
|
+
const observer = nodeMajorGcObserver();
|
|
1012
|
+
expect(() => observer.stop()).not.toThrow(); // stop before start
|
|
1013
|
+
expect(() => observer.start(() => { })).not.toThrow();
|
|
1014
|
+
expect(() => observer.start(() => { })).not.toThrow(); // re-start
|
|
1015
|
+
expect(() => observer.stop()).not.toThrow();
|
|
1016
|
+
expect(() => observer.stop()).not.toThrow(); // double stop
|
|
1017
|
+
});
|
|
1018
|
+
});
|
|
652
1019
|
describe("resolveHeapRecycleEnabled", () => {
|
|
653
1020
|
it("is ON by default and disabled only by an explicit whole-string off value", () => {
|
|
654
1021
|
expect(resolveHeapRecycleEnabled(undefined)).toBe(true);
|
|
@@ -663,35 +1030,428 @@ describe("resolveHeapRecycleEnabled", () => {
|
|
|
663
1030
|
expect(resolveHeapRecycleEnabled("disabled")).toBe(true);
|
|
664
1031
|
});
|
|
665
1032
|
});
|
|
666
|
-
describe("
|
|
667
|
-
it("parses a valid fraction and falls back to 0.
|
|
668
|
-
expect(
|
|
669
|
-
expect(
|
|
670
|
-
expect(
|
|
671
|
-
expect(
|
|
672
|
-
expect(
|
|
673
|
-
expect(
|
|
674
|
-
expect(
|
|
1033
|
+
describe("resolveHeapRecycleLiveFraction", () => {
|
|
1034
|
+
it("parses a valid fraction and falls back to 0.60 on garbage or out-of-range input", () => {
|
|
1035
|
+
expect(resolveHeapRecycleLiveFraction("0.5")).toBe(0.5);
|
|
1036
|
+
expect(resolveHeapRecycleLiveFraction("1")).toBe(1);
|
|
1037
|
+
expect(resolveHeapRecycleLiveFraction(undefined)).toBe(DEFAULT_HEAP_RECYCLE_LIVE_FRACTION);
|
|
1038
|
+
expect(resolveHeapRecycleLiveFraction("")).toBe(DEFAULT_HEAP_RECYCLE_LIVE_FRACTION);
|
|
1039
|
+
expect(resolveHeapRecycleLiveFraction("2")).toBe(DEFAULT_HEAP_RECYCLE_LIVE_FRACTION);
|
|
1040
|
+
expect(resolveHeapRecycleLiveFraction("-1")).toBe(DEFAULT_HEAP_RECYCLE_LIVE_FRACTION);
|
|
1041
|
+
expect(resolveHeapRecycleLiveFraction("0.6x")).toBe(DEFAULT_HEAP_RECYCLE_LIVE_FRACTION);
|
|
1042
|
+
});
|
|
1043
|
+
});
|
|
1044
|
+
describe("resolveHeapRecycleEmergencyFraction", () => {
|
|
1045
|
+
it("parses a valid fraction and falls back to 0.90 on garbage or out-of-range input", () => {
|
|
1046
|
+
expect(resolveHeapRecycleEmergencyFraction("0.95")).toBe(0.95);
|
|
1047
|
+
expect(resolveHeapRecycleEmergencyFraction("1")).toBe(1);
|
|
1048
|
+
expect(resolveHeapRecycleEmergencyFraction(undefined)).toBe(DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION);
|
|
1049
|
+
expect(resolveHeapRecycleEmergencyFraction("")).toBe(DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION);
|
|
1050
|
+
expect(resolveHeapRecycleEmergencyFraction("2")).toBe(DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION);
|
|
1051
|
+
expect(resolveHeapRecycleEmergencyFraction("0.9x")).toBe(DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION);
|
|
675
1052
|
});
|
|
676
1053
|
});
|
|
677
|
-
describe("
|
|
678
|
-
it("parses
|
|
679
|
-
expect(
|
|
680
|
-
expect(
|
|
681
|
-
expect(
|
|
682
|
-
expect(
|
|
683
|
-
expect(
|
|
684
|
-
expect(
|
|
1054
|
+
describe("resolveHeapRecycleSustainWindowMs", () => {
|
|
1055
|
+
it("parses whole ms >= 0 (0 removes the window) and rejects '2m' rather than its prefix", () => {
|
|
1056
|
+
expect(resolveHeapRecycleSustainWindowMs("120000")).toBe(120_000);
|
|
1057
|
+
expect(resolveHeapRecycleSustainWindowMs("0")).toBe(0); // 0 is a valid window
|
|
1058
|
+
expect(resolveHeapRecycleSustainWindowMs(undefined)).toBe(DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS);
|
|
1059
|
+
expect(resolveHeapRecycleSustainWindowMs("2m")).toBe(DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS);
|
|
1060
|
+
expect(resolveHeapRecycleSustainWindowMs("-1")).toBe(DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS);
|
|
1061
|
+
expect(resolveHeapRecycleSustainWindowMs("")).toBe(DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS);
|
|
685
1062
|
});
|
|
686
1063
|
});
|
|
687
1064
|
describe("resolveHeapRecycleMinUptimeMs", () => {
|
|
688
|
-
it("parses whole ms >= 0 (0 disables the floor) and rejects '
|
|
1065
|
+
it("parses whole ms >= 0 (0 disables the floor) and rejects '35m' rather than its prefix", () => {
|
|
689
1066
|
expect(resolveHeapRecycleMinUptimeMs("60000")).toBe(60_000);
|
|
690
1067
|
expect(resolveHeapRecycleMinUptimeMs("0")).toBe(0); // 0 is a valid floor
|
|
691
1068
|
expect(resolveHeapRecycleMinUptimeMs(undefined)).toBe(DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS);
|
|
692
|
-
expect(resolveHeapRecycleMinUptimeMs("
|
|
1069
|
+
expect(resolveHeapRecycleMinUptimeMs("35m")).toBe(DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS);
|
|
693
1070
|
expect(resolveHeapRecycleMinUptimeMs("-1")).toBe(DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS);
|
|
694
1071
|
expect(resolveHeapRecycleMinUptimeMs("")).toBe(DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS);
|
|
695
1072
|
});
|
|
696
1073
|
});
|
|
1074
|
+
// --- v2-layered governor: effective limit, pinned + hard-exit tiers ----------
|
|
1075
|
+
describe("RunnerHeapTelemetry — effective-limit re-denomination", () => {
|
|
1076
|
+
it("divides every fraction by min(declared, reported) so a declared ceiling makes the trigger real", () => {
|
|
1077
|
+
const declared = 2048 * MB;
|
|
1078
|
+
const reported = 2233 * MB; // Node-25 reported limit sits above the declared ceiling
|
|
1079
|
+
const withDeclared = governor({
|
|
1080
|
+
declaredCeilingBytes: declared,
|
|
1081
|
+
limitBytes: reported,
|
|
1082
|
+
minUptimeMs: 0,
|
|
1083
|
+
});
|
|
1084
|
+
// 0.91 of the DECLARED ceiling: 1863 MB. Over effective (declared) that is
|
|
1085
|
+
// 0.91 >= the 0.90 emergency line, so a single post-GC reading recycles.
|
|
1086
|
+
withDeclared.state.used = Math.round(declared * 0.91);
|
|
1087
|
+
withDeclared.state.now = 30_000;
|
|
1088
|
+
const armed = withDeclared.controller.observeMajorGc().recycle;
|
|
1089
|
+
expect(armed).not.toBeNull();
|
|
1090
|
+
expect(armed?.trigger).toBe("emergency");
|
|
1091
|
+
expect(armed?.effectiveLimitBytes).toBe(declared);
|
|
1092
|
+
expect(armed?.declaredCeilingBytes).toBe(declared);
|
|
1093
|
+
expect(armed?.heapLimitBytes).toBe(reported); // reported kept for provenance
|
|
1094
|
+
expect(armed?.usedFraction).toBeCloseTo(0.91, 2);
|
|
1095
|
+
});
|
|
1096
|
+
it("the SAME bytes never reach emergency when the reported limit is the denominator (no declared)", () => {
|
|
1097
|
+
const declared = 2048 * MB;
|
|
1098
|
+
const reported = 2233 * MB;
|
|
1099
|
+
const noDeclared = governor({ declaredCeilingBytes: null, limitBytes: reported, minUptimeMs: 0 });
|
|
1100
|
+
// 1863 MB is 0.834 of the reported limit — below the 0.90 emergency line, so
|
|
1101
|
+
// the post-GC path only anchors the live floor (no immediate recycle).
|
|
1102
|
+
noDeclared.state.used = Math.round(declared * 0.91);
|
|
1103
|
+
noDeclared.state.now = 30_000;
|
|
1104
|
+
expect(noDeclared.controller.observeMajorGc().recycle).toBeNull();
|
|
1105
|
+
});
|
|
1106
|
+
it("falls back to the reported limit (bit-identical shipped semantics) when no ceiling is declared", () => {
|
|
1107
|
+
const g = governor({ declaredCeilingBytes: null });
|
|
1108
|
+
expect(g.observeAt(0.62, PAST_FLOOR_MS)).toBeNull();
|
|
1109
|
+
const recycle = g.observeAt(0.66, PAST_FLOOR_MS + 121_000);
|
|
1110
|
+
expect(recycle?.trigger).toBe("live");
|
|
1111
|
+
expect(recycle?.effectiveLimitBytes).toBe(recycle?.heapLimitBytes);
|
|
1112
|
+
expect(recycle?.declaredCeilingBytes).toBe(0);
|
|
1113
|
+
});
|
|
1114
|
+
});
|
|
1115
|
+
describe("RunnerHeapTelemetry — pinned tier (T2, instantaneous sustained)", () => {
|
|
1116
|
+
it("arms on an instantaneous fraction held at/above the floor across the window, past the uptime floor", () => {
|
|
1117
|
+
const g = governor({ pinnedFraction: 0.84, pinnedSustainWindowMs: 20_000, minUptimeMs: 0 });
|
|
1118
|
+
expect(g.sampleAt(0.85, 0).recycle).toBeNull(); // anchor (count 1)
|
|
1119
|
+
expect(g.sampleAt(0.85, 10_000).recycle).toBeNull(); // 10 s < 20 s window
|
|
1120
|
+
const { recycle } = g.sampleAt(0.85, 21_000); // 21 s >= window, count >= 2
|
|
1121
|
+
expect(recycle).not.toBeNull();
|
|
1122
|
+
expect(recycle?.trigger).toBe("pinned");
|
|
1123
|
+
expect(recycle?.reason).toBe("cooperative");
|
|
1124
|
+
expect(recycle?.pinnedFraction).toBe(0.84);
|
|
1125
|
+
});
|
|
1126
|
+
it("resets its anchor on any single below-floor observation (a healthy sawtooth never sustains)", () => {
|
|
1127
|
+
const g = governor({ pinnedFraction: 0.84, pinnedSustainWindowMs: 20_000, minUptimeMs: 0 });
|
|
1128
|
+
expect(g.sampleAt(0.85, 0).recycle).toBeNull(); // anchor
|
|
1129
|
+
expect(g.sampleAt(0.80, 5_000).recycle).toBeNull(); // dip -> anchor reset
|
|
1130
|
+
expect(g.sampleAt(0.85, 25_000).recycle).toBeNull(); // re-anchor here (count 1)
|
|
1131
|
+
const { recycle } = g.sampleAt(0.85, 46_000); // 21 s past the re-anchor
|
|
1132
|
+
expect(recycle?.trigger).toBe("pinned");
|
|
1133
|
+
});
|
|
1134
|
+
it("honours the 35-minute uptime floor (unlike the emergency tier)", () => {
|
|
1135
|
+
const g = governor({ pinnedFraction: 0.84, pinnedSustainWindowMs: 0, minUptimeMs: 100_000 });
|
|
1136
|
+
expect(g.sampleAt(0.85, 0).recycle).toBeNull(); // anchor
|
|
1137
|
+
expect(g.sampleAt(0.85, 50_000).recycle).toBeNull(); // window satisfied but 50 s < 100 s floor
|
|
1138
|
+
const { recycle } = g.sampleAt(0.85, 120_000); // past the floor
|
|
1139
|
+
expect(recycle?.trigger).toBe("pinned");
|
|
1140
|
+
});
|
|
1141
|
+
it("never fires off the post-GC live path, and the live path never fires 'pinned'", () => {
|
|
1142
|
+
// Instantaneous readings drive pinned; post-GC readings drive live. They keep
|
|
1143
|
+
// separate anchors, so neither impersonates the other.
|
|
1144
|
+
const gPinned = governor({ pinnedFraction: 0.84, pinnedSustainWindowMs: 0, minUptimeMs: 0 });
|
|
1145
|
+
gPinned.sampleAt(0.85, 0); // instantaneous only
|
|
1146
|
+
expect(gPinned.sampleAt(0.85, 1_000).recycle?.trigger).toBe("pinned");
|
|
1147
|
+
const gLive = governor({ minUptimeMs: 0 });
|
|
1148
|
+
gLive.observeAt(0.65, 0);
|
|
1149
|
+
expect(gLive.observeAt(0.65, 121_000)?.trigger).toBe("live");
|
|
1150
|
+
});
|
|
1151
|
+
});
|
|
1152
|
+
describe("RunnerHeapTelemetry — hard-exit tiers (T4)", () => {
|
|
1153
|
+
it("hard_fraction: a single instantaneous observation at/above the hard line hard-exits with no prior latch", () => {
|
|
1154
|
+
const g = governor({ minUptimeMs: DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS });
|
|
1155
|
+
// 0.96 >= 0.955, well BELOW the uptime floor: the hard tier bypasses the floor
|
|
1156
|
+
// exactly like a SIGABRT would, and preempts the cooperative recycle.
|
|
1157
|
+
const { recycle, hardExit } = g.evaluateFullAt(0.96, 10_000);
|
|
1158
|
+
expect(recycle).toBeNull();
|
|
1159
|
+
expect(hardExit).not.toBeNull();
|
|
1160
|
+
expect(hardExit?.reason).toBe("hard_fraction");
|
|
1161
|
+
expect(hardExit?.trigger).toBe("emergency"); // no prior latch -> instantaneous family
|
|
1162
|
+
expect(hardExit?.hardExitFraction).toBe(DEFAULT_HEAP_HARD_EXIT_FRACTION);
|
|
1163
|
+
});
|
|
1164
|
+
it("latch_timeout: an unhonoured cooperative latch ages out by TIME into a hard exit", () => {
|
|
1165
|
+
const g = governor({ minUptimeMs: 0, latchHardExitMs: 60_000, latchHardExitObservations: 1_000_000 });
|
|
1166
|
+
const armed = g.evaluateFullAt(0.92, 0); // cooperative emergency latch
|
|
1167
|
+
expect(armed.recycle?.reason).toBe("cooperative");
|
|
1168
|
+
expect(armed.recycle?.trigger).toBe("emergency");
|
|
1169
|
+
expect(g.evaluateFullAt(0.92, 30_000).hardExit).toBeNull(); // 30 s < 60 s deadline
|
|
1170
|
+
const late = g.evaluateFullAt(0.92, 61_000); // 61 s >= deadline
|
|
1171
|
+
expect(late.recycle).toBeNull();
|
|
1172
|
+
expect(late.hardExit?.reason).toBe("latch_timeout");
|
|
1173
|
+
expect(late.hardExit?.trigger).toBe("emergency"); // inherits the latched tier
|
|
1174
|
+
expect(late.hardExit?.postLatchObservations).toBeGreaterThanOrEqual(1);
|
|
1175
|
+
});
|
|
1176
|
+
it("latch_timeout: ages out by observation COUNT before the time deadline", () => {
|
|
1177
|
+
const g = governor({ minUptimeMs: 0, latchHardExitMs: 10 * 60_000, latchHardExitObservations: 3 });
|
|
1178
|
+
g.evaluateFullAt(0.92, 0); // cooperative emergency latch
|
|
1179
|
+
expect(g.evaluateFullAt(0.7, 1_000).hardExit).toBeNull(); // post-latch obs 1 < 3
|
|
1180
|
+
expect(g.evaluateFullAt(0.7, 2_000).hardExit).toBeNull(); // post-latch obs 2 < 3
|
|
1181
|
+
const third = g.evaluateFullAt(0.7, 3_000); // post-latch obs 3 >= 3
|
|
1182
|
+
expect(third.hardExit?.reason).toBe("latch_timeout");
|
|
1183
|
+
});
|
|
1184
|
+
it("the latch-age arm fires from EVERY observation path (interval sample, loop-top, post-GC)", () => {
|
|
1185
|
+
const g = governor({ minUptimeMs: 0, latchHardExitMs: 0, latchHardExitObservations: 1_000_000 });
|
|
1186
|
+
// latchHardExitMs=0: the first post-latch observation from any path hard-exits.
|
|
1187
|
+
g.observeFullAt(0.92, 0); // post-GC emergency cooperative latch
|
|
1188
|
+
const viaSample = g.sampleAt(0.7, 1);
|
|
1189
|
+
expect(viaSample.hardExit?.reason).toBe("latch_timeout");
|
|
1190
|
+
});
|
|
1191
|
+
it("fires at most one hard exit per session and never a cooperative verdict afterwards", () => {
|
|
1192
|
+
const g = governor({ minUptimeMs: 0, latchHardExitMs: 0, latchHardExitObservations: 1_000_000 });
|
|
1193
|
+
g.evaluateFullAt(0.92, 0); // cooperative latch
|
|
1194
|
+
expect(g.evaluateFullAt(0.7, 1).hardExit?.reason).toBe("latch_timeout"); // first hard exit
|
|
1195
|
+
expect(g.evaluateFullAt(0.99, 2).hardExit).toBeNull(); // already hard-signalled
|
|
1196
|
+
expect(g.evaluateFullAt(0.99, 3).recycle).toBeNull();
|
|
1197
|
+
});
|
|
1198
|
+
});
|
|
1199
|
+
describe("RunnerHeapTelemetry — tier ordering and derivation pin (guard-constant policy)", () => {
|
|
1200
|
+
it("pins 0.60 < 0.84 < 0.90 < 0.955 and each against the observed abort band", () => {
|
|
1201
|
+
expect(DEFAULT_HEAP_RECYCLE_LIVE_FRACTION).toBe(0.6);
|
|
1202
|
+
expect(DEFAULT_HEAP_PINNED_FRACTION).toBe(0.84);
|
|
1203
|
+
expect(DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION).toBe(0.9);
|
|
1204
|
+
expect(DEFAULT_HEAP_HARD_EXIT_FRACTION).toBe(0.955);
|
|
1205
|
+
const order = [
|
|
1206
|
+
DEFAULT_HEAP_RECYCLE_LIVE_FRACTION,
|
|
1207
|
+
DEFAULT_HEAP_PINNED_FRACTION,
|
|
1208
|
+
DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION,
|
|
1209
|
+
DEFAULT_HEAP_HARD_EXIT_FRACTION,
|
|
1210
|
+
];
|
|
1211
|
+
for (let i = 1; i < order.length; i += 1)
|
|
1212
|
+
expect(order[i]).toBeGreaterThan(order[i - 1]);
|
|
1213
|
+
// Pinned sits above the max healthy instantaneous evidence (~0.816 of declared)
|
|
1214
|
+
// and below the legacy abort floor (1788/2048 = 0.873).
|
|
1215
|
+
expect(DEFAULT_HEAP_PINNED_FRACTION).toBeGreaterThan(0.816);
|
|
1216
|
+
expect(DEFAULT_HEAP_PINNED_FRACTION).toBeLessThan(1788 / 2048);
|
|
1217
|
+
// Emergency and hard-exit both sit strictly below the minimum post-#471
|
|
1218
|
+
// observed abort ratio (1995/2048 = 0.974), so both preempt the SIGABRT.
|
|
1219
|
+
expect(DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION).toBeLessThan(1995 / 2048);
|
|
1220
|
+
expect(DEFAULT_HEAP_HARD_EXIT_FRACTION).toBeLessThan(1995 / 2048);
|
|
1221
|
+
});
|
|
1222
|
+
});
|
|
1223
|
+
describe("RunnerHeapTelemetry — kill switch disables every tier (incl. hard)", () => {
|
|
1224
|
+
it("disables live, pinned, emergency, hard_fraction, AND latch_timeout at once", () => {
|
|
1225
|
+
const g = governor({ recycleEnabled: false, minUptimeMs: 0 });
|
|
1226
|
+
let t = 0;
|
|
1227
|
+
for (let i = 0; i < 20; i += 1) {
|
|
1228
|
+
// 0.99 would fire hard_fraction; 0.85 sustained would fire pinned — none do.
|
|
1229
|
+
const viaSample = g.sampleAt(0.99, t);
|
|
1230
|
+
expect(viaSample.recycle).toBeNull();
|
|
1231
|
+
expect(viaSample.hardExit).toBeNull();
|
|
1232
|
+
const viaEval = g.evaluateFullAt(0.99, t + 1);
|
|
1233
|
+
expect(viaEval.recycle).toBeNull();
|
|
1234
|
+
expect(viaEval.hardExit).toBeNull();
|
|
1235
|
+
const viaGc = g.observeFullAt(0.99, t + 2);
|
|
1236
|
+
expect(viaGc.recycle).toBeNull();
|
|
1237
|
+
expect(viaGc.hardExit).toBeNull();
|
|
1238
|
+
t += 60_000;
|
|
1239
|
+
}
|
|
1240
|
+
});
|
|
1241
|
+
});
|
|
1242
|
+
describe("startRunnerHeapSampling — work-driven checkpoint gate", () => {
|
|
1243
|
+
function pinnedHandle(overrides) {
|
|
1244
|
+
const census = new HeapCensus();
|
|
1245
|
+
census.register("x", () => 1);
|
|
1246
|
+
const { state, readers } = mutableReaders();
|
|
1247
|
+
state.limit = 2048 * MB;
|
|
1248
|
+
state.used = Math.round(state.limit * 0.85); // pinned range
|
|
1249
|
+
const fake = fakeScheduler();
|
|
1250
|
+
const recs = [];
|
|
1251
|
+
const handle = startRunnerHeapSampling({
|
|
1252
|
+
readers,
|
|
1253
|
+
census,
|
|
1254
|
+
sessionStartedAtMs: 0,
|
|
1255
|
+
pressureThresholdFraction: 0.99,
|
|
1256
|
+
recyclePinnedFraction: 0.84,
|
|
1257
|
+
recyclePinnedSustainWindowMs: 0, // any two gated observations suffice
|
|
1258
|
+
recycleMinUptimeMs: 0,
|
|
1259
|
+
intervalMs: 60_000,
|
|
1260
|
+
checkpointMs: overrides.checkpointMs,
|
|
1261
|
+
scheduler: fake.scheduler,
|
|
1262
|
+
onSample: () => { },
|
|
1263
|
+
onPressure: () => { },
|
|
1264
|
+
onRecycle: (record, counters) => recs.push({ record, counters }),
|
|
1265
|
+
log: () => { },
|
|
1266
|
+
});
|
|
1267
|
+
return { state, handle, recs };
|
|
1268
|
+
}
|
|
1269
|
+
it("observes only when the gate is open (>= checkpointMs since the last observation)", () => {
|
|
1270
|
+
const { state, handle, recs } = pinnedHandle({ checkpointMs: 100 });
|
|
1271
|
+
state.now = 0;
|
|
1272
|
+
handle.maybeObserve(); // first checkpoint: no prior observation -> observes (pinned count 1)
|
|
1273
|
+
state.now = 50;
|
|
1274
|
+
handle.maybeObserve(); // 50 < 100 -> below the gate, no observation
|
|
1275
|
+
state.now = 150;
|
|
1276
|
+
handle.maybeObserve(); // 150 >= 100 -> observes (pinned count 2 -> recycle)
|
|
1277
|
+
expect(recs).toHaveLength(1);
|
|
1278
|
+
expect(recs[0].record.trigger).toBe("pinned");
|
|
1279
|
+
// Exactly two checkpoint observations (the 50 ms one was gated out); the 60 s
|
|
1280
|
+
// interval timer never ticked, proving the feed was work-driven, not the timer.
|
|
1281
|
+
expect(recs[0].counters.heapCheckpointObservationsCount).toBe(2);
|
|
1282
|
+
expect(recs[0].counters.heapIntervalObservationsCount).toBe(0);
|
|
1283
|
+
expect(recs[0].counters.msSincePreviousObservationMs).toBe(150);
|
|
1284
|
+
});
|
|
1285
|
+
it("checkpointMs=0 disables the work-driven checkpoint but keeps the loop-top evaluate()", () => {
|
|
1286
|
+
const { state, handle, recs } = pinnedHandle({ checkpointMs: 0 });
|
|
1287
|
+
state.now = 0;
|
|
1288
|
+
handle.maybeObserve();
|
|
1289
|
+
state.now = 100_000;
|
|
1290
|
+
handle.maybeObserve(); // both disabled -> never observes
|
|
1291
|
+
expect(recs).toHaveLength(0);
|
|
1292
|
+
// The loop-top evaluate() is ungated and still works with checkpoints off.
|
|
1293
|
+
state.used = Math.round(2048 * MB * 0.96); // hard-fraction range
|
|
1294
|
+
// ... but this handle has no onHardExit, so route through a recycle instead:
|
|
1295
|
+
state.used = Math.round(2048 * MB * 0.92); // emergency cooperative
|
|
1296
|
+
state.now = 200_000;
|
|
1297
|
+
handle.evaluate();
|
|
1298
|
+
expect(recs).toHaveLength(1);
|
|
1299
|
+
expect(recs[0].record.trigger).toBe("emergency");
|
|
1300
|
+
});
|
|
1301
|
+
it("delivers a hard exit to onHardExit (never onRecycle) and preempts the cooperative verdict", () => {
|
|
1302
|
+
const census = new HeapCensus();
|
|
1303
|
+
census.register("x", () => 1);
|
|
1304
|
+
const { state, readers } = mutableReaders();
|
|
1305
|
+
state.limit = 2048 * MB;
|
|
1306
|
+
state.used = Math.round(state.limit * 0.96); // hard-fraction range
|
|
1307
|
+
const fake = fakeScheduler();
|
|
1308
|
+
const recycles = [];
|
|
1309
|
+
const hardExits = [];
|
|
1310
|
+
const handle = startRunnerHeapSampling({
|
|
1311
|
+
readers,
|
|
1312
|
+
census,
|
|
1313
|
+
sessionStartedAtMs: 0,
|
|
1314
|
+
pressureThresholdFraction: 0.99,
|
|
1315
|
+
recycleMinUptimeMs: 0,
|
|
1316
|
+
intervalMs: 60_000,
|
|
1317
|
+
checkpointMs: 10,
|
|
1318
|
+
scheduler: fake.scheduler,
|
|
1319
|
+
onSample: () => { },
|
|
1320
|
+
onPressure: () => { },
|
|
1321
|
+
onRecycle: (record) => recycles.push(record),
|
|
1322
|
+
onHardExit: (record) => hardExits.push(record),
|
|
1323
|
+
log: () => { },
|
|
1324
|
+
});
|
|
1325
|
+
state.now = 0;
|
|
1326
|
+
handle.maybeObserve();
|
|
1327
|
+
expect(hardExits).toHaveLength(1);
|
|
1328
|
+
expect(hardExits[0].reason).toBe("hard_fraction");
|
|
1329
|
+
expect(recycles).toHaveLength(0);
|
|
1330
|
+
handle.stop();
|
|
1331
|
+
});
|
|
1332
|
+
});
|
|
1333
|
+
describe("performHeapHardExit", () => {
|
|
1334
|
+
function hardFractionRecord() {
|
|
1335
|
+
const record = governor({ minUptimeMs: 0 }).sampleAt(0.96, 10_000).hardExit;
|
|
1336
|
+
expect(record).not.toBeNull();
|
|
1337
|
+
return record;
|
|
1338
|
+
}
|
|
1339
|
+
it("writes the NDJSON record to stdout, then the marker to stderr, then exits — in that order", () => {
|
|
1340
|
+
const calls = [];
|
|
1341
|
+
performHeapHardExit(hardFractionRecord(), {
|
|
1342
|
+
writeStdout: (line) => calls.push(`out:${line}`),
|
|
1343
|
+
writeStderr: (line) => calls.push(`err:${line}`),
|
|
1344
|
+
exit: (code) => calls.push(`exit:${code}`),
|
|
1345
|
+
}, 1);
|
|
1346
|
+
expect(calls).toHaveLength(3);
|
|
1347
|
+
expect(calls[0]).toContain("out:");
|
|
1348
|
+
expect(calls[0]).toContain('"type":"heap-recycle"');
|
|
1349
|
+
expect(calls[0]).toContain('"reason":"hard_fraction"');
|
|
1350
|
+
expect(calls[1]).toContain("err:");
|
|
1351
|
+
expect(calls[1]).toContain("hq-sync-runner: heap-recycle");
|
|
1352
|
+
expect(calls[2]).toBe("exit:1");
|
|
1353
|
+
});
|
|
1354
|
+
it("still exits even if BOTH stream writes throw (a closed/blocked pipe)", () => {
|
|
1355
|
+
let exited = -1;
|
|
1356
|
+
performHeapHardExit(hardFractionRecord(), {
|
|
1357
|
+
writeStdout: () => {
|
|
1358
|
+
throw new Error("EPIPE");
|
|
1359
|
+
},
|
|
1360
|
+
writeStderr: () => {
|
|
1361
|
+
throw new Error("EPIPE");
|
|
1362
|
+
},
|
|
1363
|
+
exit: (code) => {
|
|
1364
|
+
exited = code;
|
|
1365
|
+
},
|
|
1366
|
+
}, 1);
|
|
1367
|
+
expect(exited).toBe(1);
|
|
1368
|
+
});
|
|
1369
|
+
it("the hard-exit stderr marker matches NONE of hq-desktop-app's fatal classifiers (both hard reasons)", () => {
|
|
1370
|
+
const DESKTOP_FATAL_MARKERS = [
|
|
1371
|
+
"javascript heap out of memory",
|
|
1372
|
+
"ineffective mark-compacts",
|
|
1373
|
+
"fatal error",
|
|
1374
|
+
"uncaught exception",
|
|
1375
|
+
"unhandledrejection",
|
|
1376
|
+
"panicked at",
|
|
1377
|
+
"assertion failed",
|
|
1378
|
+
"npm error ",
|
|
1379
|
+
"npm err! ",
|
|
1380
|
+
"enospc",
|
|
1381
|
+
"permission denied",
|
|
1382
|
+
"no such file or directory",
|
|
1383
|
+
"command not found",
|
|
1384
|
+
];
|
|
1385
|
+
const hardFraction = hardFractionRecord();
|
|
1386
|
+
const latch = (() => {
|
|
1387
|
+
const g = governor({ minUptimeMs: 0, latchHardExitMs: 0, latchHardExitObservations: 1_000_000 });
|
|
1388
|
+
g.evaluateFullAt(0.92, 0);
|
|
1389
|
+
return g.evaluateFullAt(0.7, 1).hardExit;
|
|
1390
|
+
})();
|
|
1391
|
+
for (const record of [hardFraction, latch]) {
|
|
1392
|
+
const line = heapRecycleStderrLine(record).toLowerCase();
|
|
1393
|
+
for (const marker of DESKTOP_FATAL_MARKERS) {
|
|
1394
|
+
expect(line, `reason ${record.reason} must not contain "${marker}"`).not.toContain(marker);
|
|
1395
|
+
}
|
|
1396
|
+
expect(line.trimStart().startsWith("#")).toBe(false);
|
|
1397
|
+
expect(line).toContain("hq-sync-runner: heap-recycle");
|
|
1398
|
+
}
|
|
1399
|
+
expect(heapRecycleStderrLine(hardFraction)).toContain("reason=hard_fraction");
|
|
1400
|
+
expect(heapRecycleStderrLine(latch)).toContain("reason=latch_timeout");
|
|
1401
|
+
});
|
|
1402
|
+
});
|
|
1403
|
+
describe("resolveDeclaredHeapCeilingBytes", () => {
|
|
1404
|
+
const MBb = 1024 * 1024;
|
|
1405
|
+
it("parses --max-old-space-size from execArgv in both forms, last occurrence wins", () => {
|
|
1406
|
+
expect(resolveDeclaredHeapCeilingBytes(["--max-old-space-size=2048"], undefined)).toBe(2048 * MBb);
|
|
1407
|
+
expect(resolveDeclaredHeapCeilingBytes(["--max-old-space-size", "2048"], undefined)).toBe(2048 * MBb);
|
|
1408
|
+
expect(resolveDeclaredHeapCeilingBytes(["--max-old-space-size=1024", "--max-old-space-size=2048"], undefined)).toBe(2048 * MBb);
|
|
1409
|
+
expect(resolveDeclaredHeapCeilingBytes(["--max_old_space_size=2048"], undefined)).toBe(2048 * MBb);
|
|
1410
|
+
});
|
|
1411
|
+
it("falls back to NODE_OPTIONS only when execArgv has none, honours one pair of quotes, execArgv wins", () => {
|
|
1412
|
+
expect(resolveDeclaredHeapCeilingBytes([], "--max-old-space-size=2048")).toBe(2048 * MBb);
|
|
1413
|
+
expect(resolveDeclaredHeapCeilingBytes([], '--max-old-space-size="2048"')).toBe(2048 * MBb);
|
|
1414
|
+
expect(resolveDeclaredHeapCeilingBytes(["--max-old-space-size=4096"], "--max-old-space-size=2048")).toBe(4096 * MBb);
|
|
1415
|
+
});
|
|
1416
|
+
it("returns null on absent/garbage/zero so the governor falls back to the reported limit", () => {
|
|
1417
|
+
expect(resolveDeclaredHeapCeilingBytes([], undefined)).toBeNull();
|
|
1418
|
+
expect(resolveDeclaredHeapCeilingBytes(["--max-old-space-size=abc"], undefined)).toBeNull();
|
|
1419
|
+
expect(resolveDeclaredHeapCeilingBytes(["--max-old-space-size=0"], undefined)).toBeNull();
|
|
1420
|
+
expect(resolveDeclaredHeapCeilingBytes(["--other=5"], " ")).toBeNull();
|
|
1421
|
+
});
|
|
1422
|
+
});
|
|
1423
|
+
describe("resolve* parsers for the new governor knobs", () => {
|
|
1424
|
+
it("resolveHeapCheckpointMs: 250 default, 0 valid (disables), rejects '250x'", () => {
|
|
1425
|
+
expect(resolveHeapCheckpointMs("250")).toBe(250);
|
|
1426
|
+
expect(resolveHeapCheckpointMs("0")).toBe(0);
|
|
1427
|
+
expect(resolveHeapCheckpointMs(undefined)).toBe(DEFAULT_HEAP_CHECKPOINT_MS);
|
|
1428
|
+
expect(resolveHeapCheckpointMs("250x")).toBe(DEFAULT_HEAP_CHECKPOINT_MS);
|
|
1429
|
+
expect(resolveHeapCheckpointMs("-1")).toBe(DEFAULT_HEAP_CHECKPOINT_MS);
|
|
1430
|
+
});
|
|
1431
|
+
it("resolveHeapPinnedFraction: 0.84 default, out-of-range/garbage falls back", () => {
|
|
1432
|
+
expect(resolveHeapPinnedFraction("0.84")).toBe(0.84);
|
|
1433
|
+
expect(resolveHeapPinnedFraction("1")).toBe(1);
|
|
1434
|
+
expect(resolveHeapPinnedFraction(undefined)).toBe(DEFAULT_HEAP_PINNED_FRACTION);
|
|
1435
|
+
expect(resolveHeapPinnedFraction("2")).toBe(DEFAULT_HEAP_PINNED_FRACTION);
|
|
1436
|
+
expect(resolveHeapPinnedFraction("0.84x")).toBe(DEFAULT_HEAP_PINNED_FRACTION);
|
|
1437
|
+
});
|
|
1438
|
+
it("resolveHeapPinnedSustainMs: 20000 default, 0 valid, rejects '20s'", () => {
|
|
1439
|
+
expect(resolveHeapPinnedSustainMs("20000")).toBe(20_000);
|
|
1440
|
+
expect(resolveHeapPinnedSustainMs("0")).toBe(0);
|
|
1441
|
+
expect(resolveHeapPinnedSustainMs(undefined)).toBe(DEFAULT_HEAP_PINNED_SUSTAIN_MS);
|
|
1442
|
+
expect(resolveHeapPinnedSustainMs("20s")).toBe(DEFAULT_HEAP_PINNED_SUSTAIN_MS);
|
|
1443
|
+
});
|
|
1444
|
+
it("resolveHeapHardExitFraction: 0.955 default, out-of-range/garbage falls back", () => {
|
|
1445
|
+
expect(resolveHeapHardExitFraction("0.955")).toBe(0.955);
|
|
1446
|
+
expect(resolveHeapHardExitFraction(undefined)).toBe(DEFAULT_HEAP_HARD_EXIT_FRACTION);
|
|
1447
|
+
expect(resolveHeapHardExitFraction("2")).toBe(DEFAULT_HEAP_HARD_EXIT_FRACTION);
|
|
1448
|
+
expect(resolveHeapHardExitFraction("0.955x")).toBe(DEFAULT_HEAP_HARD_EXIT_FRACTION);
|
|
1449
|
+
});
|
|
1450
|
+
it("resolveHeapRecycleLatchHardExitMs: 60000 default, 0 valid, rejects '60s'", () => {
|
|
1451
|
+
expect(resolveHeapRecycleLatchHardExitMs("60000")).toBe(60_000);
|
|
1452
|
+
expect(resolveHeapRecycleLatchHardExitMs("0")).toBe(0);
|
|
1453
|
+
expect(resolveHeapRecycleLatchHardExitMs(undefined)).toBe(DEFAULT_HEAP_RECYCLE_LATCH_HARD_EXIT_MS);
|
|
1454
|
+
expect(resolveHeapRecycleLatchHardExitMs("60s")).toBe(DEFAULT_HEAP_RECYCLE_LATCH_HARD_EXIT_MS);
|
|
1455
|
+
});
|
|
1456
|
+
});
|
|
697
1457
|
//# sourceMappingURL=sync-runner-heap-telemetry.test.js.map
|