@indigoai-us/hq-cloud 6.16.2 → 6.16.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-events.test.js +22 -12
- package/dist/bin/sync-runner-events.test.js.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.d.ts +238 -84
- package/dist/bin/sync-runner-heap-telemetry.d.ts.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.js +385 -131
- package/dist/bin/sync-runner-heap-telemetry.js.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.test.js +539 -204
- package/dist/bin/sync-runner-heap-telemetry.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +65 -15
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +47 -9
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +32 -14
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +2 -253
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +10 -0
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +67 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -0
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +2 -0
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +5 -1
- package/dist/object-io.test.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +3 -1
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +6 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/s3.d.ts +2 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +1 -0
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +22 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/session-log-capture.d.ts +20 -0
- package/dist/session-log-capture.d.ts.map +1 -0
- package/dist/session-log-capture.js +648 -0
- package/dist/session-log-capture.js.map +1 -0
- package/dist/session-log-capture.test.d.ts +2 -0
- package/dist/session-log-capture.test.d.ts.map +1 -0
- package/dist/session-log-capture.test.js +276 -0
- package/dist/session-log-capture.test.js.map +1 -0
- package/dist/sync/state-store.js +5 -1
- package/dist/sync/state-store.js.map +1 -1
- package/dist/vault-client.d.ts +5 -0
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +1 -0
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +15 -0
- package/dist/vault-client.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, expect, it, vi } from "vitest";
|
|
2
|
-
import { HeapCensus, RunnerHeapTelemetry, sampleRunnerHeap, heapSampleCloudProperties, heapPressureStderrLine, heapRecycleStderrLine, startRunnerHeapSampling, resolveHeapPressureFraction, resolveHeapSampleIntervalMs, resolveHeapRecycleEnabled,
|
|
2
|
+
import { HeapCensus, RunnerHeapTelemetry, sampleRunnerHeap, heapSampleCloudProperties, heapPressureCloudProperties, heapRecycleCloudProperties, heapPressureStderrLine, heapRecycleStderrLine, startRunnerHeapSampling, nodeMajorGcObserver, isFreshGcReading, MAX_FRESH_GC_READING_MS, resolveHeapPressureFraction, resolveHeapSampleIntervalMs, resolveHeapRecycleEnabled, resolveHeapRecycleLiveFraction, resolveHeapRecycleEmergencyFraction, resolveHeapRecycleSustainWindowMs, resolveHeapRecycleMinUptimeMs, DEFAULT_HEAP_RECYCLE_LIVE_FRACTION, DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION, DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS, DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS, } from "./sync-runner-heap-telemetry.js";
|
|
3
3
|
import { emitCloudTelemetry } from "../telemetry-events.js";
|
|
4
4
|
function mutableReaders() {
|
|
5
5
|
const state = {
|
|
@@ -55,6 +55,28 @@ function fakeScheduler() {
|
|
|
55
55
|
unrefCalls: () => unrefCalls,
|
|
56
56
|
};
|
|
57
57
|
}
|
|
58
|
+
/** A manual GC observer whose registered callback the test fires by hand. */
|
|
59
|
+
function fakeGcObserver() {
|
|
60
|
+
let cb = null;
|
|
61
|
+
let started = false;
|
|
62
|
+
let stopped = false;
|
|
63
|
+
const observer = {
|
|
64
|
+
start: (onMajorGc) => {
|
|
65
|
+
cb = onMajorGc;
|
|
66
|
+
started = true;
|
|
67
|
+
},
|
|
68
|
+
stop: () => {
|
|
69
|
+
stopped = true;
|
|
70
|
+
cb = null;
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
return {
|
|
74
|
+
observer,
|
|
75
|
+
fire: () => cb?.(),
|
|
76
|
+
started: () => started,
|
|
77
|
+
stopped: () => stopped,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
58
80
|
describe("sampleRunnerHeap", () => {
|
|
59
81
|
it("maps injected v8/process readings to a flat numeric record", () => {
|
|
60
82
|
const census = new HeapCensus();
|
|
@@ -88,11 +110,17 @@ describe("sampleRunnerHeap", () => {
|
|
|
88
110
|
{ site: "setB", count: 5 },
|
|
89
111
|
]);
|
|
90
112
|
});
|
|
91
|
-
it("emits only numeric properties that survive the
|
|
113
|
+
it("emits only numeric properties that survive the LOCAL telemetry sanitizer", async () => {
|
|
114
|
+
// NOTE: this exercises hq-cloud's own emit-side sanitizer, which keys on the
|
|
115
|
+
// VALUE type — a numeric value survives regardless of key. It is a DIFFERENT
|
|
116
|
+
// and more permissive layer than hq-pro's ingest contract (the vault's
|
|
117
|
+
// redactProperties, which drops any key not ending in _count/_bytes/_ms or
|
|
118
|
+
// containing a denied substring). The regression this module fixes is that
|
|
119
|
+
// the old `census_<token>` spelling passed HERE but was silently dropped at
|
|
120
|
+
// ingest; the deny-safe `_count` spelling — pinned by the ingest-key-contract
|
|
121
|
+
// test below — is what makes the attribution actually reach the store.
|
|
92
122
|
const census = new HeapCensus();
|
|
93
|
-
|
|
94
|
-
// "content": it must still survive BECAUSE its value is numeric.
|
|
95
|
-
census.register("publishedContentHashGenerations", () => 12345);
|
|
123
|
+
census.register("publishedDigestGenerations", () => 12345);
|
|
96
124
|
census.register("knownKinds", () => 10);
|
|
97
125
|
const { state, readers } = mutableReaders();
|
|
98
126
|
Object.assign(state, {
|
|
@@ -123,7 +151,7 @@ describe("sampleRunnerHeap", () => {
|
|
|
123
151
|
properties: props,
|
|
124
152
|
});
|
|
125
153
|
const survived = captured[0]?.properties ?? {};
|
|
126
|
-
expect(survived.
|
|
154
|
+
expect(survived.census_publishedDigestGenerations_count).toBe(12345);
|
|
127
155
|
expect(survived.heap_used_bytes).toBe(1);
|
|
128
156
|
// No property was dropped by the sanitizer.
|
|
129
157
|
expect(Object.keys(survived).length).toBe(Object.keys(props).length);
|
|
@@ -352,166 +380,240 @@ describe("heapPressureStderrLine", () => {
|
|
|
352
380
|
expect(line).toContain("top=knownKinds:3");
|
|
353
381
|
});
|
|
354
382
|
});
|
|
355
|
-
// --- Self-recycling governor
|
|
383
|
+
// --- Self-recycling governor (GC-anchored post-GC live floor) -----------------
|
|
356
384
|
const MB = 1024 * 1024;
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
function runToRecycle(opts) {
|
|
364
|
-
const census = opts.census ?? new HeapCensus();
|
|
365
|
-
if (!opts.census)
|
|
385
|
+
// Comfortably past the 35-minute uptime floor, so the floor never gates a test
|
|
386
|
+
// that is exercising a different property.
|
|
387
|
+
const PAST_FLOOR_MS = DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS + 5 * 60_000;
|
|
388
|
+
function governor(overrides = {}) {
|
|
389
|
+
const census = overrides.census ?? new HeapCensus();
|
|
390
|
+
if (!overrides.census)
|
|
366
391
|
census.register("knownKinds", () => 1);
|
|
367
392
|
const { state, readers } = mutableReaders();
|
|
368
|
-
state.limit =
|
|
369
|
-
state.used = opts.usedBytes;
|
|
370
|
-
state.now = opts.uptimeMs ?? ONE_HOUR_MS;
|
|
393
|
+
state.limit = overrides.limitBytes ?? 2048 * MB;
|
|
371
394
|
const controller = new RunnerHeapTelemetry({
|
|
372
395
|
readers,
|
|
373
396
|
census,
|
|
374
397
|
sessionStartedAtMs: 0,
|
|
375
|
-
pressureThresholdFraction:
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
398
|
+
pressureThresholdFraction: 0.75,
|
|
399
|
+
recycleLiveFraction: overrides.liveFraction ?? 0.6,
|
|
400
|
+
recycleSustainWindowMs: overrides.windowMs ?? 120_000,
|
|
401
|
+
recycleEmergencyFraction: overrides.emergencyFraction ?? 0.9,
|
|
402
|
+
recycleMinUptimeMs: overrides.minUptimeMs ?? DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS,
|
|
403
|
+
recycleEnabled: overrides.recycleEnabled ?? true,
|
|
404
|
+
});
|
|
405
|
+
const observeAt = (fraction, nowMs) => {
|
|
406
|
+
state.used = Math.round(state.limit * fraction);
|
|
407
|
+
state.now = nowMs;
|
|
408
|
+
return controller.observeMajorGc();
|
|
409
|
+
};
|
|
410
|
+
const sampleAt = (fraction, nowMs) => {
|
|
411
|
+
state.used = Math.round(state.limit * fraction);
|
|
412
|
+
state.now = nowMs;
|
|
413
|
+
return controller.sample();
|
|
414
|
+
};
|
|
415
|
+
const evaluateAt = (fraction, nowMs) => {
|
|
416
|
+
state.used = Math.round(state.limit * fraction);
|
|
417
|
+
state.now = nowMs;
|
|
418
|
+
return controller.evaluate();
|
|
419
|
+
};
|
|
420
|
+
return { state, readers, census, controller, observeAt, sampleAt, evaluateAt };
|
|
391
421
|
}
|
|
392
|
-
describe("RunnerHeapTelemetry — recycle governor", () => {
|
|
393
|
-
it("
|
|
394
|
-
const
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
recycleMinUptimeMs: 30 * 60_000,
|
|
407
|
-
});
|
|
408
|
-
// One spike above the fraction, then a dip below: the streak resets, so no
|
|
409
|
-
// recycle can ever accumulate from a single excursion.
|
|
410
|
-
state.used = Math.round(2048 * MB * 0.9);
|
|
411
|
-
expect(controller.sample().recycle).toBeNull(); // 1 consecutive
|
|
412
|
-
state.used = Math.round(2048 * MB * 0.5);
|
|
413
|
-
expect(controller.sample().recycle).toBeNull(); // dip -> streak reset
|
|
414
|
-
state.used = Math.round(2048 * MB * 0.9);
|
|
415
|
-
expect(controller.sample().recycle).toBeNull(); // 1 consecutive again
|
|
416
|
-
// Now a genuine sustained run of two consecutive over-fraction samples.
|
|
417
|
-
const second = controller.sample().recycle; // 2 consecutive -> recycle
|
|
418
|
-
expect(second).not.toBeNull();
|
|
419
|
-
expect(second?.observedSustainedSamples).toBe(2);
|
|
420
|
-
expect(second?.sustainedSamples).toBe(2);
|
|
421
|
-
});
|
|
422
|
-
it("never recycles before the uptime floor, however high the heap climbs", () => {
|
|
423
|
-
// Heap pinned at 0.99 of the ceiling for many samples, but uptime is below
|
|
424
|
-
// the 30-minute floor: the desktop's fast-fail (60s) and slow-death
|
|
425
|
-
// (30-min) counters can never be advanced by a recycle.
|
|
426
|
-
const belowFloor = runToRecycle({
|
|
427
|
-
limitBytes: 2048 * MB,
|
|
428
|
-
usedBytes: Math.round(2048 * MB * 0.99),
|
|
429
|
-
uptimeMs: 30 * 60_000 - 1, // one ms below the floor
|
|
430
|
-
samples: 12,
|
|
431
|
-
});
|
|
432
|
-
expect(belowFloor.recycle).toBeNull();
|
|
433
|
-
expect(belowFloor.recycleCount).toBe(0);
|
|
434
|
-
// At the floor, the sustained streak (already satisfied) lets it recycle.
|
|
435
|
-
const atFloor = runToRecycle({
|
|
436
|
-
limitBytes: 2048 * MB,
|
|
437
|
-
usedBytes: Math.round(2048 * MB * 0.99),
|
|
438
|
-
uptimeMs: 30 * 60_000,
|
|
439
|
-
samples: 12,
|
|
440
|
-
});
|
|
441
|
-
expect(atFloor.recycle).not.toBeNull();
|
|
422
|
+
describe("RunnerHeapTelemetry — recycle governor (primary post-GC live path)", () => {
|
|
423
|
+
it("recycles on a sustained post-GC live floor across the window, past the uptime floor", () => {
|
|
424
|
+
const g = governor();
|
|
425
|
+
// First qualifying post-GC observation at/above the live floor: anchors only.
|
|
426
|
+
expect(g.observeAt(0.62, PAST_FLOOR_MS)).toBeNull();
|
|
427
|
+
// Too soon (less than the 120 s window since the anchor): still no verdict.
|
|
428
|
+
expect(g.observeAt(0.65, PAST_FLOOR_MS + 119_000)).toBeNull();
|
|
429
|
+
// A window later, still at/above the floor: recycle, trigger "live".
|
|
430
|
+
const recycle = g.observeAt(0.66, PAST_FLOOR_MS + 121_000);
|
|
431
|
+
expect(recycle).not.toBeNull();
|
|
432
|
+
expect(recycle?.trigger).toBe("live");
|
|
433
|
+
expect(recycle?.liveFraction).toBeCloseTo(0.66, 2);
|
|
434
|
+
expect(recycle?.observedSustainMs).toBe(121_000);
|
|
435
|
+
expect(recycle?.recycleLiveFraction).toBe(0.6);
|
|
442
436
|
});
|
|
443
|
-
it("
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
//
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
expect(
|
|
468
|
-
expect(
|
|
469
|
-
expect(
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
//
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
const
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
437
|
+
it("is immune to an instantaneous sawtooth: neither the sample path nor a below-floor live reading ever recycles", () => {
|
|
438
|
+
// The exact defect: instantaneous used_heap_size oscillates high (0.60..0.89,
|
|
439
|
+
// never at the 0.90 emergency line) while the POST-GC live floor stays low
|
|
440
|
+
// (0.30). The old rule watched the instantaneous series and would have tripped;
|
|
441
|
+
// the GC-anchored rule watches only the post-GC floor and never does.
|
|
442
|
+
const g = governor();
|
|
443
|
+
const inst = [0.62, 0.30, 0.88, 0.35, 0.89, 0.28, 0.87, 0.40];
|
|
444
|
+
let t = PAST_FLOOR_MS;
|
|
445
|
+
for (const f of inst) {
|
|
446
|
+
// The interval sampler sees the instantaneous sawtooth — emergency only,
|
|
447
|
+
// and 0.89 < 0.90, so it never fires.
|
|
448
|
+
expect(g.sampleAt(f, t).recycle).toBeNull();
|
|
449
|
+
// The post-GC live floor is flat at 0.30, well below the 0.60 floor.
|
|
450
|
+
expect(g.observeAt(0.3, t)).toBeNull();
|
|
451
|
+
t += 60_000;
|
|
452
|
+
}
|
|
453
|
+
});
|
|
454
|
+
it("requires the window between the TWO qualifying observations, and resets the anchor on a dip", () => {
|
|
455
|
+
const g = governor();
|
|
456
|
+
// Two qualifying observations less than a window apart never recycle.
|
|
457
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS)).toBeNull();
|
|
458
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS + 60_000)).toBeNull(); // 60 s < 120 s
|
|
459
|
+
// A dip below the floor resets the anchor: the clock since the FIRST
|
|
460
|
+
// observation no longer counts.
|
|
461
|
+
expect(g.observeAt(0.4, PAST_FLOOR_MS + 90_000)).toBeNull(); // dip -> anchor cleared
|
|
462
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS + 100_000)).toBeNull(); // re-anchor here
|
|
463
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS + 180_000)).toBeNull(); // only 80 s since re-anchor
|
|
464
|
+
const recycle = g.observeAt(0.7, PAST_FLOOR_MS + 221_000); // 121 s since re-anchor
|
|
465
|
+
expect(recycle).not.toBeNull();
|
|
466
|
+
// The sustain span is measured from the RE-ANCHOR, not the first observation.
|
|
467
|
+
expect(recycle?.observedSustainMs).toBe(121_000);
|
|
468
|
+
});
|
|
469
|
+
it("never recycles (live path) before the uptime floor, however high the live floor climbs", () => {
|
|
470
|
+
const g = governor();
|
|
471
|
+
// Live floor pinned at 0.80 for many observations, but every one is below the
|
|
472
|
+
// 35-minute floor: no verdict, so the desktop's crash streak is never touched.
|
|
473
|
+
let t = 60_000;
|
|
474
|
+
for (let i = 0; i < 40; i += 1) {
|
|
475
|
+
expect(g.observeAt(0.8, t)).toBeNull();
|
|
476
|
+
t += 30_000; // stays under DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS (35 min)
|
|
477
|
+
}
|
|
478
|
+
// Cross the floor with the window already satisfied -> recycle.
|
|
479
|
+
const recycle = g.observeAt(0.8, DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS + 1);
|
|
480
|
+
expect(recycle).not.toBeNull();
|
|
481
|
+
expect(recycle?.trigger).toBe("live");
|
|
482
|
+
});
|
|
483
|
+
it("keeps the uptime floor strictly above hq-desktop-app's 30-minute HEALTHY_RUN_WINDOW", () => {
|
|
484
|
+
// A recycle must OUTLIVE the desktop's 30-minute window so should_reset_after_recovery
|
|
485
|
+
// clears the crash streak instead of pacing respawn backoff. The literal is
|
|
486
|
+
// quoted so a future change that would drop the floor to/under it fails here.
|
|
487
|
+
const DESKTOP_HEALTHY_RUN_WINDOW_MS = 1_800_000;
|
|
488
|
+
expect(DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS).toBe(2_100_000); // 35 min
|
|
489
|
+
expect(DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS).toBeGreaterThan(DESKTOP_HEALTHY_RUN_WINDOW_MS);
|
|
490
|
+
});
|
|
491
|
+
it("scales the live floor with the reported heap limit, so a larger ceiling never recycles prematurely", () => {
|
|
492
|
+
// 1700 MB is 0.83 of a 2048 MB ceiling (recycles) but only 0.415 of a 4096 MB
|
|
493
|
+
// ceiling (below the 0.60 live floor -> never), because the floor is a
|
|
494
|
+
// fraction of the injected heap_size_limit, not an absolute figure.
|
|
495
|
+
const small = governor({ limitBytes: 2048 * MB });
|
|
496
|
+
small.state.used = 1700 * MB;
|
|
497
|
+
small.state.now = PAST_FLOOR_MS;
|
|
498
|
+
expect(small.controller.observeMajorGc()).toBeNull(); // anchor
|
|
499
|
+
small.state.now = PAST_FLOOR_MS + 121_000;
|
|
500
|
+
expect(small.controller.observeMajorGc()).not.toBeNull();
|
|
501
|
+
const large = governor({ limitBytes: 4096 * MB });
|
|
502
|
+
let t = PAST_FLOOR_MS;
|
|
503
|
+
for (let i = 0; i < 10; i += 1) {
|
|
504
|
+
large.state.used = 1700 * MB; // 0.415 of 4096 MB
|
|
505
|
+
large.state.now = t;
|
|
506
|
+
expect(large.controller.observeMajorGc()).toBeNull();
|
|
507
|
+
t += 121_000;
|
|
508
|
+
}
|
|
509
|
+
});
|
|
510
|
+
it("maintains the anchor on every major GC, so a close below-floor recovery resets the sustain run", () => {
|
|
511
|
+
// The anchor is updated on EVERY observed major GC — there is no throttle
|
|
512
|
+
// that could skip a below-floor recovery and leave a stale anchor faking a
|
|
513
|
+
// continuous run (the reviewed defect).
|
|
514
|
+
const g = governor();
|
|
515
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS)).toBeNull(); // anchor
|
|
516
|
+
expect(g.observeAt(0.4, PAST_FLOOR_MS + 1_000)).toBeNull(); // 1 s later, below floor -> reset
|
|
517
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS + 2_000)).toBeNull(); // re-anchor here
|
|
518
|
+
// 121 s past the FIRST anchor but only 119 s past the RE-anchor: no recycle.
|
|
519
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS + 121_000)).toBeNull();
|
|
520
|
+
// A window past the re-anchor -> recycle, sustain measured from the re-anchor.
|
|
521
|
+
const recycle = g.observeAt(0.7, PAST_FLOOR_MS + 123_000);
|
|
522
|
+
expect(recycle).not.toBeNull();
|
|
523
|
+
expect(recycle?.observedSustainMs).toBe(121_000);
|
|
524
|
+
});
|
|
525
|
+
it("attaches census growth and rss/external/arrayBuffers deltas to the recycle record", () => {
|
|
526
|
+
let entries = 100;
|
|
487
527
|
const census = new HeapCensus();
|
|
488
|
-
census.register("
|
|
489
|
-
const {
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
528
|
+
census.register("watcherJournalStores", () => entries);
|
|
529
|
+
const g = governor({ census });
|
|
530
|
+
// Prime the growth baseline at session start (entries=100, rss=0), exactly as
|
|
531
|
+
// startRunnerHeapSampling does before the first interval.
|
|
532
|
+
g.controller.primeBaseline();
|
|
533
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS)).toBeNull(); // anchor
|
|
534
|
+
entries = 900;
|
|
535
|
+
g.state.rss = 1_500 * MB;
|
|
536
|
+
g.state.external = 40 * MB;
|
|
537
|
+
g.state.arrayBuffers = 12 * MB;
|
|
538
|
+
const recycle = g.observeAt(0.7, PAST_FLOOR_MS + 121_000);
|
|
539
|
+
expect(recycle).not.toBeNull();
|
|
540
|
+
expect(recycle?.topSites[0]).toMatchObject({ site: "watcherJournalStores", growth: 800 });
|
|
541
|
+
expect(recycle?.rssGrowthBytes).toBe(1_500 * MB);
|
|
542
|
+
expect(recycle?.externalGrowthBytes).toBe(40 * MB);
|
|
543
|
+
expect(recycle?.arrayBuffersGrowthBytes).toBe(12 * MB);
|
|
544
|
+
expect(recycle?.censusTotal).toBe(900);
|
|
545
|
+
});
|
|
546
|
+
});
|
|
547
|
+
describe("RunnerHeapTelemetry — emergency backstop", () => {
|
|
548
|
+
it("recycles on a single instantaneous sample at/above the emergency fraction, even below the uptime floor", () => {
|
|
549
|
+
const g = governor();
|
|
550
|
+
// Well below the uptime floor, but a single instantaneous sample above the
|
|
551
|
+
// 0.90 emergency line: at that reading V8 is minutes from aborting, recycle now.
|
|
552
|
+
const { recycle } = g.sampleAt(0.92, 30_000);
|
|
553
|
+
expect(recycle).not.toBeNull();
|
|
554
|
+
expect(recycle?.trigger).toBe("emergency");
|
|
555
|
+
expect(recycle?.observedSustainMs).toBe(0);
|
|
556
|
+
});
|
|
557
|
+
it("does not fire the emergency backstop below the emergency fraction", () => {
|
|
558
|
+
const g = governor();
|
|
559
|
+
// 0.89 instantaneous, sustained across many samples below the floor: the
|
|
560
|
+
// emergency arm never fires (and the live path is off — no GC observations).
|
|
561
|
+
let t = 30_000;
|
|
562
|
+
for (let i = 0; i < 20; i += 1) {
|
|
563
|
+
expect(g.sampleAt(0.89, t).recycle).toBeNull();
|
|
564
|
+
t += 60_000;
|
|
565
|
+
}
|
|
566
|
+
});
|
|
567
|
+
it("fires the emergency path even when a post-GC observation is already abort-adjacent (bypassing the window)", () => {
|
|
568
|
+
const g = governor();
|
|
569
|
+
// A post-GC live reading already at 0.95 recycles immediately, without the
|
|
570
|
+
// sustain window and without a prior anchor.
|
|
571
|
+
const recycle = g.observeAt(0.95, 20_000);
|
|
572
|
+
expect(recycle).not.toBeNull();
|
|
573
|
+
expect(recycle?.trigger).toBe("emergency");
|
|
574
|
+
});
|
|
575
|
+
it("evaluate() is a loop-top backstop that runs the emergency check off the timer path", () => {
|
|
576
|
+
const g = governor();
|
|
577
|
+
// The interval sampler never ticked, but a loop-top evaluate() catches an
|
|
578
|
+
// abort-adjacent instantaneous reading — the starved-timer safety net.
|
|
579
|
+
expect(g.evaluateAt(0.6, 30_000)).toBeNull(); // below emergency, no window seen
|
|
580
|
+
const recycle = g.evaluateAt(0.92, 60_000);
|
|
581
|
+
expect(recycle).not.toBeNull();
|
|
582
|
+
expect(recycle?.trigger).toBe("emergency");
|
|
583
|
+
});
|
|
584
|
+
});
|
|
585
|
+
describe("RunnerHeapTelemetry — recycle governor invariants", () => {
|
|
586
|
+
it("emits at most one recycle verdict per session, across every path", () => {
|
|
587
|
+
const g = governor();
|
|
588
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS)).toBeNull();
|
|
589
|
+
const first = g.observeAt(0.7, PAST_FLOOR_MS + 121_000);
|
|
590
|
+
expect(first).not.toBeNull();
|
|
591
|
+
// No second verdict from any path.
|
|
592
|
+
expect(g.observeAt(0.95, PAST_FLOOR_MS + 200_000)).toBeNull();
|
|
593
|
+
expect(g.sampleAt(0.99, PAST_FLOOR_MS + 260_000).recycle).toBeNull();
|
|
594
|
+
expect(g.evaluateAt(0.99, PAST_FLOOR_MS + 320_000)).toBeNull();
|
|
595
|
+
});
|
|
596
|
+
it("the kill switch disables every recycle path; pressure still behaves as base", () => {
|
|
597
|
+
const g = governor({ recycleEnabled: false });
|
|
501
598
|
let pressures = 0;
|
|
599
|
+
let recycles = 0;
|
|
600
|
+
let t = PAST_FLOOR_MS;
|
|
502
601
|
for (let i = 0; i < 20; i += 1) {
|
|
503
|
-
const { pressure, recycle } =
|
|
504
|
-
if (recycle)
|
|
505
|
-
recycles += 1;
|
|
602
|
+
const { pressure, recycle } = g.sampleAt(0.99, t); // far above any fraction
|
|
506
603
|
if (pressure)
|
|
507
604
|
pressures += 1;
|
|
605
|
+
if (recycle)
|
|
606
|
+
recycles += 1;
|
|
607
|
+
if (g.observeAt(0.99, t))
|
|
608
|
+
recycles += 1;
|
|
609
|
+
if (g.evaluateAt(0.99, t))
|
|
610
|
+
recycles += 1;
|
|
611
|
+
t += 60_000;
|
|
508
612
|
}
|
|
509
|
-
expect(recycles).toBe(0); // governor off — never recycles
|
|
613
|
+
expect(recycles).toBe(0); // governor off — never recycles on ANY path
|
|
510
614
|
expect(pressures).toBe(1); // pressure record still fires once per crossing
|
|
511
615
|
});
|
|
512
616
|
it("a recycle record and stderr line carry only fixed tokens and integers — never a runner byte", () => {
|
|
513
|
-
// A census whose backing structure is keyed by REAL-looking absolute paths,
|
|
514
|
-
// a company slug, and a filename. Only its numeric COUNT may ever be emitted.
|
|
515
617
|
const SECRET_PATH = "/Users/hassaan/hq/companies/acme-corp/vault/report.md";
|
|
516
618
|
const SLUG = "acme-corp";
|
|
517
619
|
const FILENAME = "report.md";
|
|
@@ -521,12 +623,9 @@ describe("RunnerHeapTelemetry — recycle governor", () => {
|
|
|
521
623
|
backing.set(FILENAME, 3);
|
|
522
624
|
const census = new HeapCensus();
|
|
523
625
|
census.register("watcherJournalStores", () => backing.size);
|
|
524
|
-
census.register("
|
|
525
|
-
const
|
|
526
|
-
|
|
527
|
-
usedBytes: Math.round(2048 * MB * 0.95),
|
|
528
|
-
census,
|
|
529
|
-
});
|
|
626
|
+
census.register("publishedDigestGenerations", () => backing.size);
|
|
627
|
+
const g = governor({ census });
|
|
628
|
+
const recycle = g.sampleAt(0.95, PAST_FLOOR_MS).recycle; // emergency path
|
|
530
629
|
expect(recycle).not.toBeNull();
|
|
531
630
|
const line = heapRecycleStderrLine(recycle);
|
|
532
631
|
const json = JSON.stringify(recycle);
|
|
@@ -534,12 +633,9 @@ describe("RunnerHeapTelemetry — recycle governor", () => {
|
|
|
534
633
|
expect(line).not.toContain(secret);
|
|
535
634
|
expect(json).not.toContain(secret);
|
|
536
635
|
}
|
|
537
|
-
// The FIXED census tokens are allowed; the runner bytes are not.
|
|
538
636
|
expect(json).toContain("watcherJournalStores");
|
|
539
637
|
});
|
|
540
638
|
it("the recycle stderr line matches NONE of hq-desktop-app's fatal stderr markers", () => {
|
|
541
|
-
// The exact marker vocabulary of classify_runner_fatal_class /
|
|
542
|
-
// is_disk_exhaustion_failure in hq-desktop-app crates/hq-desktop-core.
|
|
543
639
|
const DESKTOP_FATAL_MARKERS = [
|
|
544
640
|
"javascript heap out of memory",
|
|
545
641
|
"ineffective mark-compacts",
|
|
@@ -556,53 +652,199 @@ describe("RunnerHeapTelemetry — recycle governor", () => {
|
|
|
556
652
|
"command not found",
|
|
557
653
|
];
|
|
558
654
|
const census = new HeapCensus();
|
|
559
|
-
// Register the real production census tokens so the `top=` list is realistic.
|
|
560
655
|
for (const token of [
|
|
561
656
|
"pendingAppliedUpserts",
|
|
562
657
|
"watcherJournalStores",
|
|
563
|
-
"
|
|
658
|
+
"publishedDigestGenerations",
|
|
564
659
|
"knownKinds",
|
|
565
660
|
"deleteSnapshotEpochs",
|
|
566
661
|
]) {
|
|
567
662
|
census.register(token, () => 7);
|
|
568
663
|
}
|
|
569
|
-
const
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
census,
|
|
573
|
-
});
|
|
664
|
+
const g = governor({ census });
|
|
665
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS)).toBeNull();
|
|
666
|
+
const recycle = g.observeAt(0.7, PAST_FLOOR_MS + 121_000);
|
|
574
667
|
const line = heapRecycleStderrLine(recycle);
|
|
575
668
|
const lowered = line.toLowerCase();
|
|
576
669
|
for (const marker of DESKTOP_FATAL_MARKERS) {
|
|
577
670
|
expect(lowered).not.toContain(marker);
|
|
578
671
|
}
|
|
579
|
-
// NodeCheckAbort keys on a leading '#'; the recycle line never starts with one.
|
|
580
672
|
expect(line.trimStart().startsWith("#")).toBe(false);
|
|
581
|
-
// Sanity: the line still carries its own recognizable, benign shape.
|
|
582
673
|
expect(line).toContain("hq-sync-runner: heap-recycle");
|
|
583
|
-
expect(line).toContain("
|
|
674
|
+
expect(line).toContain("trigger=live");
|
|
675
|
+
expect(line).toContain("sustain_ms=");
|
|
676
|
+
});
|
|
677
|
+
});
|
|
678
|
+
describe("RunnerHeapTelemetry — recorded production series replay (regression reopen)", () => {
|
|
679
|
+
// The aborting generation's own recorded telemetry (person#prs_01KQ2TZQMA...,
|
|
680
|
+
// 2026-09-01 18:57..19:48Z). heap_size_limit was 2348810240 B; the SIGABRT
|
|
681
|
+
// landed at uptime 55m16s = 3_316_000 ms.
|
|
682
|
+
const RECORDED_LIMIT_BYTES = 2_348_810_240;
|
|
683
|
+
const ABORT_UPTIME_MS = 3_316_000;
|
|
684
|
+
// The recorded per-sample INSTANTANEOUS used/limit fractions. Max is 0.741 —
|
|
685
|
+
// it never reaches the old 0.78 rule and never two-in-a-row, and never the
|
|
686
|
+
// 0.90 emergency line, so the instantaneous path is inert (the base failure).
|
|
687
|
+
const INSTANTANEOUS = [
|
|
688
|
+
0.072, 0.106, 0.109, 0.458, 0.546, 0.251, 0.592, 0.564, 0.559, 0.427, 0.457,
|
|
689
|
+
0.564, 0.397, 0.584, 0.438, 0.523, 0.64, 0.548, 0.66, 0.664, 0.512, 0.513,
|
|
690
|
+
0.69, 0.703, 0.741,
|
|
691
|
+
];
|
|
692
|
+
it("base instantaneous logic is inert: no sample-path recycle across the entire recorded series", () => {
|
|
693
|
+
const g = governor({ limitBytes: RECORDED_LIMIT_BYTES });
|
|
694
|
+
let recycles = 0;
|
|
695
|
+
let t = 4.24 * 60_000;
|
|
696
|
+
for (const f of INSTANTANEOUS) {
|
|
697
|
+
if (g.sampleAt(f, t).recycle)
|
|
698
|
+
recycles += 1;
|
|
699
|
+
t += 2 * 60_000;
|
|
700
|
+
}
|
|
701
|
+
expect(recycles).toBe(0); // the exact inertness the regression exhibited
|
|
702
|
+
});
|
|
703
|
+
it("GC-anchored logic produces exactly one live recycle before the recorded 55m16s abort", () => {
|
|
704
|
+
// The implied POST-GC live floor: monotonic, crossing 0.60 between ~36.7 and
|
|
705
|
+
// ~39 min (the lower envelope of the instantaneous series). One major-GC
|
|
706
|
+
// observation per minute.
|
|
707
|
+
const LIVE_FLOOR = [
|
|
708
|
+
{ uptimeMin: 4, fraction: 0.07 },
|
|
709
|
+
{ uptimeMin: 12, fraction: 0.11 },
|
|
710
|
+
{ uptimeMin: 20, fraction: 0.25 },
|
|
711
|
+
{ uptimeMin: 28, fraction: 0.42 },
|
|
712
|
+
{ uptimeMin: 34, fraction: 0.51 },
|
|
713
|
+
{ uptimeMin: 37, fraction: 0.6 }, // crosses the floor here -> anchor
|
|
714
|
+
{ uptimeMin: 39, fraction: 0.63 }, // >= 120 s later -> recycle
|
|
715
|
+
{ uptimeMin: 46, fraction: 0.69 },
|
|
716
|
+
{ uptimeMin: 52, fraction: 0.74 },
|
|
717
|
+
];
|
|
718
|
+
const g = governor({ limitBytes: RECORDED_LIMIT_BYTES });
|
|
719
|
+
const verdicts = [];
|
|
720
|
+
for (const point of LIVE_FLOOR) {
|
|
721
|
+
const recycle = g.observeAt(point.fraction, point.uptimeMin * 60_000);
|
|
722
|
+
if (recycle)
|
|
723
|
+
verdicts.push(recycle);
|
|
724
|
+
}
|
|
725
|
+
expect(verdicts).toHaveLength(1);
|
|
726
|
+
expect(verdicts[0].trigger).toBe("live");
|
|
727
|
+
expect(verdicts[0].uptimeMs).toBeLessThan(ABORT_UPTIME_MS); // wins the race to V8
|
|
728
|
+
expect(verdicts[0].uptimeMs).toBeGreaterThanOrEqual(DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS);
|
|
729
|
+
// ~16 minutes of headroom before the recorded abort.
|
|
730
|
+
expect(ABORT_UPTIME_MS - verdicts[0].uptimeMs).toBeGreaterThan(10 * 60_000);
|
|
584
731
|
});
|
|
585
732
|
});
|
|
586
|
-
describe("
|
|
587
|
-
|
|
733
|
+
describe("ingest key contract — every emitted property key survives hq-pro redactProperties", () => {
|
|
734
|
+
// hq-pro's raw-telemetry-envelope ingest keeps a numeric property ONLY when its
|
|
735
|
+
// key matches this shape, ends in _count/_bytes/_ms, and contains none of these
|
|
736
|
+
// denied substrings. Quoted here as literals: a future key that violates the
|
|
737
|
+
// contract (the exact defect that dropped every census field to date) fails.
|
|
738
|
+
const KEY_SHAPE = /^[A-Za-z][A-Za-z0-9_]{0,63}$/;
|
|
739
|
+
const DENY_PARTS = [
|
|
740
|
+
"authorization",
|
|
741
|
+
"body",
|
|
742
|
+
"content",
|
|
743
|
+
"cookie",
|
|
744
|
+
"email",
|
|
745
|
+
"file",
|
|
746
|
+
"message",
|
|
747
|
+
"password",
|
|
748
|
+
"path",
|
|
749
|
+
"prompt",
|
|
750
|
+
"query",
|
|
751
|
+
"secret",
|
|
752
|
+
"text",
|
|
753
|
+
"token",
|
|
754
|
+
"transcript",
|
|
755
|
+
];
|
|
756
|
+
// Every FIXED census token the production runner registers (watch-loop ownMaps
|
|
757
|
+
// + the decoupled emitter/watcher tokens). If a new token is added, add it here.
|
|
758
|
+
const PRODUCTION_CENSUS_TOKENS = [
|
|
759
|
+
"pendingAppliedUpserts",
|
|
760
|
+
"pendingWatcherEntries",
|
|
761
|
+
"pendingWatcherChanges",
|
|
762
|
+
"pendingWatcherDroppedRouteHints",
|
|
763
|
+
"quarantinedDeleteRoots",
|
|
764
|
+
"watcherJournalStores",
|
|
765
|
+
"watcherJournalRows",
|
|
766
|
+
"targetedJournalSessions",
|
|
767
|
+
"targetedJournalRows",
|
|
768
|
+
"cachedAreaJournalMigrations",
|
|
769
|
+
"pendingSlowPasses",
|
|
770
|
+
"pendingFastPasses",
|
|
771
|
+
"lastPublishedDigests",
|
|
772
|
+
"publishedDigestGenerations",
|
|
773
|
+
"deferredPublishes",
|
|
774
|
+
"durablePublishedHashRetries",
|
|
775
|
+
"initialPublishedDigests",
|
|
776
|
+
"knownKinds",
|
|
777
|
+
"deleteSnapshotEpochs",
|
|
778
|
+
"ignoreFilterMemo",
|
|
779
|
+
];
|
|
780
|
+
function assertIngestSafe(props) {
|
|
781
|
+
for (const key of Object.keys(props)) {
|
|
782
|
+
const lower = key.toLowerCase();
|
|
783
|
+
expect(KEY_SHAPE.test(key), `key shape: ${key}`).toBe(true);
|
|
784
|
+
expect(lower.endsWith("count") || lower.endsWith("bytes") || lower.endsWith("ms"), `key suffix: ${key}`).toBe(true);
|
|
785
|
+
for (const part of DENY_PARTS) {
|
|
786
|
+
expect(lower.includes(part), `deny part "${part}" in ${key}`).toBe(false);
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
function fullCensus() {
|
|
791
|
+
const census = new HeapCensus();
|
|
792
|
+
for (const token of PRODUCTION_CENSUS_TOKENS)
|
|
793
|
+
census.register(token, () => 7);
|
|
794
|
+
return census;
|
|
795
|
+
}
|
|
796
|
+
it("heapSampleCloudProperties keys are all ingest-safe", () => {
|
|
797
|
+
const { readers } = mutableReaders();
|
|
798
|
+
const sample = sampleRunnerHeap({ readers, census: fullCensus(), sessionStartedAtMs: 0 });
|
|
799
|
+
assertIngestSafe(heapSampleCloudProperties(sample));
|
|
800
|
+
});
|
|
801
|
+
it("heapPressureCloudProperties keys are all ingest-safe", () => {
|
|
802
|
+
const census = fullCensus();
|
|
803
|
+
const { state, readers } = mutableReaders();
|
|
804
|
+
state.limit = 2048 * MB;
|
|
805
|
+
state.used = Math.round(2048 * MB * 0.8);
|
|
806
|
+
const controller = new RunnerHeapTelemetry({
|
|
807
|
+
readers,
|
|
808
|
+
census,
|
|
809
|
+
sessionStartedAtMs: 0,
|
|
810
|
+
pressureThresholdFraction: 0.75,
|
|
811
|
+
});
|
|
812
|
+
const { pressure } = controller.sample();
|
|
813
|
+
expect(pressure).not.toBeNull();
|
|
814
|
+
assertIngestSafe(heapPressureCloudProperties(pressure));
|
|
815
|
+
});
|
|
816
|
+
it("heapRecycleCloudProperties keys are all ingest-safe (both triggers)", () => {
|
|
817
|
+
const emergency = governor({ census: fullCensus() }).sampleAt(0.95, PAST_FLOOR_MS).recycle;
|
|
818
|
+
expect(emergency).not.toBeNull();
|
|
819
|
+
assertIngestSafe(heapRecycleCloudProperties(emergency));
|
|
820
|
+
const g = governor({ census: fullCensus() });
|
|
821
|
+
expect(g.observeAt(0.7, PAST_FLOOR_MS)).toBeNull();
|
|
822
|
+
const live = g.observeAt(0.7, PAST_FLOOR_MS + 121_000);
|
|
823
|
+
expect(live).not.toBeNull();
|
|
824
|
+
assertIngestSafe(heapRecycleCloudProperties(live));
|
|
825
|
+
});
|
|
826
|
+
});
|
|
827
|
+
describe("startRunnerHeapSampling — GC observer + recycle callback", () => {
|
|
828
|
+
it("drives the governor to a LIVE recycle via the injected GC observer and swallows a throwing onRecycle", () => {
|
|
588
829
|
const census = new HeapCensus();
|
|
589
830
|
census.register("x", () => 1);
|
|
590
831
|
const { state, readers } = mutableReaders();
|
|
591
|
-
state.limit =
|
|
592
|
-
state.used = 950; // 0.95 -> above both the 0.75 pressure and 0.78 recycle fractions
|
|
593
|
-
state.now = ONE_HOUR_MS; // past the floor
|
|
832
|
+
state.limit = 2048 * MB;
|
|
594
833
|
const fake = fakeScheduler();
|
|
834
|
+
const gc = fakeGcObserver();
|
|
595
835
|
const recycles = [];
|
|
596
|
-
startRunnerHeapSampling({
|
|
836
|
+
const handle = startRunnerHeapSampling({
|
|
597
837
|
readers,
|
|
598
838
|
census,
|
|
599
839
|
sessionStartedAtMs: 0,
|
|
600
840
|
pressureThresholdFraction: 0.75,
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
841
|
+
recycleLiveFraction: 0.6,
|
|
842
|
+
recycleSustainWindowMs: 120_000,
|
|
843
|
+
recycleEmergencyFraction: 0.9,
|
|
844
|
+
recycleMinUptimeMs: DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS,
|
|
604
845
|
intervalMs: 1,
|
|
605
846
|
scheduler: fake.scheduler,
|
|
847
|
+
gcObserver: gc.observer,
|
|
606
848
|
onSample: () => { },
|
|
607
849
|
onPressure: () => { },
|
|
608
850
|
onRecycle: (record) => {
|
|
@@ -611,15 +853,51 @@ describe("startRunnerHeapSampling — recycle callback", () => {
|
|
|
611
853
|
},
|
|
612
854
|
log: () => { },
|
|
613
855
|
});
|
|
614
|
-
|
|
615
|
-
|
|
856
|
+
expect(gc.started()).toBe(true);
|
|
857
|
+
// First major GC at/above the live floor: anchors, no verdict.
|
|
858
|
+
state.used = Math.round(state.limit * 0.7);
|
|
859
|
+
state.now = PAST_FLOOR_MS;
|
|
860
|
+
expect(() => gc.fire()).not.toThrow();
|
|
616
861
|
expect(recycles).toHaveLength(0);
|
|
617
|
-
//
|
|
618
|
-
|
|
862
|
+
// A window later: the second major GC produces the live recycle (throw swallowed).
|
|
863
|
+
state.now = PAST_FLOOR_MS + 121_000;
|
|
864
|
+
expect(() => gc.fire()).not.toThrow();
|
|
619
865
|
expect(recycles).toHaveLength(1);
|
|
620
|
-
|
|
621
|
-
|
|
866
|
+
expect(recycles[0].trigger).toBe("live");
|
|
867
|
+
// At most once per session, even as GCs continue.
|
|
868
|
+
state.now = PAST_FLOOR_MS + 300_000;
|
|
869
|
+
expect(() => gc.fire()).not.toThrow();
|
|
622
870
|
expect(recycles).toHaveLength(1);
|
|
871
|
+
handle.stop();
|
|
872
|
+
expect(gc.stopped()).toBe(true);
|
|
873
|
+
});
|
|
874
|
+
it("the loop-top evaluate() fires the emergency backstop when the interval timer never ticks", () => {
|
|
875
|
+
const census = new HeapCensus();
|
|
876
|
+
census.register("x", () => 1);
|
|
877
|
+
const { state, readers } = mutableReaders();
|
|
878
|
+
state.limit = 2048 * MB;
|
|
879
|
+
const fake = fakeScheduler();
|
|
880
|
+
const recycles = [];
|
|
881
|
+
const handle = startRunnerHeapSampling({
|
|
882
|
+
readers,
|
|
883
|
+
census,
|
|
884
|
+
sessionStartedAtMs: 0,
|
|
885
|
+
pressureThresholdFraction: 0.75,
|
|
886
|
+
recycleEmergencyFraction: 0.9,
|
|
887
|
+
intervalMs: 60_000,
|
|
888
|
+
scheduler: fake.scheduler,
|
|
889
|
+
onSample: () => { },
|
|
890
|
+
onPressure: () => { },
|
|
891
|
+
onRecycle: (record) => recycles.push(record),
|
|
892
|
+
log: () => { },
|
|
893
|
+
});
|
|
894
|
+
// The timer never ticked. An abort-adjacent instantaneous heap: the loop-top
|
|
895
|
+
// evaluate() catches it anyway.
|
|
896
|
+
state.used = Math.round(state.limit * 0.93);
|
|
897
|
+
state.now = 30_000;
|
|
898
|
+
handle.evaluate();
|
|
899
|
+
expect(recycles).toHaveLength(1);
|
|
900
|
+
expect(recycles[0].trigger).toBe("emergency");
|
|
623
901
|
});
|
|
624
902
|
it("produces no recycle when the governor is disabled by option", () => {
|
|
625
903
|
const census = new HeapCensus();
|
|
@@ -627,10 +905,11 @@ describe("startRunnerHeapSampling — recycle callback", () => {
|
|
|
627
905
|
const { state, readers } = mutableReaders();
|
|
628
906
|
state.limit = 1000;
|
|
629
907
|
state.used = 999;
|
|
630
|
-
state.now =
|
|
908
|
+
state.now = PAST_FLOOR_MS;
|
|
631
909
|
const fake = fakeScheduler();
|
|
910
|
+
const gc = fakeGcObserver();
|
|
632
911
|
const recycles = [];
|
|
633
|
-
startRunnerHeapSampling({
|
|
912
|
+
const handle = startRunnerHeapSampling({
|
|
634
913
|
readers,
|
|
635
914
|
census,
|
|
636
915
|
sessionStartedAtMs: 0,
|
|
@@ -639,16 +918,62 @@ describe("startRunnerHeapSampling — recycle callback", () => {
|
|
|
639
918
|
recycleMinUptimeMs: 0,
|
|
640
919
|
intervalMs: 1,
|
|
641
920
|
scheduler: fake.scheduler,
|
|
921
|
+
gcObserver: gc.observer,
|
|
642
922
|
onSample: () => { },
|
|
643
923
|
onPressure: () => { },
|
|
644
924
|
onRecycle: (record) => recycles.push(record),
|
|
645
925
|
log: () => { },
|
|
646
926
|
});
|
|
647
|
-
for (let i = 0; i < 10; i += 1)
|
|
927
|
+
for (let i = 0; i < 10; i += 1) {
|
|
648
928
|
fake.tick();
|
|
929
|
+
gc.fire();
|
|
930
|
+
handle.evaluate();
|
|
931
|
+
}
|
|
649
932
|
expect(recycles).toHaveLength(0);
|
|
650
933
|
});
|
|
651
934
|
});
|
|
935
|
+
describe("isFreshGcReading", () => {
|
|
936
|
+
it("accepts a promptly-captured reading and rejects a stale (blocked-loop) one", () => {
|
|
937
|
+
expect(isFreshGcReading(1000, 1000)).toBe(true); // captured at the GC end
|
|
938
|
+
expect(isFreshGcReading(1000, 1000 + MAX_FRESH_GC_READING_MS)).toBe(true); // exactly at the bound
|
|
939
|
+
expect(isFreshGcReading(1000, 1000 + MAX_FRESH_GC_READING_MS + 1)).toBe(false); // just over
|
|
940
|
+
expect(isFreshGcReading(1000, 5000)).toBe(false); // blocked loop: 4 s stale reading
|
|
941
|
+
expect(isFreshGcReading(1000, 900)).toBe(true); // reading at/just before the GC end
|
|
942
|
+
// Missing entry timing degrades to "not fresh" (skip), never a false accept.
|
|
943
|
+
expect(isFreshGcReading(Number.NaN, 1000)).toBe(false);
|
|
944
|
+
expect(isFreshGcReading(1000, Number.NaN)).toBe(false);
|
|
945
|
+
expect(isFreshGcReading(1000, 1400, 500)).toBe(true); // custom threshold honored
|
|
946
|
+
});
|
|
947
|
+
});
|
|
948
|
+
describe("nodeMajorGcObserver", () => {
|
|
949
|
+
it("constructs, observes major GCs, and disconnects without throwing", () => {
|
|
950
|
+
const observer = nodeMajorGcObserver();
|
|
951
|
+
let fires = 0;
|
|
952
|
+
expect(() => observer.start(() => (fires += 1))).not.toThrow();
|
|
953
|
+
// Force a real major GC if the runtime exposes it; otherwise this is a no-op
|
|
954
|
+
// and the observer simply never fires — both are acceptable (degrade to
|
|
955
|
+
// silence). The contract under test is that neither start nor stop throws.
|
|
956
|
+
const globalGc = globalThis.gc;
|
|
957
|
+
if (typeof globalGc === "function") {
|
|
958
|
+
try {
|
|
959
|
+
globalGc({ type: "major" });
|
|
960
|
+
}
|
|
961
|
+
catch {
|
|
962
|
+
globalGc();
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
expect(() => observer.stop()).not.toThrow();
|
|
966
|
+
expect(fires).toBeGreaterThanOrEqual(0);
|
|
967
|
+
});
|
|
968
|
+
it("start() twice and stop() before start() never throw (degrade to silence)", () => {
|
|
969
|
+
const observer = nodeMajorGcObserver();
|
|
970
|
+
expect(() => observer.stop()).not.toThrow(); // stop before start
|
|
971
|
+
expect(() => observer.start(() => { })).not.toThrow();
|
|
972
|
+
expect(() => observer.start(() => { })).not.toThrow(); // re-start
|
|
973
|
+
expect(() => observer.stop()).not.toThrow();
|
|
974
|
+
expect(() => observer.stop()).not.toThrow(); // double stop
|
|
975
|
+
});
|
|
976
|
+
});
|
|
652
977
|
describe("resolveHeapRecycleEnabled", () => {
|
|
653
978
|
it("is ON by default and disabled only by an explicit whole-string off value", () => {
|
|
654
979
|
expect(resolveHeapRecycleEnabled(undefined)).toBe(true);
|
|
@@ -663,33 +988,43 @@ describe("resolveHeapRecycleEnabled", () => {
|
|
|
663
988
|
expect(resolveHeapRecycleEnabled("disabled")).toBe(true);
|
|
664
989
|
});
|
|
665
990
|
});
|
|
666
|
-
describe("
|
|
667
|
-
it("parses a valid fraction and falls back to 0.
|
|
668
|
-
expect(
|
|
669
|
-
expect(
|
|
670
|
-
expect(
|
|
671
|
-
expect(
|
|
672
|
-
expect(
|
|
673
|
-
expect(
|
|
674
|
-
expect(
|
|
991
|
+
describe("resolveHeapRecycleLiveFraction", () => {
|
|
992
|
+
it("parses a valid fraction and falls back to 0.60 on garbage or out-of-range input", () => {
|
|
993
|
+
expect(resolveHeapRecycleLiveFraction("0.5")).toBe(0.5);
|
|
994
|
+
expect(resolveHeapRecycleLiveFraction("1")).toBe(1);
|
|
995
|
+
expect(resolveHeapRecycleLiveFraction(undefined)).toBe(DEFAULT_HEAP_RECYCLE_LIVE_FRACTION);
|
|
996
|
+
expect(resolveHeapRecycleLiveFraction("")).toBe(DEFAULT_HEAP_RECYCLE_LIVE_FRACTION);
|
|
997
|
+
expect(resolveHeapRecycleLiveFraction("2")).toBe(DEFAULT_HEAP_RECYCLE_LIVE_FRACTION);
|
|
998
|
+
expect(resolveHeapRecycleLiveFraction("-1")).toBe(DEFAULT_HEAP_RECYCLE_LIVE_FRACTION);
|
|
999
|
+
expect(resolveHeapRecycleLiveFraction("0.6x")).toBe(DEFAULT_HEAP_RECYCLE_LIVE_FRACTION);
|
|
1000
|
+
});
|
|
1001
|
+
});
|
|
1002
|
+
describe("resolveHeapRecycleEmergencyFraction", () => {
|
|
1003
|
+
it("parses a valid fraction and falls back to 0.90 on garbage or out-of-range input", () => {
|
|
1004
|
+
expect(resolveHeapRecycleEmergencyFraction("0.95")).toBe(0.95);
|
|
1005
|
+
expect(resolveHeapRecycleEmergencyFraction("1")).toBe(1);
|
|
1006
|
+
expect(resolveHeapRecycleEmergencyFraction(undefined)).toBe(DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION);
|
|
1007
|
+
expect(resolveHeapRecycleEmergencyFraction("")).toBe(DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION);
|
|
1008
|
+
expect(resolveHeapRecycleEmergencyFraction("2")).toBe(DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION);
|
|
1009
|
+
expect(resolveHeapRecycleEmergencyFraction("0.9x")).toBe(DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION);
|
|
675
1010
|
});
|
|
676
1011
|
});
|
|
677
|
-
describe("
|
|
678
|
-
it("parses
|
|
679
|
-
expect(
|
|
680
|
-
expect(
|
|
681
|
-
expect(
|
|
682
|
-
expect(
|
|
683
|
-
expect(
|
|
684
|
-
expect(
|
|
1012
|
+
describe("resolveHeapRecycleSustainWindowMs", () => {
|
|
1013
|
+
it("parses whole ms >= 0 (0 removes the window) and rejects '2m' rather than its prefix", () => {
|
|
1014
|
+
expect(resolveHeapRecycleSustainWindowMs("120000")).toBe(120_000);
|
|
1015
|
+
expect(resolveHeapRecycleSustainWindowMs("0")).toBe(0); // 0 is a valid window
|
|
1016
|
+
expect(resolveHeapRecycleSustainWindowMs(undefined)).toBe(DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS);
|
|
1017
|
+
expect(resolveHeapRecycleSustainWindowMs("2m")).toBe(DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS);
|
|
1018
|
+
expect(resolveHeapRecycleSustainWindowMs("-1")).toBe(DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS);
|
|
1019
|
+
expect(resolveHeapRecycleSustainWindowMs("")).toBe(DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS);
|
|
685
1020
|
});
|
|
686
1021
|
});
|
|
687
1022
|
describe("resolveHeapRecycleMinUptimeMs", () => {
|
|
688
|
-
it("parses whole ms >= 0 (0 disables the floor) and rejects '
|
|
1023
|
+
it("parses whole ms >= 0 (0 disables the floor) and rejects '35m' rather than its prefix", () => {
|
|
689
1024
|
expect(resolveHeapRecycleMinUptimeMs("60000")).toBe(60_000);
|
|
690
1025
|
expect(resolveHeapRecycleMinUptimeMs("0")).toBe(0); // 0 is a valid floor
|
|
691
1026
|
expect(resolveHeapRecycleMinUptimeMs(undefined)).toBe(DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS);
|
|
692
|
-
expect(resolveHeapRecycleMinUptimeMs("
|
|
1027
|
+
expect(resolveHeapRecycleMinUptimeMs("35m")).toBe(DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS);
|
|
693
1028
|
expect(resolveHeapRecycleMinUptimeMs("-1")).toBe(DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS);
|
|
694
1029
|
expect(resolveHeapRecycleMinUptimeMs("")).toBe(DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS);
|
|
695
1030
|
});
|