@indigoai-us/hq-cloud 6.16.39 → 6.16.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-heap-telemetry.d.ts +145 -9
- package/dist/bin/sync-runner-heap-telemetry.d.ts.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.js +178 -11
- package/dist/bin/sync-runner-heap-telemetry.js.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.test.js +511 -2
- package/dist/bin/sync-runner-heap-telemetry.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +11 -1
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +21 -0
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +16 -5
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, expect, it, vi } from "vitest";
|
|
2
|
-
import { HeapCensus, RunnerHeapTelemetry, sampleRunnerHeap, heapSampleCloudProperties, heapPressureCloudProperties, heapRecycleCloudProperties, heapPressureStderrLine, heapRecycleStderrLine, performHeapHardExit, startRunnerHeapSampling, nodeMajorGcObserver, isFreshGcReading, MAX_FRESH_GC_READING_MS, resolveHeapPressureFraction, resolveHeapSampleIntervalMs, resolveHeapRecycleEnabled, resolveHeapRecycleLiveFraction, resolveHeapRecycleEmergencyFraction, resolveHeapRecycleSustainWindowMs, resolveHeapRecycleMinUptimeMs, resolveHeapCheckpointMs, resolveHeapPinnedFraction, resolveHeapPinnedSustainMs, resolveHeapHardExitFraction, resolveHeapRecycleLatchHardExitMs, resolveDeclaredHeapCeilingBytes, DEFAULT_HEAP_RECYCLE_LIVE_FRACTION, DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION, DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS, DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS, DEFAULT_HEAP_CHECKPOINT_MS, DEFAULT_HEAP_PINNED_FRACTION, DEFAULT_HEAP_PINNED_SUSTAIN_MS, DEFAULT_HEAP_HARD_EXIT_FRACTION, DEFAULT_HEAP_RECYCLE_LATCH_HARD_EXIT_MS, } from "./sync-runner-heap-telemetry.js";
|
|
2
|
+
import { HeapCensus, RunnerHeapTelemetry, sampleRunnerHeap, heapSampleCloudProperties, heapPressureCloudProperties, heapRecycleCloudProperties, heapPressureStderrLine, heapRecycleStderrLine, performHeapHardExit, startRunnerHeapSampling, nodeMajorGcObserver, isFreshGcReading, MAX_FRESH_GC_READING_MS, resolveHeapPressureFraction, resolveHeapSampleIntervalMs, resolveHeapRecycleEnabled, resolveHeapRecycleLiveFraction, resolveHeapRecycleEmergencyFraction, resolveHeapRecycleSustainWindowMs, resolveHeapRecycleMinUptimeMs, resolveHeapCheckpointMs, resolveHeapPinnedFraction, resolveHeapPinnedSustainMs, resolveHeapHardExitFraction, resolveHeapRecycleLatchHardExitMs, resolveHeapFootprintRecycleEnabled, resolveHeapFootprintRecycleBytes, resolveHeapFootprintHardExitBytes, resolveDeclaredHeapCeilingBytes, DEFAULT_HEAP_RECYCLE_LIVE_FRACTION, DEFAULT_HEAP_RECYCLE_EMERGENCY_FRACTION, DEFAULT_HEAP_RECYCLE_SUSTAIN_WINDOW_MS, DEFAULT_HEAP_RECYCLE_MIN_UPTIME_MS, DEFAULT_HEAP_CHECKPOINT_MS, DEFAULT_HEAP_PINNED_FRACTION, DEFAULT_HEAP_PINNED_SUSTAIN_MS, DEFAULT_HEAP_HARD_EXIT_FRACTION, DEFAULT_HEAP_RECYCLE_LATCH_HARD_EXIT_MS, DEFAULT_HEAP_FOOTPRINT_RECYCLE_BYTES, DEFAULT_HEAP_FOOTPRINT_HARD_EXIT_BYTES, } from "./sync-runner-heap-telemetry.js";
|
|
3
3
|
import { emitCloudTelemetry } from "../telemetry-events.js";
|
|
4
4
|
function mutableReaders() {
|
|
5
5
|
const state = {
|
|
@@ -421,11 +421,28 @@ function governor(overrides = {}) {
|
|
|
421
421
|
...(overrides.latchHardExitObservations !== undefined
|
|
422
422
|
? { latchHardExitObservations: overrides.latchHardExitObservations }
|
|
423
423
|
: {}),
|
|
424
|
+
...(overrides.footprintRecycleEnabled !== undefined
|
|
425
|
+
? { footprintRecycleEnabled: overrides.footprintRecycleEnabled }
|
|
426
|
+
: {}),
|
|
427
|
+
...(overrides.footprintRecycleBytes !== undefined
|
|
428
|
+
? { footprintRecycleBytes: overrides.footprintRecycleBytes }
|
|
429
|
+
: {}),
|
|
430
|
+
...(overrides.footprintHardExitBytes !== undefined
|
|
431
|
+
? { footprintHardExitBytes: overrides.footprintHardExitBytes }
|
|
432
|
+
: {}),
|
|
424
433
|
});
|
|
425
434
|
const setAt = (fraction, nowMs) => {
|
|
426
435
|
state.used = Math.round(state.limit * fraction);
|
|
427
436
|
state.now = nowMs;
|
|
428
437
|
};
|
|
438
|
+
// Footprint driver: a HIGH resident rss (absolute bytes) with a QUIET heap
|
|
439
|
+
// (usedFraction default 0.30, below every heap floor), so only the footprint
|
|
440
|
+
// tier can produce a verdict.
|
|
441
|
+
const setFootprint = (rssBytes, nowMs, usedFraction = 0.3) => {
|
|
442
|
+
state.rss = rssBytes;
|
|
443
|
+
state.used = Math.round(state.limit * usedFraction);
|
|
444
|
+
state.now = nowMs;
|
|
445
|
+
};
|
|
429
446
|
// The cooperative recycle channel (backward-compatible with the shipped tests).
|
|
430
447
|
const observeAt = (fraction, nowMs) => {
|
|
431
448
|
setAt(fraction, nowMs);
|
|
@@ -454,6 +471,7 @@ function governor(overrides = {}) {
|
|
|
454
471
|
census,
|
|
455
472
|
controller,
|
|
456
473
|
setAt,
|
|
474
|
+
setFootprint,
|
|
457
475
|
observeAt,
|
|
458
476
|
sampleAt,
|
|
459
477
|
evaluateAt,
|
|
@@ -855,7 +873,7 @@ describe("ingest key contract — every emitted property key survives hq-pro red
|
|
|
855
873
|
expect(pressure).not.toBeNull();
|
|
856
874
|
assertIngestSafe(heapPressureCloudProperties(pressure));
|
|
857
875
|
});
|
|
858
|
-
it("heapRecycleCloudProperties keys are all ingest-safe (
|
|
876
|
+
it("heapRecycleCloudProperties keys are all ingest-safe (every trigger)", () => {
|
|
859
877
|
const emergency = governor({ census: fullCensus() }).sampleAt(0.95, PAST_FLOOR_MS).recycle;
|
|
860
878
|
expect(emergency).not.toBeNull();
|
|
861
879
|
assertIngestSafe(heapRecycleCloudProperties(emergency));
|
|
@@ -864,6 +882,19 @@ describe("ingest key contract — every emitted property key survives hq-pro red
|
|
|
864
882
|
const live = g.observeAt(0.7, PAST_FLOOR_MS + 121_000);
|
|
865
883
|
expect(live).not.toBeNull();
|
|
866
884
|
assertIngestSafe(heapRecycleCloudProperties(live));
|
|
885
|
+
// A footprint recycle carries the three NEW keys (trigger flag + two byte
|
|
886
|
+
// floors); they must satisfy the same contract or ingest would drop them.
|
|
887
|
+
const gFoot = governor({ census: fullCensus(), minUptimeMs: 0 });
|
|
888
|
+
gFoot.setFootprint(4600 * MB, 30_000); // > 4096 recycle floor, < 4864 hard floor
|
|
889
|
+
const footprint = gFoot.controller.evaluate().recycle;
|
|
890
|
+
expect(footprint?.trigger).toBe("footprint");
|
|
891
|
+
const footProps = heapRecycleCloudProperties(footprint);
|
|
892
|
+
assertIngestSafe(footProps);
|
|
893
|
+
expect(footProps.recycle_footprint_trigger_count).toBe(1);
|
|
894
|
+
expect(footProps.recycle_footprint_recycle_floor_bytes).toBe(DEFAULT_HEAP_FOOTPRINT_RECYCLE_BYTES);
|
|
895
|
+
expect(footProps.recycle_footprint_hard_exit_floor_bytes).toBe(DEFAULT_HEAP_FOOTPRINT_HARD_EXIT_BYTES);
|
|
896
|
+
// A heap trigger reports 0 for the footprint trigger flag.
|
|
897
|
+
expect(heapRecycleCloudProperties(live).recycle_footprint_trigger_count).toBe(0);
|
|
867
898
|
});
|
|
868
899
|
});
|
|
869
900
|
describe("startRunnerHeapSampling — GC observer + recycle callback", () => {
|
|
@@ -1239,6 +1270,484 @@ describe("RunnerHeapTelemetry — kill switch disables every tier (incl. hard)",
|
|
|
1239
1270
|
}
|
|
1240
1271
|
});
|
|
1241
1272
|
});
|
|
1273
|
+
// --- Resident-footprint tier (T5): the regression-reopen fix ------------------
|
|
1274
|
+
//
|
|
1275
|
+
// The four heap tiers denominate every fraction in min(declaredCeiling,
|
|
1276
|
+
// heap_size_limit) — the V8 old-space cap. A non-heap (external/ArrayBuffer)
|
|
1277
|
+
// burst outruns that cap while used_heap_size stays quiet, so NONE of them can
|
|
1278
|
+
// see a 5,598 MB resident process with a calm heap; the desktop's whole-tree
|
|
1279
|
+
// footprint backstop is left to pre-empt it, which IS this Sentry issue. The
|
|
1280
|
+
// footprint tier adds an absolute rss BUDGET the heap tiers never had.
|
|
1281
|
+
describe("RunnerHeapTelemetry — resident-footprint tier (T5)", () => {
|
|
1282
|
+
// hq-desktop-app crates/hq-desktop-core: WATCHER_FOOTPRINT_HARD_CEILING_MB.
|
|
1283
|
+
const DESKTOP_FOOTPRINT_HARD_CEILING_MB = 5120;
|
|
1284
|
+
// Max npx launcher sibling across the 8 decomposed events (8-84 MB observed).
|
|
1285
|
+
const LAUNCHER_HEADROOM_MB = 84;
|
|
1286
|
+
// Highest healthy cold-ramp tree peak that must NOT recycle.
|
|
1287
|
+
const HIGHEST_COLD_RAMP_MB = 3623;
|
|
1288
|
+
// Drive ONE footprint reading (rss in bytes, quiet heap) through a chosen path
|
|
1289
|
+
// on a fresh session, so the one-verdict-per-session latch never carries over.
|
|
1290
|
+
function footprintVerdict(rssBytes, path) {
|
|
1291
|
+
const g = governor({ minUptimeMs: 0 });
|
|
1292
|
+
g.setFootprint(rssBytes, 30_000);
|
|
1293
|
+
if (path === "sample") {
|
|
1294
|
+
const r = g.controller.sample();
|
|
1295
|
+
return { recycle: r.recycle, hardExit: r.hardExit };
|
|
1296
|
+
}
|
|
1297
|
+
if (path === "evaluate")
|
|
1298
|
+
return g.controller.evaluate();
|
|
1299
|
+
return g.controller.observeMajorGc();
|
|
1300
|
+
}
|
|
1301
|
+
it("pins the budgets from the production traces and hq-desktop-app's published constants", () => {
|
|
1302
|
+
expect(DEFAULT_HEAP_FOOTPRINT_RECYCLE_BYTES).toBe(4096 * MB);
|
|
1303
|
+
expect(DEFAULT_HEAP_FOOTPRINT_HARD_EXIT_BYTES).toBe(4864 * MB);
|
|
1304
|
+
// Both budgets recycle the runner's OWN rss STRICTLY below the desktop's
|
|
1305
|
+
// whole-tree pre-empt line (5120 MB) minus the launcher-sibling headroom.
|
|
1306
|
+
const preemptLineBytes = (DESKTOP_FOOTPRINT_HARD_CEILING_MB - LAUNCHER_HEADROOM_MB) * MB;
|
|
1307
|
+
expect(DEFAULT_HEAP_FOOTPRINT_RECYCLE_BYTES).toBeLessThan(preemptLineBytes);
|
|
1308
|
+
expect(DEFAULT_HEAP_FOOTPRINT_HARD_EXIT_BYTES).toBeLessThan(preemptLineBytes);
|
|
1309
|
+
// Cooperative < hard: a moderate climb recycles cooperatively; only an
|
|
1310
|
+
// abort-adjacent reading hard-exits.
|
|
1311
|
+
expect(DEFAULT_HEAP_FOOTPRINT_RECYCLE_BYTES).toBeLessThan(DEFAULT_HEAP_FOOTPRINT_HARD_EXIT_BYTES);
|
|
1312
|
+
// The cooperative floor sits above the highest healthy cold ramp, so a large
|
|
1313
|
+
// sync never trades this issue for respawn churn (the r1 failure mode).
|
|
1314
|
+
expect(DEFAULT_HEAP_FOOTPRINT_RECYCLE_BYTES).toBeGreaterThan(HIGHEST_COLD_RAMP_MB * MB);
|
|
1315
|
+
});
|
|
1316
|
+
it("replays the evented production traces: runaways recycle/hard-exit, cold ramps never do", () => {
|
|
1317
|
+
// The four cold ramps (healthy large syncs) must NEVER produce a verdict, on
|
|
1318
|
+
// ANY path. On BASE the tier does not exist and these produce no verdict
|
|
1319
|
+
// either — but neither do the runaways below, which is the base defect.
|
|
1320
|
+
for (const treeMb of [2776, 3080, 3337, 3623]) {
|
|
1321
|
+
for (const path of ["sample", "evaluate", "gc"]) {
|
|
1322
|
+
const v = footprintVerdict(treeMb * MB, path);
|
|
1323
|
+
expect(v.recycle, `cold ramp ${treeMb} via ${path}`).toBeNull();
|
|
1324
|
+
expect(v.hardExit, `cold ramp ${treeMb} via ${path}`).toBeNull();
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
// 4493 MB (below the hard floor): a COOPERATIVE footprint recycle.
|
|
1328
|
+
for (const path of ["sample", "evaluate", "gc"]) {
|
|
1329
|
+
const v = footprintVerdict(4493 * MB, path);
|
|
1330
|
+
expect(v.recycle?.trigger, `4493 via ${path}`).toBe("footprint");
|
|
1331
|
+
expect(v.recycle?.reason).toBe("cooperative");
|
|
1332
|
+
expect(v.recycle?.observedSustainMs).toBe(0); // single observation, no window
|
|
1333
|
+
expect(v.hardExit).toBeNull();
|
|
1334
|
+
}
|
|
1335
|
+
// 5598 and 7949 MB (at/above the hard floor, already past the desktop line on
|
|
1336
|
+
// the runner's own rss): a footprint HARD EXIT, no waiting for a pass boundary.
|
|
1337
|
+
for (const rssMb of [5598, 7949]) {
|
|
1338
|
+
for (const path of ["sample", "evaluate", "gc"]) {
|
|
1339
|
+
const v = footprintVerdict(rssMb * MB, path);
|
|
1340
|
+
expect(v.hardExit?.trigger, `${rssMb} via ${path}`).toBe("footprint");
|
|
1341
|
+
expect(v.hardExit?.reason).toBe("hard_fraction");
|
|
1342
|
+
expect(v.recycle).toBeNull();
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
});
|
|
1346
|
+
it("arms on a SINGLE observation with no sustain window and below the uptime floor", () => {
|
|
1347
|
+
// Default 35-minute uptime floor; a burst at 5 s uptime still recycles, so an
|
|
1348
|
+
// 83-1142 MB/s climb can never outrun the tier the way a sustain window would.
|
|
1349
|
+
const g = governor(); // default minUptimeMs = 35 min
|
|
1350
|
+
g.setFootprint(4200 * MB, 5_000);
|
|
1351
|
+
const { recycle } = g.controller.evaluate();
|
|
1352
|
+
expect(recycle?.trigger).toBe("footprint");
|
|
1353
|
+
expect(recycle?.reason).toBe("cooperative");
|
|
1354
|
+
expect(recycle?.observedSustainMs).toBe(0);
|
|
1355
|
+
expect(recycle?.uptimeMs).toBe(5_000);
|
|
1356
|
+
});
|
|
1357
|
+
it("carries rss/external/arrayBuffers, their growth deltas, and census attribution", () => {
|
|
1358
|
+
let entries = 100;
|
|
1359
|
+
const census = new HeapCensus();
|
|
1360
|
+
census.register("watcherJournalStores", () => entries);
|
|
1361
|
+
const g = governor({ census, minUptimeMs: 0 });
|
|
1362
|
+
g.controller.primeBaseline(); // baseline at entries=100, rss=0
|
|
1363
|
+
entries = 900;
|
|
1364
|
+
g.state.external = 40 * MB;
|
|
1365
|
+
g.state.arrayBuffers = 2000 * MB; // the non-heap growth that the heap cap misses
|
|
1366
|
+
g.setFootprint(4600 * MB, 30_000);
|
|
1367
|
+
const { recycle } = g.controller.evaluate();
|
|
1368
|
+
expect(recycle?.trigger).toBe("footprint");
|
|
1369
|
+
expect(recycle?.rssBytes).toBe(4600 * MB);
|
|
1370
|
+
expect(recycle?.rssGrowthBytes).toBe(4600 * MB);
|
|
1371
|
+
expect(recycle?.arrayBuffersGrowthBytes).toBe(2000 * MB);
|
|
1372
|
+
expect(recycle?.externalGrowthBytes).toBe(40 * MB);
|
|
1373
|
+
expect(recycle?.topSites[0]).toMatchObject({ site: "watcherJournalStores", growth: 800 });
|
|
1374
|
+
});
|
|
1375
|
+
it("adds a verdict only where the heap tiers are silent — a heap verdict always wins", () => {
|
|
1376
|
+
// Quiet heap + high rss -> footprint fires (heap silent).
|
|
1377
|
+
expect(footprintVerdict(4200 * MB, "evaluate").recycle?.trigger).toBe("footprint");
|
|
1378
|
+
// A heap emergency with rss ALSO over the footprint budget keeps trigger
|
|
1379
|
+
// "emergency": the heap tiers are evaluated first and their verdict is returned
|
|
1380
|
+
// unchanged, so the footprint tier never suppresses or re-denominates it.
|
|
1381
|
+
const g = governor({ minUptimeMs: 0 });
|
|
1382
|
+
g.state.rss = 9000 * MB;
|
|
1383
|
+
g.state.used = Math.round(g.state.limit * 0.95); // heap emergency line
|
|
1384
|
+
g.state.now = 30_000;
|
|
1385
|
+
expect(g.controller.evaluate().recycle?.trigger).toBe("emergency");
|
|
1386
|
+
// A heap HARD-fraction with rss over the footprint budget stays a heap hard
|
|
1387
|
+
// exit (trigger from the instantaneous family, not "footprint").
|
|
1388
|
+
const gh = governor({ minUptimeMs: 0 });
|
|
1389
|
+
gh.state.rss = 9000 * MB;
|
|
1390
|
+
gh.state.used = Math.round(gh.state.limit * 0.96); // hard-exit fraction
|
|
1391
|
+
gh.state.now = 30_000;
|
|
1392
|
+
const hv = gh.controller.evaluate();
|
|
1393
|
+
expect(hv.hardExit?.reason).toBe("hard_fraction");
|
|
1394
|
+
expect(hv.hardExit?.trigger).toBe("emergency"); // NOT footprint
|
|
1395
|
+
});
|
|
1396
|
+
it("fires from evaluate() (loop-top) when the interval timer never ticks and no GC is observed", () => {
|
|
1397
|
+
// The starved-sampler backstop: the module's own 11-minute-hole safety net now
|
|
1398
|
+
// covers footprint too. Only evaluate() is ever called here.
|
|
1399
|
+
const g = governor({ minUptimeMs: 0 });
|
|
1400
|
+
g.setFootprint(4200 * MB, 30_000);
|
|
1401
|
+
expect(g.controller.evaluate().recycle?.trigger).toBe("footprint");
|
|
1402
|
+
});
|
|
1403
|
+
it("hard-exits on a single reading at/above the footprint hard budget", () => {
|
|
1404
|
+
const g = governor({ minUptimeMs: 0 });
|
|
1405
|
+
g.setFootprint(DEFAULT_HEAP_FOOTPRINT_HARD_EXIT_BYTES, 30_000); // exactly the floor
|
|
1406
|
+
const { recycle, hardExit } = g.controller.evaluate();
|
|
1407
|
+
expect(recycle).toBeNull();
|
|
1408
|
+
expect(hardExit?.trigger).toBe("footprint");
|
|
1409
|
+
expect(hardExit?.reason).toBe("hard_fraction");
|
|
1410
|
+
expect(hardExit?.rssBytes).toBe(DEFAULT_HEAP_FOOTPRINT_HARD_EXIT_BYTES);
|
|
1411
|
+
});
|
|
1412
|
+
it("ages an unhonoured footprint cooperative latch into a footprint latch_timeout hard exit", () => {
|
|
1413
|
+
const g = governor({ minUptimeMs: 0, latchHardExitMs: 0, latchHardExitObservations: 1_000_000 });
|
|
1414
|
+
g.setFootprint(4200 * MB, 0); // cooperative footprint latch (below the hard floor)
|
|
1415
|
+
expect(g.controller.evaluate().recycle?.trigger).toBe("footprint");
|
|
1416
|
+
// latchHardExitMs=0: the next observation ages the latch out. The hard exit
|
|
1417
|
+
// inherits the footprint trigger from the latched tier.
|
|
1418
|
+
g.setFootprint(4200 * MB, 1);
|
|
1419
|
+
const late = g.controller.evaluate();
|
|
1420
|
+
expect(late.recycle).toBeNull();
|
|
1421
|
+
expect(late.hardExit?.reason).toBe("latch_timeout");
|
|
1422
|
+
expect(late.hardExit?.trigger).toBe("footprint");
|
|
1423
|
+
});
|
|
1424
|
+
it("recycles at most once per session across every path, then never again", () => {
|
|
1425
|
+
const g = governor({ minUptimeMs: 0 });
|
|
1426
|
+
g.setFootprint(4200 * MB, 0);
|
|
1427
|
+
expect(g.controller.evaluate().recycle?.trigger).toBe("footprint"); // the one recycle
|
|
1428
|
+
g.setFootprint(4600 * MB, 60_000);
|
|
1429
|
+
expect(g.controller.sample().recycle).toBeNull();
|
|
1430
|
+
g.setFootprint(4600 * MB, 120_000);
|
|
1431
|
+
expect(g.controller.observeMajorGc().recycle).toBeNull();
|
|
1432
|
+
});
|
|
1433
|
+
it("the footprint-specific disable silences ONLY the footprint tier; heap tiers still fire", () => {
|
|
1434
|
+
const g = governor({ footprintRecycleEnabled: false, minUptimeMs: 0 });
|
|
1435
|
+
g.setFootprint(9000 * MB, 30_000); // way over both budgets, quiet heap
|
|
1436
|
+
const off = g.controller.evaluate();
|
|
1437
|
+
expect(off.recycle).toBeNull();
|
|
1438
|
+
expect(off.hardExit).toBeNull();
|
|
1439
|
+
// A heap emergency on the same governor still fires — base behaviour intact.
|
|
1440
|
+
g.state.used = Math.round(g.state.limit * 0.95);
|
|
1441
|
+
g.state.now = 60_000;
|
|
1442
|
+
expect(g.controller.evaluate().recycle?.trigger).toBe("emergency");
|
|
1443
|
+
});
|
|
1444
|
+
it("the global kill switch also disables the footprint tier", () => {
|
|
1445
|
+
const g = governor({ recycleEnabled: false, minUptimeMs: 0 });
|
|
1446
|
+
g.setFootprint(9000 * MB, 30_000);
|
|
1447
|
+
const v = g.controller.evaluate();
|
|
1448
|
+
expect(v.recycle).toBeNull();
|
|
1449
|
+
expect(v.hardExit).toBeNull();
|
|
1450
|
+
});
|
|
1451
|
+
it("a disabled footprint tier reports 0 for both budget floors in the verdict record", () => {
|
|
1452
|
+
// A reader must be able to tell an OFF tier (0) from a live one (the budget).
|
|
1453
|
+
const g = governor({ footprintRecycleEnabled: false, minUptimeMs: 0 });
|
|
1454
|
+
const emg = g.sampleAt(0.95, PAST_FLOOR_MS).recycle; // heap emergency still fires
|
|
1455
|
+
expect(emg?.footprintRecycleBytes).toBe(0);
|
|
1456
|
+
expect(emg?.footprintHardExitBytes).toBe(0);
|
|
1457
|
+
// With the tier ON, a heap verdict still carries the configured budgets.
|
|
1458
|
+
const g2 = governor({ minUptimeMs: 0 });
|
|
1459
|
+
const emg2 = g2.sampleAt(0.95, PAST_FLOOR_MS).recycle;
|
|
1460
|
+
expect(emg2?.footprintRecycleBytes).toBe(DEFAULT_HEAP_FOOTPRINT_RECYCLE_BYTES);
|
|
1461
|
+
expect(emg2?.footprintHardExitBytes).toBe(DEFAULT_HEAP_FOOTPRINT_HARD_EXIT_BYTES);
|
|
1462
|
+
});
|
|
1463
|
+
it("honours a custom footprint budget and never fires just below it", () => {
|
|
1464
|
+
const g = governor({ footprintRecycleBytes: 3000 * MB, footprintHardExitBytes: 4000 * MB, minUptimeMs: 0 });
|
|
1465
|
+
g.setFootprint(2999 * MB, 10_000);
|
|
1466
|
+
expect(g.controller.evaluate().recycle).toBeNull(); // 1 MB under the floor
|
|
1467
|
+
g.setFootprint(3000 * MB, 20_000);
|
|
1468
|
+
expect(g.controller.evaluate().recycle?.trigger).toBe("footprint"); // exactly the floor
|
|
1469
|
+
});
|
|
1470
|
+
it("a disabled footprint tier adds NO rss read, so a throwing memoryUsage never breaks a quiet-heap observation", () => {
|
|
1471
|
+
// Regression: the kill switch must restore the exact prior observation path.
|
|
1472
|
+
// With the tier off and a quiet heap (no verdict), evaluate()/observeMajorGc()
|
|
1473
|
+
// must not touch memoryUsage() at all — so a throwing reader cannot swallow the
|
|
1474
|
+
// observation before the heap tiers run.
|
|
1475
|
+
const census = new HeapCensus();
|
|
1476
|
+
census.register("x", () => 1);
|
|
1477
|
+
let memReads = 0;
|
|
1478
|
+
const readers = {
|
|
1479
|
+
getHeapStatistics: () => ({
|
|
1480
|
+
used_heap_size: Math.round(2048 * MB * 0.3), // quiet heap: no heap verdict
|
|
1481
|
+
total_heap_size: 0,
|
|
1482
|
+
heap_size_limit: 2048 * MB,
|
|
1483
|
+
external_memory: 0,
|
|
1484
|
+
}),
|
|
1485
|
+
memoryUsage: () => {
|
|
1486
|
+
memReads += 1;
|
|
1487
|
+
throw new Error("memoryUsage boom");
|
|
1488
|
+
},
|
|
1489
|
+
monotonicNowMs: () => 30_000,
|
|
1490
|
+
};
|
|
1491
|
+
const controller = new RunnerHeapTelemetry({
|
|
1492
|
+
readers,
|
|
1493
|
+
census,
|
|
1494
|
+
sessionStartedAtMs: 0,
|
|
1495
|
+
pressureThresholdFraction: 0.75,
|
|
1496
|
+
recycleMinUptimeMs: 0,
|
|
1497
|
+
footprintRecycleEnabled: false,
|
|
1498
|
+
});
|
|
1499
|
+
expect(() => controller.evaluate()).not.toThrow();
|
|
1500
|
+
expect(() => controller.observeMajorGc()).not.toThrow();
|
|
1501
|
+
expect(controller.evaluate().recycle).toBeNull();
|
|
1502
|
+
expect(controller.observeMajorGc().recycle).toBeNull();
|
|
1503
|
+
expect(memReads).toBe(0); // the disabled tier read rss zero times
|
|
1504
|
+
});
|
|
1505
|
+
it("records the rss value that CROSSED the budget, not a later drifted re-read", () => {
|
|
1506
|
+
// The decision reads rss once; the verdict sample re-reads memoryUsage a hair
|
|
1507
|
+
// later. A reader whose rss drops between the two reads must not make the
|
|
1508
|
+
// durable record claim an rss below the floor that triggered the hard exit.
|
|
1509
|
+
const census = new HeapCensus();
|
|
1510
|
+
census.register("x", () => 1);
|
|
1511
|
+
const rssSequence = [5000 * MB, 100 * MB]; // decision sees 5000 (>= hard floor); re-read drifts to 100
|
|
1512
|
+
let i = 0;
|
|
1513
|
+
const readers = {
|
|
1514
|
+
getHeapStatistics: () => ({
|
|
1515
|
+
used_heap_size: Math.round(2048 * MB * 0.3), // quiet heap
|
|
1516
|
+
total_heap_size: 0,
|
|
1517
|
+
heap_size_limit: 2048 * MB,
|
|
1518
|
+
external_memory: 0,
|
|
1519
|
+
}),
|
|
1520
|
+
memoryUsage: () => ({
|
|
1521
|
+
rss: rssSequence[Math.min(i++, rssSequence.length - 1)],
|
|
1522
|
+
heapTotal: 0,
|
|
1523
|
+
heapUsed: 0,
|
|
1524
|
+
external: 0,
|
|
1525
|
+
arrayBuffers: 0,
|
|
1526
|
+
}),
|
|
1527
|
+
monotonicNowMs: () => 30_000,
|
|
1528
|
+
};
|
|
1529
|
+
const controller = new RunnerHeapTelemetry({
|
|
1530
|
+
readers,
|
|
1531
|
+
census,
|
|
1532
|
+
sessionStartedAtMs: 0,
|
|
1533
|
+
pressureThresholdFraction: 0.75,
|
|
1534
|
+
recycleMinUptimeMs: 0,
|
|
1535
|
+
});
|
|
1536
|
+
const { hardExit } = controller.evaluate();
|
|
1537
|
+
expect(hardExit?.trigger).toBe("footprint");
|
|
1538
|
+
expect(hardExit?.reason).toBe("hard_fraction");
|
|
1539
|
+
// Stamped with the checked 5000 MB, never the drifted 100 MB re-read.
|
|
1540
|
+
expect(hardExit?.rssBytes).toBe(5000 * MB);
|
|
1541
|
+
expect(hardExit?.rssBytes).toBeGreaterThanOrEqual(DEFAULT_HEAP_FOOTPRINT_HARD_EXIT_BYTES);
|
|
1542
|
+
});
|
|
1543
|
+
it("the footprint recycle stderr line matches NONE of hq-desktop-app's fatal markers", () => {
|
|
1544
|
+
const DESKTOP_FATAL_MARKERS = [
|
|
1545
|
+
"javascript heap out of memory",
|
|
1546
|
+
"ineffective mark-compacts",
|
|
1547
|
+
"fatal error",
|
|
1548
|
+
"uncaught exception",
|
|
1549
|
+
"unhandledrejection",
|
|
1550
|
+
"panicked at",
|
|
1551
|
+
"assertion failed",
|
|
1552
|
+
"npm error ",
|
|
1553
|
+
"npm err! ",
|
|
1554
|
+
"enospc",
|
|
1555
|
+
"permission denied",
|
|
1556
|
+
"no such file or directory",
|
|
1557
|
+
"command not found",
|
|
1558
|
+
];
|
|
1559
|
+
const g = governor({ minUptimeMs: 0 });
|
|
1560
|
+
g.setFootprint(4200 * MB, 30_000);
|
|
1561
|
+
const line = heapRecycleStderrLine(g.controller.evaluate().recycle);
|
|
1562
|
+
const lowered = line.toLowerCase();
|
|
1563
|
+
for (const marker of DESKTOP_FATAL_MARKERS)
|
|
1564
|
+
expect(lowered).not.toContain(marker);
|
|
1565
|
+
expect(line.trimStart().startsWith("#")).toBe(false);
|
|
1566
|
+
expect(line).toContain("trigger=footprint");
|
|
1567
|
+
});
|
|
1568
|
+
});
|
|
1569
|
+
describe("startRunnerHeapSampling — footprint tier", () => {
|
|
1570
|
+
it("drives a footprint cooperative recycle through the interval tick and swallows a throwing onRecycle", () => {
|
|
1571
|
+
const census = new HeapCensus();
|
|
1572
|
+
census.register("x", () => 1);
|
|
1573
|
+
const { state, readers } = mutableReaders();
|
|
1574
|
+
state.limit = 2048 * MB;
|
|
1575
|
+
state.used = Math.round(state.limit * 0.3); // quiet heap
|
|
1576
|
+
state.rss = 4300 * MB; // over the 4096 MB recycle floor, under the hard floor
|
|
1577
|
+
const fake = fakeScheduler();
|
|
1578
|
+
const recycles = [];
|
|
1579
|
+
startRunnerHeapSampling({
|
|
1580
|
+
readers,
|
|
1581
|
+
census,
|
|
1582
|
+
sessionStartedAtMs: 0,
|
|
1583
|
+
pressureThresholdFraction: 0.99,
|
|
1584
|
+
recycleMinUptimeMs: 0,
|
|
1585
|
+
intervalMs: 60_000,
|
|
1586
|
+
scheduler: fake.scheduler,
|
|
1587
|
+
onSample: () => { },
|
|
1588
|
+
onPressure: () => { },
|
|
1589
|
+
onRecycle: (record) => {
|
|
1590
|
+
recycles.push(record);
|
|
1591
|
+
throw new Error("recycle emit boom"); // must be swallowed
|
|
1592
|
+
},
|
|
1593
|
+
log: () => { },
|
|
1594
|
+
});
|
|
1595
|
+
state.now = 30_000;
|
|
1596
|
+
expect(() => fake.tick()).not.toThrow();
|
|
1597
|
+
expect(recycles).toHaveLength(1);
|
|
1598
|
+
expect(recycles[0].trigger).toBe("footprint");
|
|
1599
|
+
});
|
|
1600
|
+
it("delivers a footprint hard exit to onHardExit (never onRecycle)", () => {
|
|
1601
|
+
const census = new HeapCensus();
|
|
1602
|
+
census.register("x", () => 1);
|
|
1603
|
+
const { state, readers } = mutableReaders();
|
|
1604
|
+
state.limit = 2048 * MB;
|
|
1605
|
+
state.used = Math.round(state.limit * 0.3); // quiet heap
|
|
1606
|
+
state.rss = 5000 * MB; // over the 4864 MB hard floor
|
|
1607
|
+
const fake = fakeScheduler();
|
|
1608
|
+
const recycles = [];
|
|
1609
|
+
const hardExits = [];
|
|
1610
|
+
const handle = startRunnerHeapSampling({
|
|
1611
|
+
readers,
|
|
1612
|
+
census,
|
|
1613
|
+
sessionStartedAtMs: 0,
|
|
1614
|
+
pressureThresholdFraction: 0.99,
|
|
1615
|
+
recycleMinUptimeMs: 0,
|
|
1616
|
+
intervalMs: 60_000,
|
|
1617
|
+
checkpointMs: 10,
|
|
1618
|
+
scheduler: fake.scheduler,
|
|
1619
|
+
onSample: () => { },
|
|
1620
|
+
onPressure: () => { },
|
|
1621
|
+
onRecycle: (record) => recycles.push(record),
|
|
1622
|
+
onHardExit: (record) => hardExits.push(record),
|
|
1623
|
+
log: () => { },
|
|
1624
|
+
});
|
|
1625
|
+
state.now = 0;
|
|
1626
|
+
handle.maybeObserve();
|
|
1627
|
+
expect(hardExits).toHaveLength(1);
|
|
1628
|
+
expect(hardExits[0].trigger).toBe("footprint");
|
|
1629
|
+
expect(hardExits[0].reason).toBe("hard_fraction");
|
|
1630
|
+
expect(recycles).toHaveLength(0);
|
|
1631
|
+
handle.stop();
|
|
1632
|
+
});
|
|
1633
|
+
it("the loop-top evaluate() fires the footprint tier when the interval timer never ticks", () => {
|
|
1634
|
+
const census = new HeapCensus();
|
|
1635
|
+
census.register("x", () => 1);
|
|
1636
|
+
const { state, readers } = mutableReaders();
|
|
1637
|
+
state.limit = 2048 * MB;
|
|
1638
|
+
state.used = Math.round(state.limit * 0.3);
|
|
1639
|
+
state.rss = 4300 * MB;
|
|
1640
|
+
const fake = fakeScheduler();
|
|
1641
|
+
const recycles = [];
|
|
1642
|
+
const handle = startRunnerHeapSampling({
|
|
1643
|
+
readers,
|
|
1644
|
+
census,
|
|
1645
|
+
sessionStartedAtMs: 0,
|
|
1646
|
+
pressureThresholdFraction: 0.99,
|
|
1647
|
+
recycleMinUptimeMs: 0,
|
|
1648
|
+
intervalMs: 60_000,
|
|
1649
|
+
scheduler: fake.scheduler,
|
|
1650
|
+
onSample: () => { },
|
|
1651
|
+
onPressure: () => { },
|
|
1652
|
+
onRecycle: (record) => recycles.push(record),
|
|
1653
|
+
log: () => { },
|
|
1654
|
+
});
|
|
1655
|
+
state.now = 30_000;
|
|
1656
|
+
handle.evaluate(); // timer never ticked
|
|
1657
|
+
expect(recycles).toHaveLength(1);
|
|
1658
|
+
expect(recycles[0].trigger).toBe("footprint");
|
|
1659
|
+
handle.stop();
|
|
1660
|
+
});
|
|
1661
|
+
it("a throwing memoryUsage reader degrades to silence on every path (never throws into a watch pass)", () => {
|
|
1662
|
+
const census = new HeapCensus();
|
|
1663
|
+
census.register("x", () => 1);
|
|
1664
|
+
const readers = {
|
|
1665
|
+
getHeapStatistics: () => ({
|
|
1666
|
+
used_heap_size: 10,
|
|
1667
|
+
total_heap_size: 20,
|
|
1668
|
+
heap_size_limit: 2048 * MB,
|
|
1669
|
+
external_memory: 0,
|
|
1670
|
+
}),
|
|
1671
|
+
memoryUsage: () => {
|
|
1672
|
+
throw new Error("memoryUsage boom");
|
|
1673
|
+
},
|
|
1674
|
+
monotonicNowMs: () => 30_000,
|
|
1675
|
+
};
|
|
1676
|
+
const fake = fakeScheduler();
|
|
1677
|
+
const gc = fakeGcObserver();
|
|
1678
|
+
const handle = startRunnerHeapSampling({
|
|
1679
|
+
readers,
|
|
1680
|
+
census,
|
|
1681
|
+
sessionStartedAtMs: 0,
|
|
1682
|
+
pressureThresholdFraction: 0.99,
|
|
1683
|
+
recycleMinUptimeMs: 0,
|
|
1684
|
+
intervalMs: 60_000,
|
|
1685
|
+
scheduler: fake.scheduler,
|
|
1686
|
+
gcObserver: gc.observer,
|
|
1687
|
+
onSample: () => { },
|
|
1688
|
+
onPressure: () => { },
|
|
1689
|
+
onRecycle: () => { },
|
|
1690
|
+
onHardExit: () => { },
|
|
1691
|
+
log: () => { },
|
|
1692
|
+
});
|
|
1693
|
+
expect(() => fake.tick()).not.toThrow();
|
|
1694
|
+
expect(() => handle.evaluate()).not.toThrow();
|
|
1695
|
+
expect(() => gc.fire()).not.toThrow();
|
|
1696
|
+
handle.stop();
|
|
1697
|
+
});
|
|
1698
|
+
it("a footprint value in the record survives the LOCAL telemetry sanitizer as numbers only", async () => {
|
|
1699
|
+
const g = governor({ minUptimeMs: 0 });
|
|
1700
|
+
g.setFootprint(4300 * MB, 30_000);
|
|
1701
|
+
const record = g.controller.evaluate().recycle;
|
|
1702
|
+
const props = heapRecycleCloudProperties(record);
|
|
1703
|
+
for (const value of Object.values(props))
|
|
1704
|
+
expect(typeof value).toBe("number");
|
|
1705
|
+
const captured = [];
|
|
1706
|
+
const client = {
|
|
1707
|
+
postTelemetryEvents: async (batch) => {
|
|
1708
|
+
for (const event of batch.events)
|
|
1709
|
+
captured.push(event);
|
|
1710
|
+
return { accepted: batch.events.length };
|
|
1711
|
+
},
|
|
1712
|
+
};
|
|
1713
|
+
await emitCloudTelemetry(client, {
|
|
1714
|
+
eventName: "sync_runner_heap_recycle",
|
|
1715
|
+
source: "sync-runner-heap",
|
|
1716
|
+
properties: props,
|
|
1717
|
+
});
|
|
1718
|
+
const survived = captured[0]?.properties ?? {};
|
|
1719
|
+
expect(survived.recycle_footprint_trigger_count).toBe(1);
|
|
1720
|
+
expect(Object.keys(survived).length).toBe(Object.keys(props).length);
|
|
1721
|
+
});
|
|
1722
|
+
});
|
|
1723
|
+
describe("resolve* parsers for the footprint knobs", () => {
|
|
1724
|
+
it("resolveHeapFootprintRecycleEnabled: ON by default, disabled only by an explicit off value", () => {
|
|
1725
|
+
expect(resolveHeapFootprintRecycleEnabled(undefined)).toBe(true);
|
|
1726
|
+
expect(resolveHeapFootprintRecycleEnabled("")).toBe(true);
|
|
1727
|
+
expect(resolveHeapFootprintRecycleEnabled("1")).toBe(true);
|
|
1728
|
+
expect(resolveHeapFootprintRecycleEnabled("0")).toBe(false);
|
|
1729
|
+
expect(resolveHeapFootprintRecycleEnabled("false")).toBe(false);
|
|
1730
|
+
expect(resolveHeapFootprintRecycleEnabled(" OFF ")).toBe(false);
|
|
1731
|
+
// A typo keeps the tier armed rather than silently disabling protection.
|
|
1732
|
+
expect(resolveHeapFootprintRecycleEnabled("disabled")).toBe(true);
|
|
1733
|
+
});
|
|
1734
|
+
it("resolveHeapFootprintRecycleBytes: whole MB -> bytes, 4096 MB default, rejects a numeric prefix", () => {
|
|
1735
|
+
expect(resolveHeapFootprintRecycleBytes("4096")).toBe(4096 * MB);
|
|
1736
|
+
expect(resolveHeapFootprintRecycleBytes("1000")).toBe(1000 * MB);
|
|
1737
|
+
expect(resolveHeapFootprintRecycleBytes(undefined)).toBe(DEFAULT_HEAP_FOOTPRINT_RECYCLE_BYTES);
|
|
1738
|
+
expect(resolveHeapFootprintRecycleBytes("")).toBe(DEFAULT_HEAP_FOOTPRINT_RECYCLE_BYTES);
|
|
1739
|
+
expect(resolveHeapFootprintRecycleBytes("0")).toBe(DEFAULT_HEAP_FOOTPRINT_RECYCLE_BYTES);
|
|
1740
|
+
// A malformed value can NEVER silently lower the budget or disable the tier.
|
|
1741
|
+
expect(resolveHeapFootprintRecycleBytes("4096x")).toBe(DEFAULT_HEAP_FOOTPRINT_RECYCLE_BYTES);
|
|
1742
|
+
expect(resolveHeapFootprintRecycleBytes("4gb")).toBe(DEFAULT_HEAP_FOOTPRINT_RECYCLE_BYTES);
|
|
1743
|
+
});
|
|
1744
|
+
it("resolveHeapFootprintHardExitBytes: whole MB -> bytes, 4864 MB default, rejects garbage", () => {
|
|
1745
|
+
expect(resolveHeapFootprintHardExitBytes("4864")).toBe(4864 * MB);
|
|
1746
|
+
expect(resolveHeapFootprintHardExitBytes(undefined)).toBe(DEFAULT_HEAP_FOOTPRINT_HARD_EXIT_BYTES);
|
|
1747
|
+
expect(resolveHeapFootprintHardExitBytes("0")).toBe(DEFAULT_HEAP_FOOTPRINT_HARD_EXIT_BYTES);
|
|
1748
|
+
expect(resolveHeapFootprintHardExitBytes("4864x")).toBe(DEFAULT_HEAP_FOOTPRINT_HARD_EXIT_BYTES);
|
|
1749
|
+
});
|
|
1750
|
+
});
|
|
1242
1751
|
describe("startRunnerHeapSampling — work-driven checkpoint gate", () => {
|
|
1243
1752
|
function pinnedHandle(overrides) {
|
|
1244
1753
|
const census = new HeapCensus();
|