@camstack/system 1.2.145 → 1.2.147
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-runner.js +2 -1
- package/dist/addon-runner.mjs +2 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +1 -1
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
- package/dist/builtins/device-manager/device-manager.addon.js +2 -2
- package/dist/builtins/device-manager/device-manager.addon.mjs +2 -2
- package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +1 -1
- package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
- package/dist/builtins/local-network/local-network.addon.js +1 -1
- package/dist/builtins/local-network/local-network.addon.mjs +1 -1
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/load-series-config.d.ts +74 -0
- package/dist/builtins/native-metrics/load-series-reader.d.ts +37 -0
- package/dist/builtins/native-metrics/load-series-store.d.ts +232 -0
- package/dist/builtins/native-metrics/native-metrics.addon.d.ts +118 -35
- package/dist/builtins/native-metrics/native-metrics.addon.js +1463 -199
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1463 -199
- package/dist/builtins/native-metrics/node-load-ring.d.ts +214 -0
- package/dist/builtins/native-metrics/process-classification.d.ts +51 -0
- package/dist/builtins/native-metrics/thread-cpu-sampler.d.ts +116 -0
- package/dist/builtins/platform-probe/index.js +1 -1
- package/dist/builtins/platform-probe/index.mjs +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +1 -1
- package/dist/builtins/snapshot/index.mjs +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings-backend.d.ts +25 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +0 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +0 -0
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +2 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +2 -1
- package/dist/builtins/system-config/system-config.addon.js +1 -1
- package/dist/builtins/system-config/system-config.addon.mjs +1 -1
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{dist-BVU5JADq.js → dist-Dl6MFXPr.js} +496 -35
- package/dist/{dist-CG7kwIZp.mjs → dist-DrBc-7Y3.mjs} +479 -36
- package/dist/index.d.ts +2 -2
- package/dist/index.js +45 -5
- package/dist/index.mjs +37 -6
- package/dist/kernel/heap-watch.d.ts +209 -1
- package/dist/{manifest-python-deps-BRyNCnnG.mjs → manifest-python-deps-DhT0oFOZ.mjs} +200 -8
- package/dist/{manifest-python-deps-GjlyPjm0.js → manifest-python-deps-FYZBHc3v.js} +253 -7
- package/dist/{retired-settings-keys-_PLI9w0k.js → retired-settings-keys-6w_JOqBg.js} +1 -1
- package/dist/{retired-settings-keys-Dm1z4mO4.mjs → retired-settings-keys-CF5r4YKq.mjs} +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import "./dist-
|
|
1
|
+
import "./dist-DrBc-7Y3.mjs";
|
|
2
2
|
import * as crypto$1 from "node:crypto";
|
|
3
3
|
import { randomBytes, randomUUID } from "node:crypto";
|
|
4
4
|
import { ensureBinary, ensureFfmpeg, ensurePython, installPythonPackages, installPythonRequirements } from "@camstack/types/node";
|
|
@@ -262,6 +262,22 @@ var consoleSink = {
|
|
|
262
262
|
* The subtrahend is heapUSED, not heapTotal, deliberately: the pages V8
|
|
263
263
|
* committed at a sawtooth peak and no longer needs are counted inside heapTotal,
|
|
264
264
|
* and they are precisely what this number exists to see.
|
|
265
|
+
*
|
|
266
|
+
* ## What this number can and CANNOT be asked
|
|
267
|
+
*
|
|
268
|
+
* It answers exactly one question: *are there pages a `last-resort` GC could
|
|
269
|
+
* hand back?* For that it is correct, and {@link shouldReclaim} is the only
|
|
270
|
+
* gate that should ever be built on it.
|
|
271
|
+
*
|
|
272
|
+
* It can never answer *is this process too big?*, because `external` is
|
|
273
|
+
* subtracted by construction. A process whose growth is entirely `external` —
|
|
274
|
+
* Buffers, `arrayBuffers`, what a native addon mallocs — moves `rss` and the
|
|
275
|
+
* subtrahend by the same amount, so this number is **algebraically constant
|
|
276
|
+
* while the process triples**. That is not a threshold set too high; the
|
|
277
|
+
* quantity cannot represent the event.
|
|
278
|
+
*
|
|
279
|
+
* It cost 11.4GB and a ~154 second stall of the whole container to establish
|
|
280
|
+
* that. See {@link RssBudgetOptions} for the number that does answer it.
|
|
265
281
|
*/
|
|
266
282
|
function strandedMb(s) {
|
|
267
283
|
return s.rssMb - s.heapUsedMb - s.externalMb;
|
|
@@ -270,6 +286,67 @@ function strandedMb(s) {
|
|
|
270
286
|
function shouldReclaim(s, triggerMb = HEAP_RECLAIM_TRIGGER_MB) {
|
|
271
287
|
return strandedMb(s) > triggerMb;
|
|
272
288
|
}
|
|
289
|
+
/** Share of RSS, as a whole percent. Never divides by zero. */
|
|
290
|
+
function share(partMb, rssMb) {
|
|
291
|
+
return rssMb > 0 ? Math.round(partMb / rssMb * 100) : 0;
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Split a sample into the three shapes and name the dominant one.
|
|
295
|
+
*
|
|
296
|
+
* `nativeResidue` is clamped at 0: `process.memoryUsage()` reads its fields at
|
|
297
|
+
* slightly different instants, so `heapUsed + external` can transiently exceed
|
|
298
|
+
* `rss` and a negative residue would misclassify the sample.
|
|
299
|
+
*/
|
|
300
|
+
function describeRss(s, budgetMb) {
|
|
301
|
+
const nativeResidueMb = Math.max(0, strandedMb(s));
|
|
302
|
+
const shape = s.heapUsedMb >= s.externalMb && s.heapUsedMb >= nativeResidueMb ? "v8-heap" : s.externalMb >= nativeResidueMb ? "external" : "native-residue";
|
|
303
|
+
return {
|
|
304
|
+
rssMb: s.rssMb,
|
|
305
|
+
budgetMb,
|
|
306
|
+
overMb: Math.max(0, s.rssMb - budgetMb),
|
|
307
|
+
heapUsedMb: s.heapUsedMb,
|
|
308
|
+
externalMb: s.externalMb,
|
|
309
|
+
nativeResidueMb,
|
|
310
|
+
arrayBuffersMb: s.arrayBuffersMb,
|
|
311
|
+
shape
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
/** What the operator can actually DO about each shape. */
|
|
315
|
+
var SHAPE_ADVICE = {
|
|
316
|
+
"v8-heap": "live JS retainers — bounded by --max-old-space-size; the fix is the retained set",
|
|
317
|
+
external: "Buffers / native external — NO V8 flag bounds this, and the old-space ceiling never will",
|
|
318
|
+
"native-residue": "freed native memory the allocator has not returned — MALLOC_ARENA_MAX/VIPS_CONCURRENCY are already applied, so this lever is spent"
|
|
319
|
+
};
|
|
320
|
+
/**
|
|
321
|
+
* Release the alarm below this share of the budget.
|
|
322
|
+
*
|
|
323
|
+
* Hysteresis, not a second budget: every process here sawtooths (the reclaim
|
|
324
|
+
* pass alone moves hub-main by ~1.6GB), so a single threshold would print a
|
|
325
|
+
* pair of lines on every cycle. 0.9 of the budget is a real retreat.
|
|
326
|
+
*/
|
|
327
|
+
var RSS_BUDGET_RELEASE_RATIO = .9;
|
|
328
|
+
/**
|
|
329
|
+
* Floor on how often a crossing may be ANNOUNCED again.
|
|
330
|
+
*
|
|
331
|
+
* Hysteresis alone bounds flapping around the edge; it does not bound a process
|
|
332
|
+
* that genuinely swings across the budget every reclaim cycle
|
|
333
|
+
* ({@link HEAP_RECLAIM_MIN_INTERVAL_MS} is 2 minutes, so that is up to 30 pairs
|
|
334
|
+
* of lines an hour, forever). A process that oscillates across its budget is ONE
|
|
335
|
+
* story, and the operator needs it told once per quarter-hour, not thirty times.
|
|
336
|
+
*
|
|
337
|
+
* This is the discipline the rest of this module already follows and the one the
|
|
338
|
+
* log needed: on 2026-08-27 two real alarms were buried under 83,198 Hikvision
|
|
339
|
+
* heartbeat lines in 12h. A watchdog that shouts is a watchdog that gets muted.
|
|
340
|
+
*/
|
|
341
|
+
var RSS_BUDGET_REANNOUNCE_MIN_MS = 9e5;
|
|
342
|
+
/**
|
|
343
|
+
* The alarm state machine, pure so the thresholds are testable without
|
|
344
|
+
* allocating five gigabytes to reach them.
|
|
345
|
+
*/
|
|
346
|
+
function nextRssBudgetState(current, rssMb, budgetMb, releaseRatio = RSS_BUDGET_RELEASE_RATIO) {
|
|
347
|
+
if (current === "over") return rssMb < budgetMb * releaseRatio ? "within" : "over";
|
|
348
|
+
return rssMb > budgetMb ? "over" : "within";
|
|
349
|
+
}
|
|
273
350
|
function isGcFunction(value) {
|
|
274
351
|
return typeof value === "function";
|
|
275
352
|
}
|
|
@@ -297,8 +374,19 @@ function createV8Reclaimer() {
|
|
|
297
374
|
return;
|
|
298
375
|
}
|
|
299
376
|
}
|
|
300
|
-
|
|
301
|
-
|
|
377
|
+
/**
|
|
378
|
+
* The line that says HOW the process grew, not merely that it did.
|
|
379
|
+
*
|
|
380
|
+
* `heapUsed`, `external` and the native residue each carry their share of RSS,
|
|
381
|
+
* because the shares are what separate the three fixes — and on the process this
|
|
382
|
+
* module failed to catch, the shares were 9% / 74% / 16%, which names the answer
|
|
383
|
+
* on sight.
|
|
384
|
+
*/
|
|
385
|
+
function formatOverBudget(label, b, source) {
|
|
386
|
+
return `[mem] ${label} OVER RSS BUDGET rss=${b.rssMb}MB budget=${b.budgetMb}MB over=+${b.overMb}MB shape=${b.shape} heapUsed=${b.heapUsedMb}MB(${share(b.heapUsedMb, b.rssMb)}%) external=${b.externalMb}MB(${share(b.externalMb, b.rssMb)}%) nativeResidue=${b.nativeResidueMb}MB(${share(b.nativeResidueMb, b.rssMb)}%) arrayBuffers=${b.arrayBuffersMb}MB budgetFrom="${source}" — ${SHAPE_ADVICE[b.shape]}`;
|
|
387
|
+
}
|
|
388
|
+
function format(label, s, loop, budgetMb) {
|
|
389
|
+
const line = `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB` + (budgetMb === void 0 ? "" : ` rssBudget=${budgetMb}MB`);
|
|
302
390
|
if (loop === void 0) return line;
|
|
303
391
|
return `${line} loopP50=${loop.p50Ms}ms loopP99=${loop.p99Ms}ms loopMax=${loop.maxMs}ms`;
|
|
304
392
|
}
|
|
@@ -308,7 +396,7 @@ function format(label, s, loop) {
|
|
|
308
396
|
* `unref`ed so it can never hold the process open, and every read is wrapped:
|
|
309
397
|
* a diagnostic that can crash the process it watches is worse than none.
|
|
310
398
|
*/
|
|
311
|
-
function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true) {
|
|
399
|
+
function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true, rssBudget) {
|
|
312
400
|
const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
|
|
313
401
|
const now = reclaimOptions?.now ?? (() => Date.now());
|
|
314
402
|
const triggerMb = reclaimOptions?.triggerMb ?? 1024;
|
|
@@ -352,6 +440,39 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
352
440
|
};
|
|
353
441
|
let mode = "steady";
|
|
354
442
|
let lastLoggedAt = Number.NEGATIVE_INFINITY;
|
|
443
|
+
const budgetMb = rssBudget?.budgetMb !== void 0 && rssBudget.budgetMb > 0 ? rssBudget.budgetMb : void 0;
|
|
444
|
+
const budgetSource = rssBudget?.source ?? "unspecified";
|
|
445
|
+
const releaseRatio = rssBudget?.releaseRatio ?? .9;
|
|
446
|
+
const reannounceMinMs = rssBudget?.reannounceMinMs ?? 9e5;
|
|
447
|
+
let budgetState = "within";
|
|
448
|
+
let lastBudgetWarnAt = Number.NEGATIVE_INFINITY;
|
|
449
|
+
/** Only pair a "back within" with an over-budget line that was actually said. */
|
|
450
|
+
let overBudgetAnnounced = false;
|
|
451
|
+
/**
|
|
452
|
+
* One line per STATE CHANGE, never one per sample.
|
|
453
|
+
*
|
|
454
|
+
* The probe runs every {@link HEAP_WATCH_FAST_INTERVAL_MS}; warning on every
|
|
455
|
+
* sample above the budget would put 30 lines a minute in the hub's shared log
|
|
456
|
+
* for a process that is legitimately, steadily large. That is precisely the
|
|
457
|
+
* failure mode that buried two real alarms under 83,198 heartbeat lines on
|
|
458
|
+
* 2026-08-27, so the crossing is the event and the level is not.
|
|
459
|
+
*/
|
|
460
|
+
const checkRssBudget = (sample, at) => {
|
|
461
|
+
if (budgetMb === void 0) return;
|
|
462
|
+
const previous = budgetState;
|
|
463
|
+
budgetState = nextRssBudgetState(previous, sample.rssMb, budgetMb, releaseRatio);
|
|
464
|
+
if (budgetState === previous) return;
|
|
465
|
+
if (budgetState === "over") {
|
|
466
|
+
if (at - lastBudgetWarnAt < reannounceMinMs) return;
|
|
467
|
+
lastBudgetWarnAt = at;
|
|
468
|
+
overBudgetAnnounced = true;
|
|
469
|
+
sink.warn(formatOverBudget(label, describeRss(sample, budgetMb), budgetSource));
|
|
470
|
+
return;
|
|
471
|
+
}
|
|
472
|
+
if (!overBudgetAnnounced) return;
|
|
473
|
+
overBudgetAnnounced = false;
|
|
474
|
+
sink.info(`[mem] ${label} back within its RSS budget — rss=${sample.rssMb}MB budget=${budgetMb}MB`);
|
|
475
|
+
};
|
|
355
476
|
/**
|
|
356
477
|
* PROBING and LOGGING are separate cadences, and that separation is the whole
|
|
357
478
|
* point.
|
|
@@ -377,18 +498,20 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
377
498
|
const due = at - lastLoggedAt >= intervalMs;
|
|
378
499
|
if (mode === "escalated" || due) {
|
|
379
500
|
lastLoggedAt = at;
|
|
380
|
-
const line = format(label, sample, loopDelay?.read());
|
|
501
|
+
const line = format(label, sample, loopDelay?.read(), budgetMb);
|
|
381
502
|
if (sample.nearLimit) sink.warn(`${line} — APPROACHING HEAP LIMIT`);
|
|
382
503
|
else if (mode === "escalated") sink.warn(`${line} — heap elevated, sampling every ${probeIntervalMs}ms`);
|
|
383
504
|
else sink.info(line);
|
|
384
505
|
}
|
|
385
506
|
if (previous === "escalated" && mode === "steady") sink.info(`[mem] ${label} heap back to routine — logging every ${intervalMs}ms`);
|
|
507
|
+
checkRssBudget(sample, at);
|
|
386
508
|
maybeReclaim(sample);
|
|
387
509
|
} catch {}
|
|
388
510
|
};
|
|
389
511
|
const timer = setInterval(tick, probeIntervalMs);
|
|
390
512
|
timer.unref?.();
|
|
391
513
|
tick();
|
|
514
|
+
if (rssBudget !== void 0 && budgetMb === void 0) sink.info(`[mem] ${label} is NOT WATCHED against an RSS budget — no budget declared (${budgetSource}). rss growth outside the V8 heap raises nothing here: --max-old-space-size bounds old space only, and stranded subtracts external by construction. Declare execution.rssBudgetMb once this process has a measured working set.`);
|
|
392
515
|
if (announceCeilingOrigin && heapCeilingOrigin(execArgv) === "v8-default") sink.info(`[mem] ${label} heap ceiling is V8's DEFAULT (${read().heapLimitMb}MB) — no --max-old-space-size on argv. Nothing here CHOSE that number; it is derived from host RAM and moves with it.`);
|
|
393
516
|
let stopped = false;
|
|
394
517
|
return () => {
|
|
@@ -414,6 +537,71 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
414
537
|
*/
|
|
415
538
|
var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
|
|
416
539
|
/**
|
|
540
|
+
* Parse a budget carried across a process boundary as an env string.
|
|
541
|
+
*
|
|
542
|
+
* Anything that is not a positive finite integer is `undefined` — i.e. the
|
|
543
|
+
* process is UNWATCHED and says so. A malformed budget must never round to a
|
|
544
|
+
* silent default: that is the failure this whole mechanism exists to remove.
|
|
545
|
+
*/
|
|
546
|
+
function parseRssBudgetMb(raw) {
|
|
547
|
+
if (raw === void 0) return void 0;
|
|
548
|
+
const parsed = Number(raw);
|
|
549
|
+
if (!Number.isFinite(parsed) || parsed <= 0) return void 0;
|
|
550
|
+
return Math.floor(parsed);
|
|
551
|
+
}
|
|
552
|
+
/** Env var carrying a runner's declared RSS budget from parent to child. */
|
|
553
|
+
var RUNNER_RSS_BUDGET_ENV = "CAMSTACK_RUNNER_RSS_BUDGET_MB";
|
|
554
|
+
/** Operator override for hub-main's budget, applied without a release. */
|
|
555
|
+
var HUB_RSS_BUDGET_ENV = "CAMSTACK_HUB_RSS_BUDGET_MB";
|
|
556
|
+
/**
|
|
557
|
+
* hub-main's declared RSS budget, in MB.
|
|
558
|
+
*
|
|
559
|
+
* ## Why the number lives HERE and not in the image
|
|
560
|
+
*
|
|
561
|
+
* hub-main has no manifest to declare it in, and the obvious alternative — the
|
|
562
|
+
* entrypoint that already knew about `CAMSTACK_HUB_HEAP_MB` — is the least
|
|
563
|
+
* deployable surface in this system. The Dockerfile says so itself: the image
|
|
564
|
+
* never contains this repo's backend, so a value written there reaches a running
|
|
565
|
+
* hub only on an image rebuild. `main.ts` is what `applyServerUpdate` replaces,
|
|
566
|
+
* which is exactly why the heartbeat was started from there in the first place.
|
|
567
|
+
* A budget that cannot ship is not a budget.
|
|
568
|
+
*
|
|
569
|
+
* ## Why 4096
|
|
570
|
+
*
|
|
571
|
+
* hub-main's entitlement is its old-space ceiling plus its measured native
|
|
572
|
+
* floor. The floor is 438MB, established live across five consecutive
|
|
573
|
+
* `gc(last-resort)` passes that landed within 4MB of each other
|
|
574
|
+
* (4559/4561/4560/4562/4563MB) — sqlite pages, libvips, sharp, thread stacks and
|
|
575
|
+
* node's own text. Against the 3072MB ceiling in force on 2026-08-27 that is
|
|
576
|
+
* ~3.5GB of legitimate footprint, and 4096 leaves it ~600MB of slack.
|
|
577
|
+
*
|
|
578
|
+
* **This WILL fire on the hub as measured on 2026-08-27, and that is the point.**
|
|
579
|
+
* That process sat at 4.87GiB with `external` at 360MB: ~2.8GB of it was native
|
|
580
|
+
* memory already freed and never returned, spread across 13 anonymous mappings
|
|
581
|
+
* >=64MB, with only 1295MB in V8 pages. `MALLOC_ARENA_MAX=2` and
|
|
582
|
+
* `VIPS_CONCURRENCY=1` were already applied — verified from `/proc/<pid>/environ`
|
|
583
|
+
* — so that lever is spent and the residue stands. The alarm reporting a
|
|
584
|
+
* genuinely over-budget process is the mechanism working; a number picked to sit
|
|
585
|
+
* just above today's RSS would be a rubber stamp.
|
|
586
|
+
*
|
|
587
|
+
* An operator who decides that residue is the cost of doing business moves it in
|
|
588
|
+
* one place with {@link HUB_RSS_BUDGET_ENV}, and the decision is on the record
|
|
589
|
+
* instead of being absorbed by a threshold nobody set.
|
|
590
|
+
*/
|
|
591
|
+
var HUB_MAIN_RSS_BUDGET_MB = 4096;
|
|
592
|
+
/** hub-main's budget: the declared number, or the operator's override. */
|
|
593
|
+
function hubMainRssBudget(env = process.env) {
|
|
594
|
+
const override = env[HUB_RSS_BUDGET_ENV];
|
|
595
|
+
if (override !== void 0) return {
|
|
596
|
+
budgetMb: parseRssBudgetMb(override),
|
|
597
|
+
source: HUB_RSS_BUDGET_ENV
|
|
598
|
+
};
|
|
599
|
+
return {
|
|
600
|
+
budgetMb: HUB_MAIN_RSS_BUDGET_MB,
|
|
601
|
+
source: "HUB_MAIN_RSS_BUDGET_MB (kernel/heap-watch)"
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
/**
|
|
417
605
|
* Start the heap watch + stranded-page reclaimer for ONE addon runner, iff it
|
|
418
606
|
* is a heavy one. Returns the stop function, or undefined when nothing started.
|
|
419
607
|
*
|
|
@@ -447,13 +635,17 @@ var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
|
|
|
447
635
|
function startRunnerHeapWatch(options) {
|
|
448
636
|
if (options.heapProfile !== "heavy") return void 0;
|
|
449
637
|
const intervalMs = options.intervalMs ?? 3e5;
|
|
450
|
-
|
|
638
|
+
const rssBudget = {
|
|
639
|
+
budgetMb: parseRssBudgetMb(options.rssBudgetMb),
|
|
640
|
+
source: `manifest execution.rssBudgetMb (via ${RUNNER_RSS_BUDGET_ENV})`
|
|
641
|
+
};
|
|
642
|
+
if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs, void 0, void 0, void 0, false, rssBudget);
|
|
451
643
|
let reclaimOptions = options.reclaimOptions;
|
|
452
644
|
if (reclaimOptions === void 0) {
|
|
453
645
|
const reclaimer = createV8Reclaimer();
|
|
454
646
|
reclaimOptions = reclaimer === void 0 ? void 0 : { reclaim: reclaimer };
|
|
455
647
|
}
|
|
456
|
-
return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, true);
|
|
648
|
+
return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, true, rssBudget);
|
|
457
649
|
}
|
|
458
650
|
//#endregion
|
|
459
651
|
//#region src/kernel/moleculer/addon-data-plane-facility.ts
|
|
@@ -8390,4 +8582,4 @@ async function installManifestPythonDeps(declaration, addonDir, deps, logger) {
|
|
|
8390
8582
|
await deps.installPythonRequirements(reqAbs);
|
|
8391
8583
|
}
|
|
8392
8584
|
//#endregion
|
|
8393
|
-
export { buildUdsNativeCapProxy as $, createParentUnownedCallHandler as A,
|
|
8585
|
+
export { buildUdsNativeCapProxy as $, createParentUnownedCallHandler as A, RSS_BUDGET_REANNOUNCE_MIN_MS as At, AGENT_CAP_FWD_SERVICE as B, shouldReclaim as Bt, brokerTransportLink as C, createAddonDataPlaneFacility as Ct, HUB_CAP_FWD_ACTION as D, HEAP_WATCH_WARN_RATIO as Dt, localProviderLink as E, HEAP_WATCH_INTERVAL_MS as Et, createUdsLoggerWithControl as F, createV8Reclaimer as Ft, createLocalTransport as G, CapRouteError as H, startRunnerHeapWatch as Ht, LocalChildClient as I, describeRss as It, SocketChannel as J, UdsLocalTransportClient as K, LocalChildRegistry as L, hubMainRssBudget as Lt, createUdsEventBus as M, RUNNER_HEAP_WATCH_INTERVAL_MS as Mt, udsChildLogToWorkerEntry as N, RUNNER_RSS_BUDGET_ENV as Nt, HUB_CAP_FWD_SERVICE as O, HUB_MAIN_RSS_BUDGET_MB as Ot, createUdsLogger as P, buildHeapSample as Pt, buildNativeCapProxy as Q, UDS_NO_ROUTE_PREFIX as R, nextRssBudgetState as Rt, brokerCallForCap as S, runNpm as St, ipcParentLink as T, HEAP_RECLAIM_TRIGGER_MB as Tt, classifyCapRoute as U, strandedMb as Ut, CapRouteResolver as V, startHeapWatch as Vt, callWithServiceDiscovery as W, FrameDecoder as X, localEndpointPath as Y, encodeFrame as Z, createKernelHwAccel as _, CapabilityUnavailableError as _t, getWorkerDeviceRegistry as a, createAddonService as at, __resetCapUsageRegistryForTests as b, resolveAddonClass as bt, setHubConnected as c, capActionName as ct, getBrokerEventBus as d, capServiceName as dt, createBrokerDeviceManagerApi as et, getMoleculerEventStats as f, parseCapAction as ft, AddonDepsManager as g, CapabilityHandle as gt, subscribePassthrough as h, DeviceRegistry as ht, createUdsAddonContext as i, setWorkerNativeCapsChangeListener as it, createUdsEventBridge as j, RSS_BUDGET_RELEASE_RATIO as jt, createHubCapForwardService as k, HUB_RSS_BUDGET_ENV as kt, EVENT_TOPIC_PREFIX as l, capActionSuffix as lt, setNodeEventInterest as m, serializeTypedArrays as mt, adaptBrokerToCluster as n, getWorkerNativeCapSnapshot as nt, getOrInitReadinessRegistry as o, validateProviderRegistrations as ot, registerEventBusService as p, deserializeTypedArrays as pt, UdsLocalTransportServer as q, createAddonContext as r, mountNativeCapService as rt, getOrInitReadinessRegistryForClient as s, NATIVE_PROVIDER_SERVICE_INFIX as st, installManifestPythonDeps as t, getWorkerNativeCapProvider as tt, clusterEventTopic as u, capBareAction as ut, resolveHwAccel as v, copyBundledNativeModules as vt, buildLinkChain as w, HEAP_RECLAIM_MIN_INTERVAL_MS as wt, getCapUsageRegistry as x, resolveNpmInvocation as xt, CapUsageRegistry as y, installManifestNativeDeps as yt, AGENT_CAP_FWD_ACTION as z, parseRssBudgetMb as zt };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
const require_chunk = require("./chunk-Cek0wNdY.js");
|
|
2
|
-
require("./dist-
|
|
2
|
+
require("./dist-Dl6MFXPr.js");
|
|
3
3
|
let node_crypto = require("node:crypto");
|
|
4
4
|
node_crypto = require_chunk.__toESM(node_crypto);
|
|
5
5
|
let _camstack_types_node = require("@camstack/types/node");
|
|
@@ -266,6 +266,22 @@ var consoleSink = {
|
|
|
266
266
|
* The subtrahend is heapUSED, not heapTotal, deliberately: the pages V8
|
|
267
267
|
* committed at a sawtooth peak and no longer needs are counted inside heapTotal,
|
|
268
268
|
* and they are precisely what this number exists to see.
|
|
269
|
+
*
|
|
270
|
+
* ## What this number can and CANNOT be asked
|
|
271
|
+
*
|
|
272
|
+
* It answers exactly one question: *are there pages a `last-resort` GC could
|
|
273
|
+
* hand back?* For that it is correct, and {@link shouldReclaim} is the only
|
|
274
|
+
* gate that should ever be built on it.
|
|
275
|
+
*
|
|
276
|
+
* It can never answer *is this process too big?*, because `external` is
|
|
277
|
+
* subtracted by construction. A process whose growth is entirely `external` —
|
|
278
|
+
* Buffers, `arrayBuffers`, what a native addon mallocs — moves `rss` and the
|
|
279
|
+
* subtrahend by the same amount, so this number is **algebraically constant
|
|
280
|
+
* while the process triples**. That is not a threshold set too high; the
|
|
281
|
+
* quantity cannot represent the event.
|
|
282
|
+
*
|
|
283
|
+
* It cost 11.4GB and a ~154 second stall of the whole container to establish
|
|
284
|
+
* that. See {@link RssBudgetOptions} for the number that does answer it.
|
|
269
285
|
*/
|
|
270
286
|
function strandedMb(s) {
|
|
271
287
|
return s.rssMb - s.heapUsedMb - s.externalMb;
|
|
@@ -274,6 +290,67 @@ function strandedMb(s) {
|
|
|
274
290
|
function shouldReclaim(s, triggerMb = HEAP_RECLAIM_TRIGGER_MB) {
|
|
275
291
|
return strandedMb(s) > triggerMb;
|
|
276
292
|
}
|
|
293
|
+
/** Share of RSS, as a whole percent. Never divides by zero. */
|
|
294
|
+
function share(partMb, rssMb) {
|
|
295
|
+
return rssMb > 0 ? Math.round(partMb / rssMb * 100) : 0;
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Split a sample into the three shapes and name the dominant one.
|
|
299
|
+
*
|
|
300
|
+
* `nativeResidue` is clamped at 0: `process.memoryUsage()` reads its fields at
|
|
301
|
+
* slightly different instants, so `heapUsed + external` can transiently exceed
|
|
302
|
+
* `rss` and a negative residue would misclassify the sample.
|
|
303
|
+
*/
|
|
304
|
+
function describeRss(s, budgetMb) {
|
|
305
|
+
const nativeResidueMb = Math.max(0, strandedMb(s));
|
|
306
|
+
const shape = s.heapUsedMb >= s.externalMb && s.heapUsedMb >= nativeResidueMb ? "v8-heap" : s.externalMb >= nativeResidueMb ? "external" : "native-residue";
|
|
307
|
+
return {
|
|
308
|
+
rssMb: s.rssMb,
|
|
309
|
+
budgetMb,
|
|
310
|
+
overMb: Math.max(0, s.rssMb - budgetMb),
|
|
311
|
+
heapUsedMb: s.heapUsedMb,
|
|
312
|
+
externalMb: s.externalMb,
|
|
313
|
+
nativeResidueMb,
|
|
314
|
+
arrayBuffersMb: s.arrayBuffersMb,
|
|
315
|
+
shape
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
/** What the operator can actually DO about each shape. */
|
|
319
|
+
var SHAPE_ADVICE = {
|
|
320
|
+
"v8-heap": "live JS retainers — bounded by --max-old-space-size; the fix is the retained set",
|
|
321
|
+
external: "Buffers / native external — NO V8 flag bounds this, and the old-space ceiling never will",
|
|
322
|
+
"native-residue": "freed native memory the allocator has not returned — MALLOC_ARENA_MAX/VIPS_CONCURRENCY are already applied, so this lever is spent"
|
|
323
|
+
};
|
|
324
|
+
/**
|
|
325
|
+
* Release the alarm below this share of the budget.
|
|
326
|
+
*
|
|
327
|
+
* Hysteresis, not a second budget: every process here sawtooths (the reclaim
|
|
328
|
+
* pass alone moves hub-main by ~1.6GB), so a single threshold would print a
|
|
329
|
+
* pair of lines on every cycle. 0.9 of the budget is a real retreat.
|
|
330
|
+
*/
|
|
331
|
+
var RSS_BUDGET_RELEASE_RATIO = .9;
|
|
332
|
+
/**
|
|
333
|
+
* Floor on how often a crossing may be ANNOUNCED again.
|
|
334
|
+
*
|
|
335
|
+
* Hysteresis alone bounds flapping around the edge; it does not bound a process
|
|
336
|
+
* that genuinely swings across the budget every reclaim cycle
|
|
337
|
+
* ({@link HEAP_RECLAIM_MIN_INTERVAL_MS} is 2 minutes, so that is up to 30 pairs
|
|
338
|
+
* of lines an hour, forever). A process that oscillates across its budget is ONE
|
|
339
|
+
* story, and the operator needs it told once per quarter-hour, not thirty times.
|
|
340
|
+
*
|
|
341
|
+
* This is the discipline the rest of this module already follows and the one the
|
|
342
|
+
* log needed: on 2026-08-27 two real alarms were buried under 83,198 Hikvision
|
|
343
|
+
* heartbeat lines in 12h. A watchdog that shouts is a watchdog that gets muted.
|
|
344
|
+
*/
|
|
345
|
+
var RSS_BUDGET_REANNOUNCE_MIN_MS = 9e5;
|
|
346
|
+
/**
|
|
347
|
+
* The alarm state machine, pure so the thresholds are testable without
|
|
348
|
+
* allocating five gigabytes to reach them.
|
|
349
|
+
*/
|
|
350
|
+
function nextRssBudgetState(current, rssMb, budgetMb, releaseRatio = RSS_BUDGET_RELEASE_RATIO) {
|
|
351
|
+
if (current === "over") return rssMb < budgetMb * releaseRatio ? "within" : "over";
|
|
352
|
+
return rssMb > budgetMb ? "over" : "within";
|
|
353
|
+
}
|
|
277
354
|
function isGcFunction(value) {
|
|
278
355
|
return typeof value === "function";
|
|
279
356
|
}
|
|
@@ -301,8 +378,19 @@ function createV8Reclaimer() {
|
|
|
301
378
|
return;
|
|
302
379
|
}
|
|
303
380
|
}
|
|
304
|
-
|
|
305
|
-
|
|
381
|
+
/**
|
|
382
|
+
* The line that says HOW the process grew, not merely that it did.
|
|
383
|
+
*
|
|
384
|
+
* `heapUsed`, `external` and the native residue each carry their share of RSS,
|
|
385
|
+
* because the shares are what separate the three fixes — and on the process this
|
|
386
|
+
* module failed to catch, the shares were 9% / 74% / 16%, which names the answer
|
|
387
|
+
* on sight.
|
|
388
|
+
*/
|
|
389
|
+
function formatOverBudget(label, b, source) {
|
|
390
|
+
return `[mem] ${label} OVER RSS BUDGET rss=${b.rssMb}MB budget=${b.budgetMb}MB over=+${b.overMb}MB shape=${b.shape} heapUsed=${b.heapUsedMb}MB(${share(b.heapUsedMb, b.rssMb)}%) external=${b.externalMb}MB(${share(b.externalMb, b.rssMb)}%) nativeResidue=${b.nativeResidueMb}MB(${share(b.nativeResidueMb, b.rssMb)}%) arrayBuffers=${b.arrayBuffersMb}MB budgetFrom="${source}" — ${SHAPE_ADVICE[b.shape]}`;
|
|
391
|
+
}
|
|
392
|
+
function format(label, s, loop, budgetMb) {
|
|
393
|
+
const line = `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB` + (budgetMb === void 0 ? "" : ` rssBudget=${budgetMb}MB`);
|
|
306
394
|
if (loop === void 0) return line;
|
|
307
395
|
return `${line} loopP50=${loop.p50Ms}ms loopP99=${loop.p99Ms}ms loopMax=${loop.maxMs}ms`;
|
|
308
396
|
}
|
|
@@ -312,7 +400,7 @@ function format(label, s, loop) {
|
|
|
312
400
|
* `unref`ed so it can never hold the process open, and every read is wrapped:
|
|
313
401
|
* a diagnostic that can crash the process it watches is worse than none.
|
|
314
402
|
*/
|
|
315
|
-
function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true) {
|
|
403
|
+
function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true, rssBudget) {
|
|
316
404
|
const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
|
|
317
405
|
const now = reclaimOptions?.now ?? (() => Date.now());
|
|
318
406
|
const triggerMb = reclaimOptions?.triggerMb ?? 1024;
|
|
@@ -356,6 +444,39 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
356
444
|
};
|
|
357
445
|
let mode = "steady";
|
|
358
446
|
let lastLoggedAt = Number.NEGATIVE_INFINITY;
|
|
447
|
+
const budgetMb = rssBudget?.budgetMb !== void 0 && rssBudget.budgetMb > 0 ? rssBudget.budgetMb : void 0;
|
|
448
|
+
const budgetSource = rssBudget?.source ?? "unspecified";
|
|
449
|
+
const releaseRatio = rssBudget?.releaseRatio ?? .9;
|
|
450
|
+
const reannounceMinMs = rssBudget?.reannounceMinMs ?? 9e5;
|
|
451
|
+
let budgetState = "within";
|
|
452
|
+
let lastBudgetWarnAt = Number.NEGATIVE_INFINITY;
|
|
453
|
+
/** Only pair a "back within" with an over-budget line that was actually said. */
|
|
454
|
+
let overBudgetAnnounced = false;
|
|
455
|
+
/**
|
|
456
|
+
* One line per STATE CHANGE, never one per sample.
|
|
457
|
+
*
|
|
458
|
+
* The probe runs every {@link HEAP_WATCH_FAST_INTERVAL_MS}; warning on every
|
|
459
|
+
* sample above the budget would put 30 lines a minute in the hub's shared log
|
|
460
|
+
* for a process that is legitimately, steadily large. That is precisely the
|
|
461
|
+
* failure mode that buried two real alarms under 83,198 heartbeat lines on
|
|
462
|
+
* 2026-08-27, so the crossing is the event and the level is not.
|
|
463
|
+
*/
|
|
464
|
+
const checkRssBudget = (sample, at) => {
|
|
465
|
+
if (budgetMb === void 0) return;
|
|
466
|
+
const previous = budgetState;
|
|
467
|
+
budgetState = nextRssBudgetState(previous, sample.rssMb, budgetMb, releaseRatio);
|
|
468
|
+
if (budgetState === previous) return;
|
|
469
|
+
if (budgetState === "over") {
|
|
470
|
+
if (at - lastBudgetWarnAt < reannounceMinMs) return;
|
|
471
|
+
lastBudgetWarnAt = at;
|
|
472
|
+
overBudgetAnnounced = true;
|
|
473
|
+
sink.warn(formatOverBudget(label, describeRss(sample, budgetMb), budgetSource));
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
if (!overBudgetAnnounced) return;
|
|
477
|
+
overBudgetAnnounced = false;
|
|
478
|
+
sink.info(`[mem] ${label} back within its RSS budget — rss=${sample.rssMb}MB budget=${budgetMb}MB`);
|
|
479
|
+
};
|
|
359
480
|
/**
|
|
360
481
|
* PROBING and LOGGING are separate cadences, and that separation is the whole
|
|
361
482
|
* point.
|
|
@@ -381,18 +502,20 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
381
502
|
const due = at - lastLoggedAt >= intervalMs;
|
|
382
503
|
if (mode === "escalated" || due) {
|
|
383
504
|
lastLoggedAt = at;
|
|
384
|
-
const line = format(label, sample, loopDelay?.read());
|
|
505
|
+
const line = format(label, sample, loopDelay?.read(), budgetMb);
|
|
385
506
|
if (sample.nearLimit) sink.warn(`${line} — APPROACHING HEAP LIMIT`);
|
|
386
507
|
else if (mode === "escalated") sink.warn(`${line} — heap elevated, sampling every ${probeIntervalMs}ms`);
|
|
387
508
|
else sink.info(line);
|
|
388
509
|
}
|
|
389
510
|
if (previous === "escalated" && mode === "steady") sink.info(`[mem] ${label} heap back to routine — logging every ${intervalMs}ms`);
|
|
511
|
+
checkRssBudget(sample, at);
|
|
390
512
|
maybeReclaim(sample);
|
|
391
513
|
} catch {}
|
|
392
514
|
};
|
|
393
515
|
const timer = setInterval(tick, probeIntervalMs);
|
|
394
516
|
timer.unref?.();
|
|
395
517
|
tick();
|
|
518
|
+
if (rssBudget !== void 0 && budgetMb === void 0) sink.info(`[mem] ${label} is NOT WATCHED against an RSS budget — no budget declared (${budgetSource}). rss growth outside the V8 heap raises nothing here: --max-old-space-size bounds old space only, and stranded subtracts external by construction. Declare execution.rssBudgetMb once this process has a measured working set.`);
|
|
396
519
|
if (announceCeilingOrigin && heapCeilingOrigin(execArgv) === "v8-default") sink.info(`[mem] ${label} heap ceiling is V8's DEFAULT (${read().heapLimitMb}MB) — no --max-old-space-size on argv. Nothing here CHOSE that number; it is derived from host RAM and moves with it.`);
|
|
397
520
|
let stopped = false;
|
|
398
521
|
return () => {
|
|
@@ -418,6 +541,71 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
418
541
|
*/
|
|
419
542
|
var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
|
|
420
543
|
/**
|
|
544
|
+
* Parse a budget carried across a process boundary as an env string.
|
|
545
|
+
*
|
|
546
|
+
* Anything that is not a positive finite integer is `undefined` — i.e. the
|
|
547
|
+
* process is UNWATCHED and says so. A malformed budget must never round to a
|
|
548
|
+
* silent default: that is the failure this whole mechanism exists to remove.
|
|
549
|
+
*/
|
|
550
|
+
function parseRssBudgetMb(raw) {
|
|
551
|
+
if (raw === void 0) return void 0;
|
|
552
|
+
const parsed = Number(raw);
|
|
553
|
+
if (!Number.isFinite(parsed) || parsed <= 0) return void 0;
|
|
554
|
+
return Math.floor(parsed);
|
|
555
|
+
}
|
|
556
|
+
/** Env var carrying a runner's declared RSS budget from parent to child. */
|
|
557
|
+
var RUNNER_RSS_BUDGET_ENV = "CAMSTACK_RUNNER_RSS_BUDGET_MB";
|
|
558
|
+
/** Operator override for hub-main's budget, applied without a release. */
|
|
559
|
+
var HUB_RSS_BUDGET_ENV = "CAMSTACK_HUB_RSS_BUDGET_MB";
|
|
560
|
+
/**
|
|
561
|
+
* hub-main's declared RSS budget, in MB.
|
|
562
|
+
*
|
|
563
|
+
* ## Why the number lives HERE and not in the image
|
|
564
|
+
*
|
|
565
|
+
* hub-main has no manifest to declare it in, and the obvious alternative — the
|
|
566
|
+
* entrypoint that already knew about `CAMSTACK_HUB_HEAP_MB` — is the least
|
|
567
|
+
* deployable surface in this system. The Dockerfile says so itself: the image
|
|
568
|
+
* never contains this repo's backend, so a value written there reaches a running
|
|
569
|
+
* hub only on an image rebuild. `main.ts` is what `applyServerUpdate` replaces,
|
|
570
|
+
* which is exactly why the heartbeat was started from there in the first place.
|
|
571
|
+
* A budget that cannot ship is not a budget.
|
|
572
|
+
*
|
|
573
|
+
* ## Why 4096
|
|
574
|
+
*
|
|
575
|
+
* hub-main's entitlement is its old-space ceiling plus its measured native
|
|
576
|
+
* floor. The floor is 438MB, established live across five consecutive
|
|
577
|
+
* `gc(last-resort)` passes that landed within 4MB of each other
|
|
578
|
+
* (4559/4561/4560/4562/4563MB) — sqlite pages, libvips, sharp, thread stacks and
|
|
579
|
+
* node's own text. Against the 3072MB ceiling in force on 2026-08-27 that is
|
|
580
|
+
* ~3.5GB of legitimate footprint, and 4096 leaves it ~600MB of slack.
|
|
581
|
+
*
|
|
582
|
+
* **This WILL fire on the hub as measured on 2026-08-27, and that is the point.**
|
|
583
|
+
* That process sat at 4.87GiB with `external` at 360MB: ~2.8GB of it was native
|
|
584
|
+
* memory already freed and never returned, spread across 13 anonymous mappings
|
|
585
|
+
* >=64MB, with only 1295MB in V8 pages. `MALLOC_ARENA_MAX=2` and
|
|
586
|
+
* `VIPS_CONCURRENCY=1` were already applied — verified from `/proc/<pid>/environ`
|
|
587
|
+
* — so that lever is spent and the residue stands. The alarm reporting a
|
|
588
|
+
* genuinely over-budget process is the mechanism working; a number picked to sit
|
|
589
|
+
* just above today's RSS would be a rubber stamp.
|
|
590
|
+
*
|
|
591
|
+
* An operator who decides that residue is the cost of doing business moves it in
|
|
592
|
+
* one place with {@link HUB_RSS_BUDGET_ENV}, and the decision is on the record
|
|
593
|
+
* instead of being absorbed by a threshold nobody set.
|
|
594
|
+
*/
|
|
595
|
+
var HUB_MAIN_RSS_BUDGET_MB = 4096;
|
|
596
|
+
/** hub-main's budget: the declared number, or the operator's override. */
|
|
597
|
+
function hubMainRssBudget(env = process.env) {
|
|
598
|
+
const override = env[HUB_RSS_BUDGET_ENV];
|
|
599
|
+
if (override !== void 0) return {
|
|
600
|
+
budgetMb: parseRssBudgetMb(override),
|
|
601
|
+
source: HUB_RSS_BUDGET_ENV
|
|
602
|
+
};
|
|
603
|
+
return {
|
|
604
|
+
budgetMb: HUB_MAIN_RSS_BUDGET_MB,
|
|
605
|
+
source: "HUB_MAIN_RSS_BUDGET_MB (kernel/heap-watch)"
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
/**
|
|
421
609
|
* Start the heap watch + stranded-page reclaimer for ONE addon runner, iff it
|
|
422
610
|
* is a heavy one. Returns the stop function, or undefined when nothing started.
|
|
423
611
|
*
|
|
@@ -451,13 +639,17 @@ var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
|
|
|
451
639
|
function startRunnerHeapWatch(options) {
|
|
452
640
|
if (options.heapProfile !== "heavy") return void 0;
|
|
453
641
|
const intervalMs = options.intervalMs ?? 3e5;
|
|
454
|
-
|
|
642
|
+
const rssBudget = {
|
|
643
|
+
budgetMb: parseRssBudgetMb(options.rssBudgetMb),
|
|
644
|
+
source: `manifest execution.rssBudgetMb (via ${RUNNER_RSS_BUDGET_ENV})`
|
|
645
|
+
};
|
|
646
|
+
if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs, void 0, void 0, void 0, false, rssBudget);
|
|
455
647
|
let reclaimOptions = options.reclaimOptions;
|
|
456
648
|
if (reclaimOptions === void 0) {
|
|
457
649
|
const reclaimer = createV8Reclaimer();
|
|
458
650
|
reclaimOptions = reclaimer === void 0 ? void 0 : { reclaim: reclaimer };
|
|
459
651
|
}
|
|
460
|
-
return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, true);
|
|
652
|
+
return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, true, rssBudget);
|
|
461
653
|
}
|
|
462
654
|
//#endregion
|
|
463
655
|
//#region src/kernel/moleculer/addon-data-plane-facility.ts
|
|
@@ -8496,6 +8688,18 @@ Object.defineProperty(exports, "HUB_CAP_FWD_SERVICE", {
|
|
|
8496
8688
|
return HUB_CAP_FWD_SERVICE;
|
|
8497
8689
|
}
|
|
8498
8690
|
});
|
|
8691
|
+
Object.defineProperty(exports, "HUB_MAIN_RSS_BUDGET_MB", {
|
|
8692
|
+
enumerable: true,
|
|
8693
|
+
get: function() {
|
|
8694
|
+
return HUB_MAIN_RSS_BUDGET_MB;
|
|
8695
|
+
}
|
|
8696
|
+
});
|
|
8697
|
+
Object.defineProperty(exports, "HUB_RSS_BUDGET_ENV", {
|
|
8698
|
+
enumerable: true,
|
|
8699
|
+
get: function() {
|
|
8700
|
+
return HUB_RSS_BUDGET_ENV;
|
|
8701
|
+
}
|
|
8702
|
+
});
|
|
8499
8703
|
Object.defineProperty(exports, "LocalChildClient", {
|
|
8500
8704
|
enumerable: true,
|
|
8501
8705
|
get: function() {
|
|
@@ -8514,12 +8718,30 @@ Object.defineProperty(exports, "NATIVE_PROVIDER_SERVICE_INFIX", {
|
|
|
8514
8718
|
return NATIVE_PROVIDER_SERVICE_INFIX;
|
|
8515
8719
|
}
|
|
8516
8720
|
});
|
|
8721
|
+
Object.defineProperty(exports, "RSS_BUDGET_REANNOUNCE_MIN_MS", {
|
|
8722
|
+
enumerable: true,
|
|
8723
|
+
get: function() {
|
|
8724
|
+
return RSS_BUDGET_REANNOUNCE_MIN_MS;
|
|
8725
|
+
}
|
|
8726
|
+
});
|
|
8727
|
+
Object.defineProperty(exports, "RSS_BUDGET_RELEASE_RATIO", {
|
|
8728
|
+
enumerable: true,
|
|
8729
|
+
get: function() {
|
|
8730
|
+
return RSS_BUDGET_RELEASE_RATIO;
|
|
8731
|
+
}
|
|
8732
|
+
});
|
|
8517
8733
|
Object.defineProperty(exports, "RUNNER_HEAP_WATCH_INTERVAL_MS", {
|
|
8518
8734
|
enumerable: true,
|
|
8519
8735
|
get: function() {
|
|
8520
8736
|
return RUNNER_HEAP_WATCH_INTERVAL_MS;
|
|
8521
8737
|
}
|
|
8522
8738
|
});
|
|
8739
|
+
Object.defineProperty(exports, "RUNNER_RSS_BUDGET_ENV", {
|
|
8740
|
+
enumerable: true,
|
|
8741
|
+
get: function() {
|
|
8742
|
+
return RUNNER_RSS_BUDGET_ENV;
|
|
8743
|
+
}
|
|
8744
|
+
});
|
|
8523
8745
|
Object.defineProperty(exports, "SocketChannel", {
|
|
8524
8746
|
enumerable: true,
|
|
8525
8747
|
get: function() {
|
|
@@ -8724,6 +8946,12 @@ Object.defineProperty(exports, "createV8Reclaimer", {
|
|
|
8724
8946
|
return createV8Reclaimer;
|
|
8725
8947
|
}
|
|
8726
8948
|
});
|
|
8949
|
+
Object.defineProperty(exports, "describeRss", {
|
|
8950
|
+
enumerable: true,
|
|
8951
|
+
get: function() {
|
|
8952
|
+
return describeRss;
|
|
8953
|
+
}
|
|
8954
|
+
});
|
|
8727
8955
|
Object.defineProperty(exports, "deserializeTypedArrays", {
|
|
8728
8956
|
enumerable: true,
|
|
8729
8957
|
get: function() {
|
|
@@ -8784,6 +9012,12 @@ Object.defineProperty(exports, "getWorkerNativeCapSnapshot", {
|
|
|
8784
9012
|
return getWorkerNativeCapSnapshot;
|
|
8785
9013
|
}
|
|
8786
9014
|
});
|
|
9015
|
+
Object.defineProperty(exports, "hubMainRssBudget", {
|
|
9016
|
+
enumerable: true,
|
|
9017
|
+
get: function() {
|
|
9018
|
+
return hubMainRssBudget;
|
|
9019
|
+
}
|
|
9020
|
+
});
|
|
8787
9021
|
Object.defineProperty(exports, "installManifestNativeDeps", {
|
|
8788
9022
|
enumerable: true,
|
|
8789
9023
|
get: function() {
|
|
@@ -8820,12 +9054,24 @@ Object.defineProperty(exports, "mountNativeCapService", {
|
|
|
8820
9054
|
return mountNativeCapService;
|
|
8821
9055
|
}
|
|
8822
9056
|
});
|
|
9057
|
+
Object.defineProperty(exports, "nextRssBudgetState", {
|
|
9058
|
+
enumerable: true,
|
|
9059
|
+
get: function() {
|
|
9060
|
+
return nextRssBudgetState;
|
|
9061
|
+
}
|
|
9062
|
+
});
|
|
8823
9063
|
Object.defineProperty(exports, "parseCapAction", {
|
|
8824
9064
|
enumerable: true,
|
|
8825
9065
|
get: function() {
|
|
8826
9066
|
return parseCapAction;
|
|
8827
9067
|
}
|
|
8828
9068
|
});
|
|
9069
|
+
Object.defineProperty(exports, "parseRssBudgetMb", {
|
|
9070
|
+
enumerable: true,
|
|
9071
|
+
get: function() {
|
|
9072
|
+
return parseRssBudgetMb;
|
|
9073
|
+
}
|
|
9074
|
+
});
|
|
8829
9075
|
Object.defineProperty(exports, "registerEventBusService", {
|
|
8830
9076
|
enumerable: true,
|
|
8831
9077
|
get: function() {
|