@camstack/system 1.2.144 → 1.2.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-runner.js +2 -1
- package/dist/addon-runner.mjs +2 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +1 -1
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
- package/dist/builtins/device-manager/device-manager.addon.js +157 -35
- package/dist/builtins/device-manager/device-manager.addon.mjs +157 -35
- package/dist/builtins/device-manager/device-queries.d.ts +22 -0
- package/dist/builtins/device-manager/device-row-store.d.ts +18 -0
- package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +1 -1
- package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
- package/dist/builtins/local-network/local-network.addon.js +1 -1
- package/dist/builtins/local-network/local-network.addon.mjs +1 -1
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.js +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1 -1
- package/dist/builtins/platform-probe/index.js +1 -1
- package/dist/builtins/platform-probe/index.mjs +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +1 -1
- package/dist/builtins/snapshot/index.mjs +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings-backend.d.ts +14 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +0 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +0 -0
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +2 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +2 -1
- package/dist/builtins/system-config/system-config.addon.js +1 -1
- package/dist/builtins/system-config/system-config.addon.mjs +1 -1
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{dist-D3lqzV40.js → dist-BVU5JADq.js} +401 -24
- package/dist/{dist-6rM7chzb.mjs → dist-CG7kwIZp.mjs} +401 -24
- package/dist/index.d.ts +2 -2
- package/dist/index.js +60 -11
- package/dist/index.mjs +52 -12
- package/dist/kernel/heap-watch.d.ts +209 -1
- package/dist/kernel/moleculer/children-batch-loader.d.ts +46 -0
- package/dist/kernel/moleculer/process-service.d.ts +15 -6
- package/dist/{manifest-python-deps-BuhBXCg0.mjs → manifest-python-deps-CVeckyXx.mjs} +390 -8
- package/dist/{manifest-python-deps-B_mCU6gz.js → manifest-python-deps-CktMcXzS.js} +443 -7
- package/dist/{retired-settings-keys-BtvClSPu.mjs → retired-settings-keys-Dm1z4mO4.mjs} +1 -1
- package/dist/{retired-settings-keys-OtUQtEbq.js → retired-settings-keys-_PLI9w0k.js} +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
const require_chunk = require("./chunk-Cek0wNdY.js");
|
|
2
|
+
require("./dist-BVU5JADq.js");
|
|
2
3
|
let node_crypto = require("node:crypto");
|
|
3
4
|
node_crypto = require_chunk.__toESM(node_crypto);
|
|
4
5
|
let _camstack_types_node = require("@camstack/types/node");
|
|
@@ -265,6 +266,22 @@ var consoleSink = {
|
|
|
265
266
|
* The subtrahend is heapUSED, not heapTotal, deliberately: the pages V8
|
|
266
267
|
* committed at a sawtooth peak and no longer needs are counted inside heapTotal,
|
|
267
268
|
* and they are precisely what this number exists to see.
|
|
269
|
+
*
|
|
270
|
+
* ## What this number can and CANNOT be asked
|
|
271
|
+
*
|
|
272
|
+
* It answers exactly one question: *are there pages a `last-resort` GC could
|
|
273
|
+
* hand back?* For that it is correct, and {@link shouldReclaim} is the only
|
|
274
|
+
* gate that should ever be built on it.
|
|
275
|
+
*
|
|
276
|
+
* It can never answer *is this process too big?*, because `external` is
|
|
277
|
+
* subtracted by construction. A process whose growth is entirely `external` —
|
|
278
|
+
* Buffers, `arrayBuffers`, what a native addon mallocs — moves `rss` and the
|
|
279
|
+
* subtrahend by the same amount, so this number is **algebraically constant
|
|
280
|
+
* while the process triples**. That is not a threshold set too high; the
|
|
281
|
+
* quantity cannot represent the event.
|
|
282
|
+
*
|
|
283
|
+
* It cost 11.4GB and a ~154 second stall of the whole container to establish
|
|
284
|
+
* that. See {@link RssBudgetOptions} for the number that does answer it.
|
|
268
285
|
*/
|
|
269
286
|
function strandedMb(s) {
|
|
270
287
|
return s.rssMb - s.heapUsedMb - s.externalMb;
|
|
@@ -273,6 +290,67 @@ function strandedMb(s) {
|
|
|
273
290
|
function shouldReclaim(s, triggerMb = HEAP_RECLAIM_TRIGGER_MB) {
|
|
274
291
|
return strandedMb(s) > triggerMb;
|
|
275
292
|
}
|
|
293
|
+
/** Share of RSS, as a whole percent. Never divides by zero. */
|
|
294
|
+
function share(partMb, rssMb) {
|
|
295
|
+
return rssMb > 0 ? Math.round(partMb / rssMb * 100) : 0;
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Split a sample into the three shapes and name the dominant one.
|
|
299
|
+
*
|
|
300
|
+
* `nativeResidue` is clamped at 0: `process.memoryUsage()` reads its fields at
|
|
301
|
+
* slightly different instants, so `heapUsed + external` can transiently exceed
|
|
302
|
+
* `rss` and a negative residue would misclassify the sample.
|
|
303
|
+
*/
|
|
304
|
+
function describeRss(s, budgetMb) {
|
|
305
|
+
const nativeResidueMb = Math.max(0, strandedMb(s));
|
|
306
|
+
const shape = s.heapUsedMb >= s.externalMb && s.heapUsedMb >= nativeResidueMb ? "v8-heap" : s.externalMb >= nativeResidueMb ? "external" : "native-residue";
|
|
307
|
+
return {
|
|
308
|
+
rssMb: s.rssMb,
|
|
309
|
+
budgetMb,
|
|
310
|
+
overMb: Math.max(0, s.rssMb - budgetMb),
|
|
311
|
+
heapUsedMb: s.heapUsedMb,
|
|
312
|
+
externalMb: s.externalMb,
|
|
313
|
+
nativeResidueMb,
|
|
314
|
+
arrayBuffersMb: s.arrayBuffersMb,
|
|
315
|
+
shape
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
/** What the operator can actually DO about each shape. */
|
|
319
|
+
var SHAPE_ADVICE = {
|
|
320
|
+
"v8-heap": "live JS retainers — bounded by --max-old-space-size; the fix is the retained set",
|
|
321
|
+
external: "Buffers / native external — NO V8 flag bounds this, and the old-space ceiling never will",
|
|
322
|
+
"native-residue": "freed native memory the allocator has not returned — MALLOC_ARENA_MAX/VIPS_CONCURRENCY are already applied, so this lever is spent"
|
|
323
|
+
};
|
|
324
|
+
/**
|
|
325
|
+
* Release the alarm below this share of the budget.
|
|
326
|
+
*
|
|
327
|
+
* Hysteresis, not a second budget: every process here sawtooths (the reclaim
|
|
328
|
+
* pass alone moves hub-main by ~1.6GB), so a single threshold would print a
|
|
329
|
+
* pair of lines on every cycle. 0.9 of the budget is a real retreat.
|
|
330
|
+
*/
|
|
331
|
+
var RSS_BUDGET_RELEASE_RATIO = .9;
|
|
332
|
+
/**
|
|
333
|
+
* Floor on how often a crossing may be ANNOUNCED again.
|
|
334
|
+
*
|
|
335
|
+
* Hysteresis alone bounds flapping around the edge; it does not bound a process
|
|
336
|
+
* that genuinely swings across the budget every reclaim cycle
|
|
337
|
+
* ({@link HEAP_RECLAIM_MIN_INTERVAL_MS} is 2 minutes, so that is up to 30 pairs
|
|
338
|
+
* of lines an hour, forever). A process that oscillates across its budget is ONE
|
|
339
|
+
* story, and the operator needs it told once per quarter-hour, not thirty times.
|
|
340
|
+
*
|
|
341
|
+
* This is the discipline the rest of this module already follows and the one the
|
|
342
|
+
* log needed: on 2026-08-27 two real alarms were buried under 83,198 Hikvision
|
|
343
|
+
* heartbeat lines in 12h. A watchdog that shouts is a watchdog that gets muted.
|
|
344
|
+
*/
|
|
345
|
+
var RSS_BUDGET_REANNOUNCE_MIN_MS = 9e5;
|
|
346
|
+
/**
|
|
347
|
+
* The alarm state machine, pure so the thresholds are testable without
|
|
348
|
+
* allocating five gigabytes to reach them.
|
|
349
|
+
*/
|
|
350
|
+
function nextRssBudgetState(current, rssMb, budgetMb, releaseRatio = RSS_BUDGET_RELEASE_RATIO) {
|
|
351
|
+
if (current === "over") return rssMb < budgetMb * releaseRatio ? "within" : "over";
|
|
352
|
+
return rssMb > budgetMb ? "over" : "within";
|
|
353
|
+
}
|
|
276
354
|
function isGcFunction(value) {
|
|
277
355
|
return typeof value === "function";
|
|
278
356
|
}
|
|
@@ -300,8 +378,19 @@ function createV8Reclaimer() {
|
|
|
300
378
|
return;
|
|
301
379
|
}
|
|
302
380
|
}
|
|
303
|
-
|
|
304
|
-
|
|
381
|
+
/**
|
|
382
|
+
* The line that says HOW the process grew, not merely that it did.
|
|
383
|
+
*
|
|
384
|
+
* `heapUsed`, `external` and the native residue each carry their share of RSS,
|
|
385
|
+
* because the shares are what separate the three fixes — and on the process this
|
|
386
|
+
* module failed to catch, the shares were 9% / 74% / 16%, which names the answer
|
|
387
|
+
* on sight.
|
|
388
|
+
*/
|
|
389
|
+
function formatOverBudget(label, b, source) {
|
|
390
|
+
return `[mem] ${label} OVER RSS BUDGET rss=${b.rssMb}MB budget=${b.budgetMb}MB over=+${b.overMb}MB shape=${b.shape} heapUsed=${b.heapUsedMb}MB(${share(b.heapUsedMb, b.rssMb)}%) external=${b.externalMb}MB(${share(b.externalMb, b.rssMb)}%) nativeResidue=${b.nativeResidueMb}MB(${share(b.nativeResidueMb, b.rssMb)}%) arrayBuffers=${b.arrayBuffersMb}MB budgetFrom="${source}" — ${SHAPE_ADVICE[b.shape]}`;
|
|
391
|
+
}
|
|
392
|
+
function format(label, s, loop, budgetMb) {
|
|
393
|
+
const line = `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB` + (budgetMb === void 0 ? "" : ` rssBudget=${budgetMb}MB`);
|
|
305
394
|
if (loop === void 0) return line;
|
|
306
395
|
return `${line} loopP50=${loop.p50Ms}ms loopP99=${loop.p99Ms}ms loopMax=${loop.maxMs}ms`;
|
|
307
396
|
}
|
|
@@ -311,7 +400,7 @@ function format(label, s, loop) {
|
|
|
311
400
|
* `unref`ed so it can never hold the process open, and every read is wrapped:
|
|
312
401
|
* a diagnostic that can crash the process it watches is worse than none.
|
|
313
402
|
*/
|
|
314
|
-
function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true) {
|
|
403
|
+
function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true, rssBudget) {
|
|
315
404
|
const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
|
|
316
405
|
const now = reclaimOptions?.now ?? (() => Date.now());
|
|
317
406
|
const triggerMb = reclaimOptions?.triggerMb ?? 1024;
|
|
@@ -355,6 +444,39 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
355
444
|
};
|
|
356
445
|
let mode = "steady";
|
|
357
446
|
let lastLoggedAt = Number.NEGATIVE_INFINITY;
|
|
447
|
+
const budgetMb = rssBudget?.budgetMb !== void 0 && rssBudget.budgetMb > 0 ? rssBudget.budgetMb : void 0;
|
|
448
|
+
const budgetSource = rssBudget?.source ?? "unspecified";
|
|
449
|
+
const releaseRatio = rssBudget?.releaseRatio ?? .9;
|
|
450
|
+
const reannounceMinMs = rssBudget?.reannounceMinMs ?? 9e5;
|
|
451
|
+
let budgetState = "within";
|
|
452
|
+
let lastBudgetWarnAt = Number.NEGATIVE_INFINITY;
|
|
453
|
+
/** Only pair a "back within" with an over-budget line that was actually said. */
|
|
454
|
+
let overBudgetAnnounced = false;
|
|
455
|
+
/**
|
|
456
|
+
* One line per STATE CHANGE, never one per sample.
|
|
457
|
+
*
|
|
458
|
+
* The probe runs every {@link HEAP_WATCH_FAST_INTERVAL_MS}; warning on every
|
|
459
|
+
* sample above the budget would put 30 lines a minute in the hub's shared log
|
|
460
|
+
* for a process that is legitimately, steadily large. That is precisely the
|
|
461
|
+
* failure mode that buried two real alarms under 83,198 heartbeat lines on
|
|
462
|
+
* 2026-08-27, so the crossing is the event and the level is not.
|
|
463
|
+
*/
|
|
464
|
+
const checkRssBudget = (sample, at) => {
|
|
465
|
+
if (budgetMb === void 0) return;
|
|
466
|
+
const previous = budgetState;
|
|
467
|
+
budgetState = nextRssBudgetState(previous, sample.rssMb, budgetMb, releaseRatio);
|
|
468
|
+
if (budgetState === previous) return;
|
|
469
|
+
if (budgetState === "over") {
|
|
470
|
+
if (at - lastBudgetWarnAt < reannounceMinMs) return;
|
|
471
|
+
lastBudgetWarnAt = at;
|
|
472
|
+
overBudgetAnnounced = true;
|
|
473
|
+
sink.warn(formatOverBudget(label, describeRss(sample, budgetMb), budgetSource));
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
if (!overBudgetAnnounced) return;
|
|
477
|
+
overBudgetAnnounced = false;
|
|
478
|
+
sink.info(`[mem] ${label} back within its RSS budget — rss=${sample.rssMb}MB budget=${budgetMb}MB`);
|
|
479
|
+
};
|
|
358
480
|
/**
|
|
359
481
|
* PROBING and LOGGING are separate cadences, and that separation is the whole
|
|
360
482
|
* point.
|
|
@@ -380,18 +502,20 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
380
502
|
const due = at - lastLoggedAt >= intervalMs;
|
|
381
503
|
if (mode === "escalated" || due) {
|
|
382
504
|
lastLoggedAt = at;
|
|
383
|
-
const line = format(label, sample, loopDelay?.read());
|
|
505
|
+
const line = format(label, sample, loopDelay?.read(), budgetMb);
|
|
384
506
|
if (sample.nearLimit) sink.warn(`${line} — APPROACHING HEAP LIMIT`);
|
|
385
507
|
else if (mode === "escalated") sink.warn(`${line} — heap elevated, sampling every ${probeIntervalMs}ms`);
|
|
386
508
|
else sink.info(line);
|
|
387
509
|
}
|
|
388
510
|
if (previous === "escalated" && mode === "steady") sink.info(`[mem] ${label} heap back to routine — logging every ${intervalMs}ms`);
|
|
511
|
+
checkRssBudget(sample, at);
|
|
389
512
|
maybeReclaim(sample);
|
|
390
513
|
} catch {}
|
|
391
514
|
};
|
|
392
515
|
const timer = setInterval(tick, probeIntervalMs);
|
|
393
516
|
timer.unref?.();
|
|
394
517
|
tick();
|
|
518
|
+
if (rssBudget !== void 0 && budgetMb === void 0) sink.info(`[mem] ${label} is NOT WATCHED against an RSS budget — no budget declared (${budgetSource}). rss growth outside the V8 heap raises nothing here: --max-old-space-size bounds old space only, and stranded subtracts external by construction. Declare execution.rssBudgetMb once this process has a measured working set.`);
|
|
395
519
|
if (announceCeilingOrigin && heapCeilingOrigin(execArgv) === "v8-default") sink.info(`[mem] ${label} heap ceiling is V8's DEFAULT (${read().heapLimitMb}MB) — no --max-old-space-size on argv. Nothing here CHOSE that number; it is derived from host RAM and moves with it.`);
|
|
396
520
|
let stopped = false;
|
|
397
521
|
return () => {
|
|
@@ -417,6 +541,71 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
417
541
|
*/
|
|
418
542
|
var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
|
|
419
543
|
/**
|
|
544
|
+
* Parse a budget carried across a process boundary as an env string.
|
|
545
|
+
*
|
|
546
|
+
* Anything that is not a positive finite integer is `undefined` — i.e. the
|
|
547
|
+
* process is UNWATCHED and says so. A malformed budget must never round to a
|
|
548
|
+
* silent default: that is the failure this whole mechanism exists to remove.
|
|
549
|
+
*/
|
|
550
|
+
function parseRssBudgetMb(raw) {
|
|
551
|
+
if (raw === void 0) return void 0;
|
|
552
|
+
const parsed = Number(raw);
|
|
553
|
+
if (!Number.isFinite(parsed) || parsed <= 0) return void 0;
|
|
554
|
+
return Math.floor(parsed);
|
|
555
|
+
}
|
|
556
|
+
/** Env var carrying a runner's declared RSS budget from parent to child. */
|
|
557
|
+
var RUNNER_RSS_BUDGET_ENV = "CAMSTACK_RUNNER_RSS_BUDGET_MB";
|
|
558
|
+
/** Operator override for hub-main's budget, applied without a release. */
|
|
559
|
+
var HUB_RSS_BUDGET_ENV = "CAMSTACK_HUB_RSS_BUDGET_MB";
|
|
560
|
+
/**
|
|
561
|
+
* hub-main's declared RSS budget, in MB.
|
|
562
|
+
*
|
|
563
|
+
* ## Why the number lives HERE and not in the image
|
|
564
|
+
*
|
|
565
|
+
* hub-main has no manifest to declare it in, and the obvious alternative — the
|
|
566
|
+
* entrypoint that already knew about `CAMSTACK_HUB_HEAP_MB` — is the least
|
|
567
|
+
* deployable surface in this system. The Dockerfile says so itself: the image
|
|
568
|
+
* never contains this repo's backend, so a value written there reaches a running
|
|
569
|
+
* hub only on an image rebuild. `main.ts` is what `applyServerUpdate` replaces,
|
|
570
|
+
* which is exactly why the heartbeat was started from there in the first place.
|
|
571
|
+
* A budget that cannot ship is not a budget.
|
|
572
|
+
*
|
|
573
|
+
* ## Why 4096
|
|
574
|
+
*
|
|
575
|
+
* hub-main's entitlement is its old-space ceiling plus its measured native
|
|
576
|
+
* floor. The floor is 438MB, established live across five consecutive
|
|
577
|
+
* `gc(last-resort)` passes that landed within 4MB of each other
|
|
578
|
+
* (4559/4561/4560/4562/4563MB) — sqlite pages, libvips, sharp, thread stacks and
|
|
579
|
+
* node's own text. Against the 3072MB ceiling in force on 2026-08-27 that is
|
|
580
|
+
* ~3.5GB of legitimate footprint, and 4096 leaves it ~600MB of slack.
|
|
581
|
+
*
|
|
582
|
+
* **This WILL fire on the hub as measured on 2026-08-27, and that is the point.**
|
|
583
|
+
* That process sat at 4.87GiB with `external` at 360MB: ~2.8GB of it was native
|
|
584
|
+
* memory already freed and never returned, spread across 13 anonymous mappings
|
|
585
|
+
* >=64MB, with only 1295MB in V8 pages. `MALLOC_ARENA_MAX=2` and
|
|
586
|
+
* `VIPS_CONCURRENCY=1` were already applied — verified from `/proc/<pid>/environ`
|
|
587
|
+
* — so that lever is spent and the residue stands. The alarm reporting a
|
|
588
|
+
* genuinely over-budget process is the mechanism working; a number picked to sit
|
|
589
|
+
* just above today's RSS would be a rubber stamp.
|
|
590
|
+
*
|
|
591
|
+
* An operator who decides that residue is the cost of doing business moves it in
|
|
592
|
+
* one place with {@link HUB_RSS_BUDGET_ENV}, and the decision is on the record
|
|
593
|
+
* instead of being absorbed by a threshold nobody set.
|
|
594
|
+
*/
|
|
595
|
+
var HUB_MAIN_RSS_BUDGET_MB = 4096;
|
|
596
|
+
/** hub-main's budget: the declared number, or the operator's override. */
|
|
597
|
+
function hubMainRssBudget(env = process.env) {
|
|
598
|
+
const override = env[HUB_RSS_BUDGET_ENV];
|
|
599
|
+
if (override !== void 0) return {
|
|
600
|
+
budgetMb: parseRssBudgetMb(override),
|
|
601
|
+
source: HUB_RSS_BUDGET_ENV
|
|
602
|
+
};
|
|
603
|
+
return {
|
|
604
|
+
budgetMb: HUB_MAIN_RSS_BUDGET_MB,
|
|
605
|
+
source: "HUB_MAIN_RSS_BUDGET_MB (kernel/heap-watch)"
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
/**
|
|
420
609
|
* Start the heap watch + stranded-page reclaimer for ONE addon runner, iff it
|
|
421
610
|
* is a heavy one. Returns the stop function, or undefined when nothing started.
|
|
422
611
|
*
|
|
@@ -450,13 +639,17 @@ var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
|
|
|
450
639
|
function startRunnerHeapWatch(options) {
|
|
451
640
|
if (options.heapProfile !== "heavy") return void 0;
|
|
452
641
|
const intervalMs = options.intervalMs ?? 3e5;
|
|
453
|
-
|
|
642
|
+
const rssBudget = {
|
|
643
|
+
budgetMb: parseRssBudgetMb(options.rssBudgetMb),
|
|
644
|
+
source: `manifest execution.rssBudgetMb (via ${RUNNER_RSS_BUDGET_ENV})`
|
|
645
|
+
};
|
|
646
|
+
if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs, void 0, void 0, void 0, false, rssBudget);
|
|
454
647
|
let reclaimOptions = options.reclaimOptions;
|
|
455
648
|
if (reclaimOptions === void 0) {
|
|
456
649
|
const reclaimer = createV8Reclaimer();
|
|
457
650
|
reclaimOptions = reclaimer === void 0 ? void 0 : { reclaim: reclaimer };
|
|
458
651
|
}
|
|
459
|
-
return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, true);
|
|
652
|
+
return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, true, rssBudget);
|
|
460
653
|
}
|
|
461
654
|
//#endregion
|
|
462
655
|
//#region src/kernel/moleculer/addon-data-plane-facility.ts
|
|
@@ -1653,6 +1846,182 @@ async function readModifyWrite(input) {
|
|
|
1653
1846
|
return true;
|
|
1654
1847
|
}
|
|
1655
1848
|
//#endregion
|
|
1849
|
+
//#region src/kernel/moleculer/children-batch-loader.ts
|
|
1850
|
+
/**
|
|
1851
|
+
* "Who are my children?" asked by a thousand devices, answered in a handful of
|
|
1852
|
+
* calls.
|
|
1853
|
+
*
|
|
1854
|
+
* ## The number
|
|
1855
|
+
*
|
|
1856
|
+
* Every `BaseDevice` inherits a `getAccessoryChildren()` that returns `[]`, so
|
|
1857
|
+
* the accessory reconcile in `device-cap-proxy.ts` runs its orphan-prune step
|
|
1858
|
+
* for EVERY registered device — including the leaf accessories that can never
|
|
1859
|
+
* own one. Each pass asked `device-manager.getChildren` for its own id.
|
|
1860
|
+
* Measured on the live hub 2026-08-27, 120-second boot window, 1 017 devices:
|
|
1861
|
+
* **1 024 calls returning 919 rows in total**. A thousand round-trips, and a
|
|
1862
|
+
* thousand indexed scans behind them, to move less than one row each.
|
|
1863
|
+
*
|
|
1864
|
+
* ## Why a loader and not a caller that loops
|
|
1865
|
+
*
|
|
1866
|
+
* There is no loop to unroll. Reconcile is per-device by construction: it runs
|
|
1867
|
+
* inside `register()`, and again whenever a late feature-probe writes the
|
|
1868
|
+
* slice. The repetition is a THOUSAND INDEPENDENT CALLERS, not one caller
|
|
1869
|
+
* asking a thousand times — so the collapse has to happen between them.
|
|
1870
|
+
*
|
|
1871
|
+
* ## Why the batch is self-clocking, with no window to tune
|
|
1872
|
+
*
|
|
1873
|
+
* At most ONE batch call is in flight. Anything that arrives while it is in
|
|
1874
|
+
* flight lands in the next batch, dispatched the moment the current one
|
|
1875
|
+
* settles. That makes the batch size adapt to the arrival rate on its own:
|
|
1876
|
+
*
|
|
1877
|
+
* - Idle: the first caller flushes on the next macrotask. No added wait, and
|
|
1878
|
+
* no timer to pick a value for and be wrong about.
|
|
1879
|
+
* - Boot: `BaseDeviceProvider.onRestoreDevices` issues every top-level
|
|
1880
|
+
* `create()` at once and `runBounded(specs, 12)` spawns accessory children
|
|
1881
|
+
* twelve wide, so reconciles arrive in bursts — measured as 2 372 device
|
|
1882
|
+
* log lines in one 5-second bucket at boot+16s, against 62 at boot+142s.
|
|
1883
|
+
* Every arrival inside one round-trip collapses into the next call.
|
|
1884
|
+
* - Worst case: arrivals slower than a round-trip give batches of one, which
|
|
1885
|
+
* is exactly today's cost. The loader can never be worse than what it
|
|
1886
|
+
* replaces.
|
|
1887
|
+
*
|
|
1888
|
+
* A fixed debounce window would have to be guessed against that distribution,
|
|
1889
|
+
* and would tax the idle case to pay for the busy one.
|
|
1890
|
+
*
|
|
1891
|
+
* ## Degrading, rather than breaking, against an older hub
|
|
1892
|
+
*
|
|
1893
|
+
* `getChildrenBatch` is a NEW method: a hub that predates it rejects the call
|
|
1894
|
+
* (it does not silently strip the input and answer a different question, which
|
|
1895
|
+
* is what adding a `parentDeviceIds` FIELD to `getChildren` would have done).
|
|
1896
|
+
* Every waiter in a failed batch is then answered by the per-parent
|
|
1897
|
+
* `getChildren` it would have used anyway, so the reconcile is never denied
|
|
1898
|
+
* its answer. A rejection that names the method as unknown latches the
|
|
1899
|
+
* fallback permanently — re-probing a method the hub will never have, once per
|
|
1900
|
+
* batch for a whole boot, is the storm this exists to remove.
|
|
1901
|
+
*/
|
|
1902
|
+
/**
|
|
1903
|
+
* Narrow an untyped cap answer to the fields the reconcile reads.
|
|
1904
|
+
*
|
|
1905
|
+
* `callDeviceManager` is an untyped boundary, and the answer may come from a
|
|
1906
|
+
* hub of a different vintage. A row that does not carry an id, a stableId and
|
|
1907
|
+
* a name is not a child this code can prune ON, so it is dropped rather than
|
|
1908
|
+
* cast into shape — the prune step DELETES, and a half-parsed row is the input
|
|
1909
|
+
* that would make it delete the wrong thing.
|
|
1910
|
+
*/
|
|
1911
|
+
function parseChildRows(value) {
|
|
1912
|
+
if (!Array.isArray(value)) return null;
|
|
1913
|
+
const rows = [];
|
|
1914
|
+
for (const entry of value) {
|
|
1915
|
+
if (typeof entry !== "object" || entry === null) return null;
|
|
1916
|
+
const id = Reflect.get(entry, "id");
|
|
1917
|
+
const stableId = Reflect.get(entry, "stableId");
|
|
1918
|
+
const name = Reflect.get(entry, "name");
|
|
1919
|
+
const linkDeviceId = Reflect.get(entry, "linkDeviceId");
|
|
1920
|
+
if (typeof id !== "number" || typeof stableId !== "string" || typeof name !== "string") return null;
|
|
1921
|
+
rows.push({
|
|
1922
|
+
id,
|
|
1923
|
+
stableId,
|
|
1924
|
+
name,
|
|
1925
|
+
...typeof linkDeviceId === "number" || linkDeviceId === null ? { linkDeviceId } : {}
|
|
1926
|
+
});
|
|
1927
|
+
}
|
|
1928
|
+
return rows;
|
|
1929
|
+
}
|
|
1930
|
+
/**
|
|
1931
|
+
* Narrow a `getChildrenBatch` answer: a record keyed by parent id as a string.
|
|
1932
|
+
*
|
|
1933
|
+
* An EMPTY record is a valid answer — "none of these parents has children" —
|
|
1934
|
+
* and so is what a hub that answers the call but knows none of the parents
|
|
1935
|
+
* would send. Both mean "nothing to prune", which is the safe direction. Only
|
|
1936
|
+
* a shape that is not a record at all is `null`, and the loader treats that as
|
|
1937
|
+
* a failed batch and re-asks per parent.
|
|
1938
|
+
*/
|
|
1939
|
+
function parseChildrenByParent(value) {
|
|
1940
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return null;
|
|
1941
|
+
const out = /* @__PURE__ */ new Map();
|
|
1942
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
1943
|
+
const parentDeviceId = Number(key);
|
|
1944
|
+
if (!Number.isInteger(parentDeviceId)) return null;
|
|
1945
|
+
const rows = parseChildRows(entry);
|
|
1946
|
+
if (rows === null) return null;
|
|
1947
|
+
out.set(parentDeviceId, rows);
|
|
1948
|
+
}
|
|
1949
|
+
return out;
|
|
1950
|
+
}
|
|
1951
|
+
/**
|
|
1952
|
+
* True when an error says the hub has no such procedure — the version-skew
|
|
1953
|
+
* case, and the only one worth latching on. Anything else (a timeout, a busy
|
|
1954
|
+
* broker) is transient and must not disable batching for the process lifetime.
|
|
1955
|
+
*/
|
|
1956
|
+
function looksLikeUnknownMethod(err) {
|
|
1957
|
+
const lowered = (err instanceof Error ? err.message : String(err)).toLowerCase();
|
|
1958
|
+
return lowered.includes("not_found") || lowered.includes("no procedure") || lowered.includes("unknown method") || lowered.includes("no such method") || lowered.includes("not available");
|
|
1959
|
+
}
|
|
1960
|
+
function createChildrenBatchLoader(opts) {
|
|
1961
|
+
const maxBatch = Math.max(1, opts.maxBatch ?? 256);
|
|
1962
|
+
let pending = [];
|
|
1963
|
+
let flushScheduled = false;
|
|
1964
|
+
let flushing = false;
|
|
1965
|
+
let batchUnsupported = false;
|
|
1966
|
+
const answerOneByOne = async (waiters) => {
|
|
1967
|
+
await Promise.all(waiters.map(async (waiter) => {
|
|
1968
|
+
try {
|
|
1969
|
+
waiter.resolve(await opts.fetchOne(waiter.parentDeviceId));
|
|
1970
|
+
} catch (err) {
|
|
1971
|
+
waiter.reject(err);
|
|
1972
|
+
}
|
|
1973
|
+
}));
|
|
1974
|
+
};
|
|
1975
|
+
const runBatch = async (waiters) => {
|
|
1976
|
+
const parentDeviceIds = [...new Set(waiters.map((w) => w.parentDeviceId))];
|
|
1977
|
+
try {
|
|
1978
|
+
const byParent = await opts.fetchBatch(parentDeviceIds);
|
|
1979
|
+
for (const waiter of waiters) waiter.resolve(byParent.get(waiter.parentDeviceId) ?? []);
|
|
1980
|
+
} catch (err) {
|
|
1981
|
+
const unknown = looksLikeUnknownMethod(err);
|
|
1982
|
+
if (unknown) batchUnsupported = true;
|
|
1983
|
+
opts.logger.warn(unknown ? "device-manager.getChildrenBatch is not available on this hub — falling back to per-parent getChildren for the process lifetime" : "device-manager.getChildrenBatch failed — answering this batch per parent", { meta: {
|
|
1984
|
+
parents: parentDeviceIds.length,
|
|
1985
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1986
|
+
} });
|
|
1987
|
+
await answerOneByOne(waiters);
|
|
1988
|
+
}
|
|
1989
|
+
};
|
|
1990
|
+
const flush = async () => {
|
|
1991
|
+
flushing = true;
|
|
1992
|
+
try {
|
|
1993
|
+
while (pending.length > 0) {
|
|
1994
|
+
const batch = pending.slice(0, maxBatch);
|
|
1995
|
+
pending = pending.slice(maxBatch);
|
|
1996
|
+
if (batchUnsupported) await answerOneByOne(batch);
|
|
1997
|
+
else await runBatch(batch);
|
|
1998
|
+
}
|
|
1999
|
+
} finally {
|
|
2000
|
+
flushing = false;
|
|
2001
|
+
}
|
|
2002
|
+
};
|
|
2003
|
+
const scheduleFlush = () => {
|
|
2004
|
+
if (flushScheduled || flushing) return;
|
|
2005
|
+
flushScheduled = true;
|
|
2006
|
+
setTimeout(() => {
|
|
2007
|
+
flushScheduled = false;
|
|
2008
|
+
flush().catch((err) => {
|
|
2009
|
+
opts.logger.warn("children batch flush threw", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
2010
|
+
});
|
|
2011
|
+
}, 0);
|
|
2012
|
+
};
|
|
2013
|
+
return { load(parentDeviceId) {
|
|
2014
|
+
return new Promise((resolve, reject) => {
|
|
2015
|
+
pending.push({
|
|
2016
|
+
parentDeviceId,
|
|
2017
|
+
resolve,
|
|
2018
|
+
reject
|
|
2019
|
+
});
|
|
2020
|
+
scheduleFlush();
|
|
2021
|
+
});
|
|
2022
|
+
} };
|
|
2023
|
+
}
|
|
2024
|
+
//#endregion
|
|
1656
2025
|
//#region src/kernel/moleculer/device-cap-proxy.ts
|
|
1657
2026
|
/**
|
|
1658
2027
|
* Max number of accessory children spawned concurrently during a single
|
|
@@ -1943,6 +2312,19 @@ function createBrokerDeviceManagerApi(opts) {
|
|
|
1943
2312
|
api
|
|
1944
2313
|
});
|
|
1945
2314
|
let selfApi;
|
|
2315
|
+
const childrenLoader = createChildrenBatchLoader({
|
|
2316
|
+
logger: opts.logger,
|
|
2317
|
+
fetchBatch: async (parentDeviceIds) => {
|
|
2318
|
+
const parsed = parseChildrenByParent(await callDeviceManager(api, "getChildrenBatch", { parentDeviceIds }));
|
|
2319
|
+
if (parsed === null) throw new Error("getChildrenBatch returned a shape this kernel cannot read");
|
|
2320
|
+
return parsed;
|
|
2321
|
+
},
|
|
2322
|
+
fetchOne: async (parentDeviceId) => {
|
|
2323
|
+
const parsed = parseChildRows(await callDeviceManager(api, "getChildren", { parentDeviceId }));
|
|
2324
|
+
if (parsed === null) throw new Error("getChildren returned a shape this kernel cannot read");
|
|
2325
|
+
return parsed;
|
|
2326
|
+
}
|
|
2327
|
+
});
|
|
1946
2328
|
const deviceRebuildFactories = /* @__PURE__ */ new Map();
|
|
1947
2329
|
const buildContext = (stableId, id, parentDeviceId = null, initialRuntimeState = {}, persistedConfig = {}, deviceMeta = null) => {
|
|
1948
2330
|
let runtimeStateRef = null;
|
|
@@ -2360,7 +2742,7 @@ function createBrokerDeviceManagerApi(opts) {
|
|
|
2360
2742
|
return;
|
|
2361
2743
|
}
|
|
2362
2744
|
try {
|
|
2363
|
-
const currentChildren = await
|
|
2745
|
+
const currentChildren = await childrenLoader.load(device.id);
|
|
2364
2746
|
for (const child of currentChildren) {
|
|
2365
2747
|
if (child.linkDeviceId !== device.id) continue;
|
|
2366
2748
|
if (expected.has(child.stableId)) continue;
|
|
@@ -8306,6 +8688,18 @@ Object.defineProperty(exports, "HUB_CAP_FWD_SERVICE", {
|
|
|
8306
8688
|
return HUB_CAP_FWD_SERVICE;
|
|
8307
8689
|
}
|
|
8308
8690
|
});
|
|
8691
|
+
Object.defineProperty(exports, "HUB_MAIN_RSS_BUDGET_MB", {
|
|
8692
|
+
enumerable: true,
|
|
8693
|
+
get: function() {
|
|
8694
|
+
return HUB_MAIN_RSS_BUDGET_MB;
|
|
8695
|
+
}
|
|
8696
|
+
});
|
|
8697
|
+
Object.defineProperty(exports, "HUB_RSS_BUDGET_ENV", {
|
|
8698
|
+
enumerable: true,
|
|
8699
|
+
get: function() {
|
|
8700
|
+
return HUB_RSS_BUDGET_ENV;
|
|
8701
|
+
}
|
|
8702
|
+
});
|
|
8309
8703
|
Object.defineProperty(exports, "LocalChildClient", {
|
|
8310
8704
|
enumerable: true,
|
|
8311
8705
|
get: function() {
|
|
@@ -8324,12 +8718,30 @@ Object.defineProperty(exports, "NATIVE_PROVIDER_SERVICE_INFIX", {
|
|
|
8324
8718
|
return NATIVE_PROVIDER_SERVICE_INFIX;
|
|
8325
8719
|
}
|
|
8326
8720
|
});
|
|
8721
|
+
Object.defineProperty(exports, "RSS_BUDGET_REANNOUNCE_MIN_MS", {
|
|
8722
|
+
enumerable: true,
|
|
8723
|
+
get: function() {
|
|
8724
|
+
return RSS_BUDGET_REANNOUNCE_MIN_MS;
|
|
8725
|
+
}
|
|
8726
|
+
});
|
|
8727
|
+
Object.defineProperty(exports, "RSS_BUDGET_RELEASE_RATIO", {
|
|
8728
|
+
enumerable: true,
|
|
8729
|
+
get: function() {
|
|
8730
|
+
return RSS_BUDGET_RELEASE_RATIO;
|
|
8731
|
+
}
|
|
8732
|
+
});
|
|
8327
8733
|
Object.defineProperty(exports, "RUNNER_HEAP_WATCH_INTERVAL_MS", {
|
|
8328
8734
|
enumerable: true,
|
|
8329
8735
|
get: function() {
|
|
8330
8736
|
return RUNNER_HEAP_WATCH_INTERVAL_MS;
|
|
8331
8737
|
}
|
|
8332
8738
|
});
|
|
8739
|
+
Object.defineProperty(exports, "RUNNER_RSS_BUDGET_ENV", {
|
|
8740
|
+
enumerable: true,
|
|
8741
|
+
get: function() {
|
|
8742
|
+
return RUNNER_RSS_BUDGET_ENV;
|
|
8743
|
+
}
|
|
8744
|
+
});
|
|
8333
8745
|
Object.defineProperty(exports, "SocketChannel", {
|
|
8334
8746
|
enumerable: true,
|
|
8335
8747
|
get: function() {
|
|
@@ -8534,6 +8946,12 @@ Object.defineProperty(exports, "createV8Reclaimer", {
|
|
|
8534
8946
|
return createV8Reclaimer;
|
|
8535
8947
|
}
|
|
8536
8948
|
});
|
|
8949
|
+
Object.defineProperty(exports, "describeRss", {
|
|
8950
|
+
enumerable: true,
|
|
8951
|
+
get: function() {
|
|
8952
|
+
return describeRss;
|
|
8953
|
+
}
|
|
8954
|
+
});
|
|
8537
8955
|
Object.defineProperty(exports, "deserializeTypedArrays", {
|
|
8538
8956
|
enumerable: true,
|
|
8539
8957
|
get: function() {
|
|
@@ -8594,6 +9012,12 @@ Object.defineProperty(exports, "getWorkerNativeCapSnapshot", {
|
|
|
8594
9012
|
return getWorkerNativeCapSnapshot;
|
|
8595
9013
|
}
|
|
8596
9014
|
});
|
|
9015
|
+
Object.defineProperty(exports, "hubMainRssBudget", {
|
|
9016
|
+
enumerable: true,
|
|
9017
|
+
get: function() {
|
|
9018
|
+
return hubMainRssBudget;
|
|
9019
|
+
}
|
|
9020
|
+
});
|
|
8597
9021
|
Object.defineProperty(exports, "installManifestNativeDeps", {
|
|
8598
9022
|
enumerable: true,
|
|
8599
9023
|
get: function() {
|
|
@@ -8630,12 +9054,24 @@ Object.defineProperty(exports, "mountNativeCapService", {
|
|
|
8630
9054
|
return mountNativeCapService;
|
|
8631
9055
|
}
|
|
8632
9056
|
});
|
|
9057
|
+
Object.defineProperty(exports, "nextRssBudgetState", {
|
|
9058
|
+
enumerable: true,
|
|
9059
|
+
get: function() {
|
|
9060
|
+
return nextRssBudgetState;
|
|
9061
|
+
}
|
|
9062
|
+
});
|
|
8633
9063
|
Object.defineProperty(exports, "parseCapAction", {
|
|
8634
9064
|
enumerable: true,
|
|
8635
9065
|
get: function() {
|
|
8636
9066
|
return parseCapAction;
|
|
8637
9067
|
}
|
|
8638
9068
|
});
|
|
9069
|
+
Object.defineProperty(exports, "parseRssBudgetMb", {
|
|
9070
|
+
enumerable: true,
|
|
9071
|
+
get: function() {
|
|
9072
|
+
return parseRssBudgetMb;
|
|
9073
|
+
}
|
|
9074
|
+
});
|
|
8639
9075
|
Object.defineProperty(exports, "registerEventBusService", {
|
|
8640
9076
|
enumerable: true,
|
|
8641
9077
|
get: function() {
|