@camstack/system 1.2.144 → 1.2.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-runner.js +2 -1
- package/dist/addon-runner.mjs +2 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +1 -1
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
- package/dist/builtins/device-manager/device-manager.addon.js +157 -35
- package/dist/builtins/device-manager/device-manager.addon.mjs +157 -35
- package/dist/builtins/device-manager/device-queries.d.ts +22 -0
- package/dist/builtins/device-manager/device-row-store.d.ts +18 -0
- package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +1 -1
- package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
- package/dist/builtins/local-network/local-network.addon.js +1 -1
- package/dist/builtins/local-network/local-network.addon.mjs +1 -1
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.js +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1 -1
- package/dist/builtins/platform-probe/index.js +1 -1
- package/dist/builtins/platform-probe/index.mjs +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +1 -1
- package/dist/builtins/snapshot/index.mjs +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings-backend.d.ts +14 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +0 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +0 -0
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +2 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +2 -1
- package/dist/builtins/system-config/system-config.addon.js +1 -1
- package/dist/builtins/system-config/system-config.addon.mjs +1 -1
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{dist-D3lqzV40.js → dist-BVU5JADq.js} +401 -24
- package/dist/{dist-6rM7chzb.mjs → dist-CG7kwIZp.mjs} +401 -24
- package/dist/index.d.ts +2 -2
- package/dist/index.js +60 -11
- package/dist/index.mjs +52 -12
- package/dist/kernel/heap-watch.d.ts +209 -1
- package/dist/kernel/moleculer/children-batch-loader.d.ts +46 -0
- package/dist/kernel/moleculer/process-service.d.ts +15 -6
- package/dist/{manifest-python-deps-BuhBXCg0.mjs → manifest-python-deps-CVeckyXx.mjs} +390 -8
- package/dist/{manifest-python-deps-B_mCU6gz.js → manifest-python-deps-CktMcXzS.js} +443 -7
- package/dist/{retired-settings-keys-BtvClSPu.mjs → retired-settings-keys-Dm1z4mO4.mjs} +1 -1
- package/dist/{retired-settings-keys-OtUQtEbq.js → retired-settings-keys-_PLI9w0k.js} +1 -1
- package/package.json +1 -1
|
@@ -16,10 +16,20 @@ interface RunnerAddonSpec {
|
|
|
16
16
|
* one-off burst becomes the runner's permanent RSS.
|
|
17
17
|
*
|
|
18
18
|
* Measured on the live hub, 2026-08-17: the `recorder` runner reaches **2122 MB RSS** and stays
|
|
19
|
-
* there
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
19
|
+
* there. The same addon on two agent nodes carrying the same code and a smaller archive sits at
|
|
20
|
+
* **58 MB**.
|
|
21
|
+
*
|
|
22
|
+
* ⚠ This paragraph used to attribute that ~2 GB to "walk garbage" from the deferred archive walk,
|
|
23
|
+
* and to argue from a ~21 MB `RecordingIndex`. Both were wrong, and the error cost a session on
|
|
24
|
+
* 2026-08-27. The deferred walk had NO CALLER and never ran (it has since been deleted). And the
|
|
25
|
+
* index is not 21 MB: a 2026-08-27 profile of the live recorder read `heapUsed = 1606 MB` against
|
|
26
|
+
* `external = 8 MB`, with an `smaps` census putting 1510 MB in V8's 256 KB page ranges against
|
|
27
|
+
* 54 MB of native heap — LIVE JavaScript retainers, not high-water garbage. The retained set is
|
|
28
|
+
* ~7.1 M `SegmentRow`s plus their path strings and Map entries (13 cameras x 2 profiles x 8 640
|
|
29
|
+
* segments/day x ~30 days), reseeded at boot by `SegmentHourLedger.hydrateIndex`.
|
|
30
|
+
*
|
|
31
|
+
* The lesson for anyone sizing this flag: a heap number needs a RETAINER, and "garbage the GC has
|
|
32
|
+
* no reason to collect" is the explanation to reach for last, not first.
|
|
23
33
|
*
|
|
24
34
|
* ## Why 1024
|
|
25
35
|
*
|
|
@@ -36,8 +46,7 @@ interface RunnerAddonSpec {
|
|
|
36
46
|
*
|
|
37
47
|
* A runner whose working set genuinely does not fit declares its own number in its manifest
|
|
38
48
|
* (`execution.maxOldSpaceMb`) — the number belongs next to the addon that knows its working set.
|
|
39
|
-
* `recorder` (3072 MB; peak 2341 MB heapTotal
|
|
40
|
-
* shipped addon that does. `stream-broker` does NOT, despite 1585 MB peak RSS: ~1 GB of that is
|
|
49
|
+
* `recorder` (3072 MB; peak 2341 MB heapTotal) is the one shipped addon that does. `stream-broker` does NOT, despite 1585 MB peak RSS: ~1 GB of that is
|
|
41
50
|
* `arrayBuffers` (47 RTP rings), which this flag does not bound and never will — its actual old
|
|
42
51
|
* space peaks at 224 MB. Size the declaration from `heapTotal` in the `[mem]` heartbeat, never
|
|
43
52
|
* from RSS.
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import "./dist-CG7kwIZp.mjs";
|
|
1
2
|
import * as crypto$1 from "node:crypto";
|
|
2
3
|
import { randomBytes, randomUUID } from "node:crypto";
|
|
3
4
|
import { ensureBinary, ensureFfmpeg, ensurePython, installPythonPackages, installPythonRequirements } from "@camstack/types/node";
|
|
@@ -261,6 +262,22 @@ var consoleSink = {
|
|
|
261
262
|
* The subtrahend is heapUSED, not heapTotal, deliberately: the pages V8
|
|
262
263
|
* committed at a sawtooth peak and no longer needs are counted inside heapTotal,
|
|
263
264
|
* and they are precisely what this number exists to see.
|
|
265
|
+
*
|
|
266
|
+
* ## What this number can and CANNOT be asked
|
|
267
|
+
*
|
|
268
|
+
* It answers exactly one question: *are there pages a `last-resort` GC could
|
|
269
|
+
* hand back?* For that it is correct, and {@link shouldReclaim} is the only
|
|
270
|
+
* gate that should ever be built on it.
|
|
271
|
+
*
|
|
272
|
+
* It can never answer *is this process too big?*, because `external` is
|
|
273
|
+
* subtracted by construction. A process whose growth is entirely `external` —
|
|
274
|
+
* Buffers, `arrayBuffers`, what a native addon mallocs — moves `rss` and the
|
|
275
|
+
* subtrahend by the same amount, so this number is **algebraically constant
|
|
276
|
+
* while the process triples**. That is not a threshold set too high; the
|
|
277
|
+
* quantity cannot represent the event.
|
|
278
|
+
*
|
|
279
|
+
* It cost 11.4GB and a ~154 second stall of the whole container to establish
|
|
280
|
+
* that. See {@link RssBudgetOptions} for the number that does answer it.
|
|
264
281
|
*/
|
|
265
282
|
function strandedMb(s) {
|
|
266
283
|
return s.rssMb - s.heapUsedMb - s.externalMb;
|
|
@@ -269,6 +286,67 @@ function strandedMb(s) {
|
|
|
269
286
|
function shouldReclaim(s, triggerMb = HEAP_RECLAIM_TRIGGER_MB) {
|
|
270
287
|
return strandedMb(s) > triggerMb;
|
|
271
288
|
}
|
|
289
|
+
/** Share of RSS, as a whole percent. Never divides by zero. */
|
|
290
|
+
function share(partMb, rssMb) {
|
|
291
|
+
return rssMb > 0 ? Math.round(partMb / rssMb * 100) : 0;
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Split a sample into the three shapes and name the dominant one.
|
|
295
|
+
*
|
|
296
|
+
* `nativeResidue` is clamped at 0: `process.memoryUsage()` reads its fields at
|
|
297
|
+
* slightly different instants, so `heapUsed + external` can transiently exceed
|
|
298
|
+
* `rss` and a negative residue would misclassify the sample.
|
|
299
|
+
*/
|
|
300
|
+
function describeRss(s, budgetMb) {
|
|
301
|
+
const nativeResidueMb = Math.max(0, strandedMb(s));
|
|
302
|
+
const shape = s.heapUsedMb >= s.externalMb && s.heapUsedMb >= nativeResidueMb ? "v8-heap" : s.externalMb >= nativeResidueMb ? "external" : "native-residue";
|
|
303
|
+
return {
|
|
304
|
+
rssMb: s.rssMb,
|
|
305
|
+
budgetMb,
|
|
306
|
+
overMb: Math.max(0, s.rssMb - budgetMb),
|
|
307
|
+
heapUsedMb: s.heapUsedMb,
|
|
308
|
+
externalMb: s.externalMb,
|
|
309
|
+
nativeResidueMb,
|
|
310
|
+
arrayBuffersMb: s.arrayBuffersMb,
|
|
311
|
+
shape
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
/** What the operator can actually DO about each shape. */
|
|
315
|
+
var SHAPE_ADVICE = {
|
|
316
|
+
"v8-heap": "live JS retainers — bounded by --max-old-space-size; the fix is the retained set",
|
|
317
|
+
external: "Buffers / native external — NO V8 flag bounds this, and the old-space ceiling never will",
|
|
318
|
+
"native-residue": "freed native memory the allocator has not returned — MALLOC_ARENA_MAX/VIPS_CONCURRENCY are already applied, so this lever is spent"
|
|
319
|
+
};
|
|
320
|
+
/**
|
|
321
|
+
* Release the alarm below this share of the budget.
|
|
322
|
+
*
|
|
323
|
+
* Hysteresis, not a second budget: every process here sawtooths (the reclaim
|
|
324
|
+
* pass alone moves hub-main by ~1.6GB), so a single threshold would print a
|
|
325
|
+
* pair of lines on every cycle. 0.9 of the budget is a real retreat.
|
|
326
|
+
*/
|
|
327
|
+
var RSS_BUDGET_RELEASE_RATIO = .9;
|
|
328
|
+
/**
|
|
329
|
+
* Floor on how often a crossing may be ANNOUNCED again.
|
|
330
|
+
*
|
|
331
|
+
* Hysteresis alone bounds flapping around the edge; it does not bound a process
|
|
332
|
+
* that genuinely swings across the budget every reclaim cycle
|
|
333
|
+
* ({@link HEAP_RECLAIM_MIN_INTERVAL_MS} is 2 minutes, so that is up to 30 pairs
|
|
334
|
+
* of lines an hour, forever). A process that oscillates across its budget is ONE
|
|
335
|
+
* story, and the operator needs it told once per quarter-hour, not thirty times.
|
|
336
|
+
*
|
|
337
|
+
* This is the discipline the rest of this module already follows and the one the
|
|
338
|
+
* log needed: on 2026-08-27 two real alarms were buried under 83,198 Hikvision
|
|
339
|
+
* heartbeat lines in 12h. A watchdog that shouts is a watchdog that gets muted.
|
|
340
|
+
*/
|
|
341
|
+
var RSS_BUDGET_REANNOUNCE_MIN_MS = 9e5;
|
|
342
|
+
/**
|
|
343
|
+
* The alarm state machine, pure so the thresholds are testable without
|
|
344
|
+
* allocating five gigabytes to reach them.
|
|
345
|
+
*/
|
|
346
|
+
function nextRssBudgetState(current, rssMb, budgetMb, releaseRatio = RSS_BUDGET_RELEASE_RATIO) {
|
|
347
|
+
if (current === "over") return rssMb < budgetMb * releaseRatio ? "within" : "over";
|
|
348
|
+
return rssMb > budgetMb ? "over" : "within";
|
|
349
|
+
}
|
|
272
350
|
function isGcFunction(value) {
|
|
273
351
|
return typeof value === "function";
|
|
274
352
|
}
|
|
@@ -296,8 +374,19 @@ function createV8Reclaimer() {
|
|
|
296
374
|
return;
|
|
297
375
|
}
|
|
298
376
|
}
|
|
299
|
-
|
|
300
|
-
|
|
377
|
+
/**
|
|
378
|
+
* The line that says HOW the process grew, not merely that it did.
|
|
379
|
+
*
|
|
380
|
+
* `heapUsed`, `external` and the native residue each carry their share of RSS,
|
|
381
|
+
* because the shares are what separate the three fixes — and on the process this
|
|
382
|
+
* module failed to catch, the shares were 9% / 74% / 16%, which names the answer
|
|
383
|
+
* on sight.
|
|
384
|
+
*/
|
|
385
|
+
function formatOverBudget(label, b, source) {
|
|
386
|
+
return `[mem] ${label} OVER RSS BUDGET rss=${b.rssMb}MB budget=${b.budgetMb}MB over=+${b.overMb}MB shape=${b.shape} heapUsed=${b.heapUsedMb}MB(${share(b.heapUsedMb, b.rssMb)}%) external=${b.externalMb}MB(${share(b.externalMb, b.rssMb)}%) nativeResidue=${b.nativeResidueMb}MB(${share(b.nativeResidueMb, b.rssMb)}%) arrayBuffers=${b.arrayBuffersMb}MB budgetFrom="${source}" — ${SHAPE_ADVICE[b.shape]}`;
|
|
387
|
+
}
|
|
388
|
+
function format(label, s, loop, budgetMb) {
|
|
389
|
+
const line = `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB` + (budgetMb === void 0 ? "" : ` rssBudget=${budgetMb}MB`);
|
|
301
390
|
if (loop === void 0) return line;
|
|
302
391
|
return `${line} loopP50=${loop.p50Ms}ms loopP99=${loop.p99Ms}ms loopMax=${loop.maxMs}ms`;
|
|
303
392
|
}
|
|
@@ -307,7 +396,7 @@ function format(label, s, loop) {
|
|
|
307
396
|
* `unref`ed so it can never hold the process open, and every read is wrapped:
|
|
308
397
|
* a diagnostic that can crash the process it watches is worse than none.
|
|
309
398
|
*/
|
|
310
|
-
function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true) {
|
|
399
|
+
function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions, loopDelay = createLoopDelayMeter(), execArgv = process.execArgv, announceCeilingOrigin = true, rssBudget) {
|
|
311
400
|
const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
|
|
312
401
|
const now = reclaimOptions?.now ?? (() => Date.now());
|
|
313
402
|
const triggerMb = reclaimOptions?.triggerMb ?? 1024;
|
|
@@ -351,6 +440,39 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
351
440
|
};
|
|
352
441
|
let mode = "steady";
|
|
353
442
|
let lastLoggedAt = Number.NEGATIVE_INFINITY;
|
|
443
|
+
const budgetMb = rssBudget?.budgetMb !== void 0 && rssBudget.budgetMb > 0 ? rssBudget.budgetMb : void 0;
|
|
444
|
+
const budgetSource = rssBudget?.source ?? "unspecified";
|
|
445
|
+
const releaseRatio = rssBudget?.releaseRatio ?? .9;
|
|
446
|
+
const reannounceMinMs = rssBudget?.reannounceMinMs ?? 9e5;
|
|
447
|
+
let budgetState = "within";
|
|
448
|
+
let lastBudgetWarnAt = Number.NEGATIVE_INFINITY;
|
|
449
|
+
/** Only pair a "back within" with an over-budget line that was actually said. */
|
|
450
|
+
let overBudgetAnnounced = false;
|
|
451
|
+
/**
|
|
452
|
+
* One line per STATE CHANGE, never one per sample.
|
|
453
|
+
*
|
|
454
|
+
* The probe runs every {@link HEAP_WATCH_FAST_INTERVAL_MS}; warning on every
|
|
455
|
+
* sample above the budget would put 30 lines a minute in the hub's shared log
|
|
456
|
+
* for a process that is legitimately, steadily large. That is precisely the
|
|
457
|
+
* failure mode that buried two real alarms under 83,198 heartbeat lines on
|
|
458
|
+
* 2026-08-27, so the crossing is the event and the level is not.
|
|
459
|
+
*/
|
|
460
|
+
const checkRssBudget = (sample, at) => {
|
|
461
|
+
if (budgetMb === void 0) return;
|
|
462
|
+
const previous = budgetState;
|
|
463
|
+
budgetState = nextRssBudgetState(previous, sample.rssMb, budgetMb, releaseRatio);
|
|
464
|
+
if (budgetState === previous) return;
|
|
465
|
+
if (budgetState === "over") {
|
|
466
|
+
if (at - lastBudgetWarnAt < reannounceMinMs) return;
|
|
467
|
+
lastBudgetWarnAt = at;
|
|
468
|
+
overBudgetAnnounced = true;
|
|
469
|
+
sink.warn(formatOverBudget(label, describeRss(sample, budgetMb), budgetSource));
|
|
470
|
+
return;
|
|
471
|
+
}
|
|
472
|
+
if (!overBudgetAnnounced) return;
|
|
473
|
+
overBudgetAnnounced = false;
|
|
474
|
+
sink.info(`[mem] ${label} back within its RSS budget — rss=${sample.rssMb}MB budget=${budgetMb}MB`);
|
|
475
|
+
};
|
|
354
476
|
/**
|
|
355
477
|
* PROBING and LOGGING are separate cadences, and that separation is the whole
|
|
356
478
|
* point.
|
|
@@ -376,18 +498,20 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
376
498
|
const due = at - lastLoggedAt >= intervalMs;
|
|
377
499
|
if (mode === "escalated" || due) {
|
|
378
500
|
lastLoggedAt = at;
|
|
379
|
-
const line = format(label, sample, loopDelay?.read());
|
|
501
|
+
const line = format(label, sample, loopDelay?.read(), budgetMb);
|
|
380
502
|
if (sample.nearLimit) sink.warn(`${line} — APPROACHING HEAP LIMIT`);
|
|
381
503
|
else if (mode === "escalated") sink.warn(`${line} — heap elevated, sampling every ${probeIntervalMs}ms`);
|
|
382
504
|
else sink.info(line);
|
|
383
505
|
}
|
|
384
506
|
if (previous === "escalated" && mode === "steady") sink.info(`[mem] ${label} heap back to routine — logging every ${intervalMs}ms`);
|
|
507
|
+
checkRssBudget(sample, at);
|
|
385
508
|
maybeReclaim(sample);
|
|
386
509
|
} catch {}
|
|
387
510
|
};
|
|
388
511
|
const timer = setInterval(tick, probeIntervalMs);
|
|
389
512
|
timer.unref?.();
|
|
390
513
|
tick();
|
|
514
|
+
if (rssBudget !== void 0 && budgetMb === void 0) sink.info(`[mem] ${label} is NOT WATCHED against an RSS budget — no budget declared (${budgetSource}). rss growth outside the V8 heap raises nothing here: --max-old-space-size bounds old space only, and stranded subtracts external by construction. Declare execution.rssBudgetMb once this process has a measured working set.`);
|
|
391
515
|
if (announceCeilingOrigin && heapCeilingOrigin(execArgv) === "v8-default") sink.info(`[mem] ${label} heap ceiling is V8's DEFAULT (${read().heapLimitMb}MB) — no --max-old-space-size on argv. Nothing here CHOSE that number; it is derived from host RAM and moves with it.`);
|
|
392
516
|
let stopped = false;
|
|
393
517
|
return () => {
|
|
@@ -413,6 +537,71 @@ function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEA
|
|
|
413
537
|
*/
|
|
414
538
|
var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
|
|
415
539
|
/**
|
|
540
|
+
* Parse a budget carried across a process boundary as an env string.
|
|
541
|
+
*
|
|
542
|
+
* Anything that is not a positive finite integer is `undefined` — i.e. the
|
|
543
|
+
* process is UNWATCHED and says so. A malformed budget must never round to a
|
|
544
|
+
* silent default: that is the failure this whole mechanism exists to remove.
|
|
545
|
+
*/
|
|
546
|
+
function parseRssBudgetMb(raw) {
|
|
547
|
+
if (raw === void 0) return void 0;
|
|
548
|
+
const parsed = Number(raw);
|
|
549
|
+
if (!Number.isFinite(parsed) || parsed <= 0) return void 0;
|
|
550
|
+
return Math.floor(parsed);
|
|
551
|
+
}
|
|
552
|
+
/** Env var carrying a runner's declared RSS budget from parent to child. */
|
|
553
|
+
var RUNNER_RSS_BUDGET_ENV = "CAMSTACK_RUNNER_RSS_BUDGET_MB";
|
|
554
|
+
/** Operator override for hub-main's budget, applied without a release. */
|
|
555
|
+
var HUB_RSS_BUDGET_ENV = "CAMSTACK_HUB_RSS_BUDGET_MB";
|
|
556
|
+
/**
|
|
557
|
+
* hub-main's declared RSS budget, in MB.
|
|
558
|
+
*
|
|
559
|
+
* ## Why the number lives HERE and not in the image
|
|
560
|
+
*
|
|
561
|
+
* hub-main has no manifest to declare it in, and the obvious alternative — the
|
|
562
|
+
* entrypoint that already knew about `CAMSTACK_HUB_HEAP_MB` — is the least
|
|
563
|
+
* deployable surface in this system. The Dockerfile says so itself: the image
|
|
564
|
+
* never contains this repo's backend, so a value written there reaches a running
|
|
565
|
+
* hub only on an image rebuild. `main.ts` is what `applyServerUpdate` replaces,
|
|
566
|
+
* which is exactly why the heartbeat was started from there in the first place.
|
|
567
|
+
* A budget that cannot ship is not a budget.
|
|
568
|
+
*
|
|
569
|
+
* ## Why 4096
|
|
570
|
+
*
|
|
571
|
+
* hub-main's entitlement is its old-space ceiling plus its measured native
|
|
572
|
+
* floor. The floor is 438MB, established live across five consecutive
|
|
573
|
+
* `gc(last-resort)` passes that landed within 4MB of each other
|
|
574
|
+
* (4559/4561/4560/4562/4563MB) — sqlite pages, libvips, sharp, thread stacks and
|
|
575
|
+
* node's own text. Against the 3072MB ceiling in force on 2026-08-27 that is
|
|
576
|
+
* ~3.5GB of legitimate footprint, and 4096 leaves it ~600MB of slack.
|
|
577
|
+
*
|
|
578
|
+
* **This WILL fire on the hub as measured on 2026-08-27, and that is the point.**
|
|
579
|
+
* That process sat at 4.87GiB with `external` at 360MB: ~2.8GB of it was native
|
|
580
|
+
* memory already freed and never returned, spread across 13 anonymous mappings
|
|
581
|
+
* >=64MB, with only 1295MB in V8 pages. `MALLOC_ARENA_MAX=2` and
|
|
582
|
+
* `VIPS_CONCURRENCY=1` were already applied — verified from `/proc/<pid>/environ`
|
|
583
|
+
* — so that lever is spent and the residue stands. The alarm reporting a
|
|
584
|
+
* genuinely over-budget process is the mechanism working; a number picked to sit
|
|
585
|
+
* just above today's RSS would be a rubber stamp.
|
|
586
|
+
*
|
|
587
|
+
* An operator who decides that residue is the cost of doing business moves it in
|
|
588
|
+
* one place with {@link HUB_RSS_BUDGET_ENV}, and the decision is on the record
|
|
589
|
+
* instead of being absorbed by a threshold nobody set.
|
|
590
|
+
*/
|
|
591
|
+
var HUB_MAIN_RSS_BUDGET_MB = 4096;
|
|
592
|
+
/** hub-main's budget: the declared number, or the operator's override. */
|
|
593
|
+
function hubMainRssBudget(env = process.env) {
|
|
594
|
+
const override = env[HUB_RSS_BUDGET_ENV];
|
|
595
|
+
if (override !== void 0) return {
|
|
596
|
+
budgetMb: parseRssBudgetMb(override),
|
|
597
|
+
source: HUB_RSS_BUDGET_ENV
|
|
598
|
+
};
|
|
599
|
+
return {
|
|
600
|
+
budgetMb: HUB_MAIN_RSS_BUDGET_MB,
|
|
601
|
+
source: "HUB_MAIN_RSS_BUDGET_MB (kernel/heap-watch)"
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
/**
|
|
416
605
|
* Start the heap watch + stranded-page reclaimer for ONE addon runner, iff it
|
|
417
606
|
* is a heavy one. Returns the stop function, or undefined when nothing started.
|
|
418
607
|
*
|
|
@@ -446,13 +635,17 @@ var RUNNER_HEAP_WATCH_INTERVAL_MS = 3e5;
|
|
|
446
635
|
function startRunnerHeapWatch(options) {
|
|
447
636
|
if (options.heapProfile !== "heavy") return void 0;
|
|
448
637
|
const intervalMs = options.intervalMs ?? 3e5;
|
|
449
|
-
|
|
638
|
+
const rssBudget = {
|
|
639
|
+
budgetMb: parseRssBudgetMb(options.rssBudgetMb),
|
|
640
|
+
source: `manifest execution.rssBudgetMb (via ${RUNNER_RSS_BUDGET_ENV})`
|
|
641
|
+
};
|
|
642
|
+
if (options.reclaimSwitch === "off") return startHeapWatch(options.label, options.sink, intervalMs, void 0, void 0, void 0, false, rssBudget);
|
|
450
643
|
let reclaimOptions = options.reclaimOptions;
|
|
451
644
|
if (reclaimOptions === void 0) {
|
|
452
645
|
const reclaimer = createV8Reclaimer();
|
|
453
646
|
reclaimOptions = reclaimer === void 0 ? void 0 : { reclaim: reclaimer };
|
|
454
647
|
}
|
|
455
|
-
return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, true);
|
|
648
|
+
return startHeapWatch(options.label, options.sink, intervalMs, reclaimOptions, void 0, void 0, true, rssBudget);
|
|
456
649
|
}
|
|
457
650
|
//#endregion
|
|
458
651
|
//#region src/kernel/moleculer/addon-data-plane-facility.ts
|
|
@@ -1649,6 +1842,182 @@ async function readModifyWrite(input) {
|
|
|
1649
1842
|
return true;
|
|
1650
1843
|
}
|
|
1651
1844
|
//#endregion
|
|
1845
|
+
//#region src/kernel/moleculer/children-batch-loader.ts
|
|
1846
|
+
/**
|
|
1847
|
+
* "Who are my children?" asked by a thousand devices, answered in a handful of
|
|
1848
|
+
* calls.
|
|
1849
|
+
*
|
|
1850
|
+
* ## The number
|
|
1851
|
+
*
|
|
1852
|
+
* Every `BaseDevice` inherits a `getAccessoryChildren()` that returns `[]`, so
|
|
1853
|
+
* the accessory reconcile in `device-cap-proxy.ts` runs its orphan-prune step
|
|
1854
|
+
* for EVERY registered device — including the leaf accessories that can never
|
|
1855
|
+
* own one. Each pass asked `device-manager.getChildren` for its own id.
|
|
1856
|
+
* Measured on the live hub 2026-08-27, 120-second boot window, 1 017 devices:
|
|
1857
|
+
* **1 024 calls returning 919 rows in total**. A thousand round-trips, and a
|
|
1858
|
+
* thousand indexed scans behind them, to move less than one row each.
|
|
1859
|
+
*
|
|
1860
|
+
* ## Why a loader and not a caller that loops
|
|
1861
|
+
*
|
|
1862
|
+
* There is no loop to unroll. Reconcile is per-device by construction: it runs
|
|
1863
|
+
* inside `register()`, and again whenever a late feature-probe writes the
|
|
1864
|
+
* slice. The repetition is a THOUSAND INDEPENDENT CALLERS, not one caller
|
|
1865
|
+
* asking a thousand times — so the collapse has to happen between them.
|
|
1866
|
+
*
|
|
1867
|
+
* ## Why the batch is self-clocking, with no window to tune
|
|
1868
|
+
*
|
|
1869
|
+
* At most ONE batch call is in flight. Anything that arrives while it is in
|
|
1870
|
+
* flight lands in the next batch, dispatched the moment the current one
|
|
1871
|
+
* settles. That makes the batch size adapt to the arrival rate on its own:
|
|
1872
|
+
*
|
|
1873
|
+
* - Idle: the first caller flushes on the next macrotask. No added wait, and
|
|
1874
|
+
* no timer to pick a value for and be wrong about.
|
|
1875
|
+
* - Boot: `BaseDeviceProvider.onRestoreDevices` issues every top-level
|
|
1876
|
+
* `create()` at once and `runBounded(specs, 12)` spawns accessory children
|
|
1877
|
+
* twelve wide, so reconciles arrive in bursts — measured as 2 372 device
|
|
1878
|
+
* log lines in one 5-second bucket at boot+16s, against 62 at boot+142s.
|
|
1879
|
+
* Every arrival inside one round-trip collapses into the next call.
|
|
1880
|
+
* - Worst case: arrivals slower than a round-trip give batches of one, which
|
|
1881
|
+
* is exactly today's cost. The loader can never be worse than what it
|
|
1882
|
+
* replaces.
|
|
1883
|
+
*
|
|
1884
|
+
* A fixed debounce window would have to be guessed against that distribution,
|
|
1885
|
+
* and would tax the idle case to pay for the busy one.
|
|
1886
|
+
*
|
|
1887
|
+
* ## Degrading, rather than breaking, against an older hub
|
|
1888
|
+
*
|
|
1889
|
+
* `getChildrenBatch` is a NEW method: a hub that predates it rejects the call
|
|
1890
|
+
* (it does not silently strip the input and answer a different question, which
|
|
1891
|
+
* is what adding a `parentDeviceIds` FIELD to `getChildren` would have done).
|
|
1892
|
+
* Every waiter in a failed batch is then answered by the per-parent
|
|
1893
|
+
* `getChildren` it would have used anyway, so the reconcile is never denied
|
|
1894
|
+
* its answer. A rejection that names the method as unknown latches the
|
|
1895
|
+
* fallback permanently — re-probing a method the hub will never have, once per
|
|
1896
|
+
* batch for a whole boot, is the storm this exists to remove.
|
|
1897
|
+
*/
|
|
1898
|
+
/**
|
|
1899
|
+
* Narrow an untyped cap answer to the fields the reconcile reads.
|
|
1900
|
+
*
|
|
1901
|
+
* `callDeviceManager` is an untyped boundary, and the answer may come from a
|
|
1902
|
+
* hub of a different vintage. A row that does not carry an id, a stableId and
|
|
1903
|
+
* a name is not a child this code can prune ON, so it is dropped rather than
|
|
1904
|
+
* cast into shape — the prune step DELETES, and a half-parsed row is the input
|
|
1905
|
+
* that would make it delete the wrong thing.
|
|
1906
|
+
*/
|
|
1907
|
+
function parseChildRows(value) {
|
|
1908
|
+
if (!Array.isArray(value)) return null;
|
|
1909
|
+
const rows = [];
|
|
1910
|
+
for (const entry of value) {
|
|
1911
|
+
if (typeof entry !== "object" || entry === null) return null;
|
|
1912
|
+
const id = Reflect.get(entry, "id");
|
|
1913
|
+
const stableId = Reflect.get(entry, "stableId");
|
|
1914
|
+
const name = Reflect.get(entry, "name");
|
|
1915
|
+
const linkDeviceId = Reflect.get(entry, "linkDeviceId");
|
|
1916
|
+
if (typeof id !== "number" || typeof stableId !== "string" || typeof name !== "string") return null;
|
|
1917
|
+
rows.push({
|
|
1918
|
+
id,
|
|
1919
|
+
stableId,
|
|
1920
|
+
name,
|
|
1921
|
+
...typeof linkDeviceId === "number" || linkDeviceId === null ? { linkDeviceId } : {}
|
|
1922
|
+
});
|
|
1923
|
+
}
|
|
1924
|
+
return rows;
|
|
1925
|
+
}
|
|
1926
|
+
/**
|
|
1927
|
+
* Narrow a `getChildrenBatch` answer: a record keyed by parent id as a string.
|
|
1928
|
+
*
|
|
1929
|
+
* An EMPTY record is a valid answer — "none of these parents has children" —
|
|
1930
|
+
* and so is what a hub that answers the call but knows none of the parents
|
|
1931
|
+
* would send. Both mean "nothing to prune", which is the safe direction. Only
|
|
1932
|
+
* a shape that is not a record at all is `null`, and the loader treats that as
|
|
1933
|
+
* a failed batch and re-asks per parent.
|
|
1934
|
+
*/
|
|
1935
|
+
function parseChildrenByParent(value) {
|
|
1936
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return null;
|
|
1937
|
+
const out = /* @__PURE__ */ new Map();
|
|
1938
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
1939
|
+
const parentDeviceId = Number(key);
|
|
1940
|
+
if (!Number.isInteger(parentDeviceId)) return null;
|
|
1941
|
+
const rows = parseChildRows(entry);
|
|
1942
|
+
if (rows === null) return null;
|
|
1943
|
+
out.set(parentDeviceId, rows);
|
|
1944
|
+
}
|
|
1945
|
+
return out;
|
|
1946
|
+
}
|
|
1947
|
+
/**
|
|
1948
|
+
* True when an error says the hub has no such procedure — the version-skew
|
|
1949
|
+
* case, and the only one worth latching on. Anything else (a timeout, a busy
|
|
1950
|
+
* broker) is transient and must not disable batching for the process lifetime.
|
|
1951
|
+
*/
|
|
1952
|
+
function looksLikeUnknownMethod(err) {
|
|
1953
|
+
const lowered = (err instanceof Error ? err.message : String(err)).toLowerCase();
|
|
1954
|
+
return lowered.includes("not_found") || lowered.includes("no procedure") || lowered.includes("unknown method") || lowered.includes("no such method") || lowered.includes("not available");
|
|
1955
|
+
}
|
|
1956
|
+
function createChildrenBatchLoader(opts) {
|
|
1957
|
+
const maxBatch = Math.max(1, opts.maxBatch ?? 256);
|
|
1958
|
+
let pending = [];
|
|
1959
|
+
let flushScheduled = false;
|
|
1960
|
+
let flushing = false;
|
|
1961
|
+
let batchUnsupported = false;
|
|
1962
|
+
const answerOneByOne = async (waiters) => {
|
|
1963
|
+
await Promise.all(waiters.map(async (waiter) => {
|
|
1964
|
+
try {
|
|
1965
|
+
waiter.resolve(await opts.fetchOne(waiter.parentDeviceId));
|
|
1966
|
+
} catch (err) {
|
|
1967
|
+
waiter.reject(err);
|
|
1968
|
+
}
|
|
1969
|
+
}));
|
|
1970
|
+
};
|
|
1971
|
+
const runBatch = async (waiters) => {
|
|
1972
|
+
const parentDeviceIds = [...new Set(waiters.map((w) => w.parentDeviceId))];
|
|
1973
|
+
try {
|
|
1974
|
+
const byParent = await opts.fetchBatch(parentDeviceIds);
|
|
1975
|
+
for (const waiter of waiters) waiter.resolve(byParent.get(waiter.parentDeviceId) ?? []);
|
|
1976
|
+
} catch (err) {
|
|
1977
|
+
const unknown = looksLikeUnknownMethod(err);
|
|
1978
|
+
if (unknown) batchUnsupported = true;
|
|
1979
|
+
opts.logger.warn(unknown ? "device-manager.getChildrenBatch is not available on this hub — falling back to per-parent getChildren for the process lifetime" : "device-manager.getChildrenBatch failed — answering this batch per parent", { meta: {
|
|
1980
|
+
parents: parentDeviceIds.length,
|
|
1981
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1982
|
+
} });
|
|
1983
|
+
await answerOneByOne(waiters);
|
|
1984
|
+
}
|
|
1985
|
+
};
|
|
1986
|
+
const flush = async () => {
|
|
1987
|
+
flushing = true;
|
|
1988
|
+
try {
|
|
1989
|
+
while (pending.length > 0) {
|
|
1990
|
+
const batch = pending.slice(0, maxBatch);
|
|
1991
|
+
pending = pending.slice(maxBatch);
|
|
1992
|
+
if (batchUnsupported) await answerOneByOne(batch);
|
|
1993
|
+
else await runBatch(batch);
|
|
1994
|
+
}
|
|
1995
|
+
} finally {
|
|
1996
|
+
flushing = false;
|
|
1997
|
+
}
|
|
1998
|
+
};
|
|
1999
|
+
const scheduleFlush = () => {
|
|
2000
|
+
if (flushScheduled || flushing) return;
|
|
2001
|
+
flushScheduled = true;
|
|
2002
|
+
setTimeout(() => {
|
|
2003
|
+
flushScheduled = false;
|
|
2004
|
+
flush().catch((err) => {
|
|
2005
|
+
opts.logger.warn("children batch flush threw", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
2006
|
+
});
|
|
2007
|
+
}, 0);
|
|
2008
|
+
};
|
|
2009
|
+
return { load(parentDeviceId) {
|
|
2010
|
+
return new Promise((resolve, reject) => {
|
|
2011
|
+
pending.push({
|
|
2012
|
+
parentDeviceId,
|
|
2013
|
+
resolve,
|
|
2014
|
+
reject
|
|
2015
|
+
});
|
|
2016
|
+
scheduleFlush();
|
|
2017
|
+
});
|
|
2018
|
+
} };
|
|
2019
|
+
}
|
|
2020
|
+
//#endregion
|
|
1652
2021
|
//#region src/kernel/moleculer/device-cap-proxy.ts
|
|
1653
2022
|
/**
|
|
1654
2023
|
* Max number of accessory children spawned concurrently during a single
|
|
@@ -1939,6 +2308,19 @@ function createBrokerDeviceManagerApi(opts) {
|
|
|
1939
2308
|
api
|
|
1940
2309
|
});
|
|
1941
2310
|
let selfApi;
|
|
2311
|
+
const childrenLoader = createChildrenBatchLoader({
|
|
2312
|
+
logger: opts.logger,
|
|
2313
|
+
fetchBatch: async (parentDeviceIds) => {
|
|
2314
|
+
const parsed = parseChildrenByParent(await callDeviceManager(api, "getChildrenBatch", { parentDeviceIds }));
|
|
2315
|
+
if (parsed === null) throw new Error("getChildrenBatch returned a shape this kernel cannot read");
|
|
2316
|
+
return parsed;
|
|
2317
|
+
},
|
|
2318
|
+
fetchOne: async (parentDeviceId) => {
|
|
2319
|
+
const parsed = parseChildRows(await callDeviceManager(api, "getChildren", { parentDeviceId }));
|
|
2320
|
+
if (parsed === null) throw new Error("getChildren returned a shape this kernel cannot read");
|
|
2321
|
+
return parsed;
|
|
2322
|
+
}
|
|
2323
|
+
});
|
|
1942
2324
|
const deviceRebuildFactories = /* @__PURE__ */ new Map();
|
|
1943
2325
|
const buildContext = (stableId, id, parentDeviceId = null, initialRuntimeState = {}, persistedConfig = {}, deviceMeta = null) => {
|
|
1944
2326
|
let runtimeStateRef = null;
|
|
@@ -2356,7 +2738,7 @@ function createBrokerDeviceManagerApi(opts) {
|
|
|
2356
2738
|
return;
|
|
2357
2739
|
}
|
|
2358
2740
|
try {
|
|
2359
|
-
const currentChildren = await
|
|
2741
|
+
const currentChildren = await childrenLoader.load(device.id);
|
|
2360
2742
|
for (const child of currentChildren) {
|
|
2361
2743
|
if (child.linkDeviceId !== device.id) continue;
|
|
2362
2744
|
if (expected.has(child.stableId)) continue;
|
|
@@ -8200,4 +8582,4 @@ async function installManifestPythonDeps(declaration, addonDir, deps, logger) {
|
|
|
8200
8582
|
await deps.installPythonRequirements(reqAbs);
|
|
8201
8583
|
}
|
|
8202
8584
|
//#endregion
|
|
8203
|
-
export { buildUdsNativeCapProxy as $, createParentUnownedCallHandler as A,
|
|
8585
|
+
export { buildUdsNativeCapProxy as $, createParentUnownedCallHandler as A, RSS_BUDGET_REANNOUNCE_MIN_MS as At, AGENT_CAP_FWD_SERVICE as B, shouldReclaim as Bt, brokerTransportLink as C, createAddonDataPlaneFacility as Ct, HUB_CAP_FWD_ACTION as D, HEAP_WATCH_WARN_RATIO as Dt, localProviderLink as E, HEAP_WATCH_INTERVAL_MS as Et, createUdsLoggerWithControl as F, createV8Reclaimer as Ft, createLocalTransport as G, CapRouteError as H, startRunnerHeapWatch as Ht, LocalChildClient as I, describeRss as It, SocketChannel as J, UdsLocalTransportClient as K, LocalChildRegistry as L, hubMainRssBudget as Lt, createUdsEventBus as M, RUNNER_HEAP_WATCH_INTERVAL_MS as Mt, udsChildLogToWorkerEntry as N, RUNNER_RSS_BUDGET_ENV as Nt, HUB_CAP_FWD_SERVICE as O, HUB_MAIN_RSS_BUDGET_MB as Ot, createUdsLogger as P, buildHeapSample as Pt, buildNativeCapProxy as Q, UDS_NO_ROUTE_PREFIX as R, nextRssBudgetState as Rt, brokerCallForCap as S, runNpm as St, ipcParentLink as T, HEAP_RECLAIM_TRIGGER_MB as Tt, classifyCapRoute as U, strandedMb as Ut, CapRouteResolver as V, startHeapWatch as Vt, callWithServiceDiscovery as W, FrameDecoder as X, localEndpointPath as Y, encodeFrame as Z, createKernelHwAccel as _, CapabilityUnavailableError as _t, getWorkerDeviceRegistry as a, createAddonService as at, __resetCapUsageRegistryForTests as b, resolveAddonClass as bt, setHubConnected as c, capActionName as ct, getBrokerEventBus as d, capServiceName as dt, createBrokerDeviceManagerApi as et, getMoleculerEventStats as f, parseCapAction as ft, AddonDepsManager as g, CapabilityHandle as gt, subscribePassthrough as h, DeviceRegistry as ht, createUdsAddonContext as i, setWorkerNativeCapsChangeListener as it, createUdsEventBridge as j, RSS_BUDGET_RELEASE_RATIO as jt, createHubCapForwardService as k, HUB_RSS_BUDGET_ENV as kt, EVENT_TOPIC_PREFIX as l, capActionSuffix as lt, setNodeEventInterest as m, serializeTypedArrays as mt, adaptBrokerToCluster as n, getWorkerNativeCapSnapshot as nt, getOrInitReadinessRegistry as o, validateProviderRegistrations as ot, registerEventBusService as p, deserializeTypedArrays as pt, UdsLocalTransportServer as q, createAddonContext as r, mountNativeCapService as rt, getOrInitReadinessRegistryForClient as s, NATIVE_PROVIDER_SERVICE_INFIX as st, installManifestPythonDeps as t, getWorkerNativeCapProvider as tt, clusterEventTopic as u, capBareAction as ut, resolveHwAccel as v, copyBundledNativeModules as vt, buildLinkChain as w, HEAP_RECLAIM_MIN_INTERVAL_MS as wt, getCapUsageRegistry as x, resolveNpmInvocation as xt, CapUsageRegistry as y, installManifestNativeDeps as yt, AGENT_CAP_FWD_ACTION as z, parseRssBudgetMb as zt };
|