@camstack/system 1.2.425 → 1.2.426
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/autotrack/index.js +1 -1
- package/dist/builtins/autotrack/index.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.d.ts +6 -0
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +22 -2
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +22 -2
- package/dist/builtins/calendars/calendars.addon.js +1 -1
- package/dist/builtins/calendars/calendars.addon.mjs +1 -1
- package/dist/builtins/camera-grid/index.js +1 -1
- package/dist/builtins/camera-grid/index.mjs +1 -1
- package/dist/builtins/composer/composer.addon.js +2 -2
- package/dist/builtins/composer/composer.addon.mjs +2 -2
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +2 -2
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +2 -2
- package/dist/builtins/device-manager/device-manager.addon.js +2 -2
- package/dist/builtins/device-manager/device-manager.addon.mjs +2 -2
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/keychain/index.js +3 -3
- package/dist/builtins/keychain/index.mjs +3 -3
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +2 -2
- package/dist/builtins/local-auth/local-auth.addon.mjs +2 -2
- package/dist/builtins/local-network/address-book.d.ts +21 -0
- package/dist/builtins/local-network/local-network.addon.d.ts +6 -0
- package/dist/builtins/local-network/local-network.addon.js +48 -7
- package/dist/builtins/local-network/local-network.addon.mjs +48 -7
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.js +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1 -1
- package/dist/builtins/platform-probe/index.js +1 -1
- package/dist/builtins/platform-probe/index.mjs +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +1 -1
- package/dist/builtins/snapshot/index.mjs +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +1 -1
- package/dist/builtins/sqlite-storage/index.js +1 -1
- package/dist/builtins/sqlite-storage/index.mjs +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +3 -3
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +3 -3
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +1 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +1 -1
- package/dist/builtins/system-config/system-config.addon.js +2 -2
- package/dist/builtins/system-config/system-config.addon.mjs +2 -2
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{composition-sources-197keifF.js → composition-sources-60tuHQ9N.js} +1 -1
- package/dist/{composition-sources-DmzTSy26.mjs → composition-sources-gxVpghXh.mjs} +1 -1
- package/dist/{constants-CkNLi-XW.mjs → constants-B6YTf0v2.mjs} +1 -1
- package/dist/{constants-B5Jd8Zch.js → constants-CO5zpJQY.js} +1 -1
- package/dist/{dist-DMWeqeTF.mjs → dist-CGz9u2HD.mjs} +37 -1
- package/dist/{dist-DnALrQF-.js → dist-K3WFop9Q.js} +37 -1
- package/dist/file-host-worker.js +1 -1
- package/dist/file-host-worker.mjs +1 -1
- package/dist/index.js +272 -198
- package/dist/index.mjs +272 -198
- package/dist/{jwt-signing-key-DeqG7LD9.js → jwt-signing-key-BdDy9ZJN.js} +1 -1
- package/dist/{jwt-signing-key-CCHXAtrs.mjs → jwt-signing-key-C_w5ielw.mjs} +1 -1
- package/dist/kernel/moleculer/process-service.d.ts +1 -110
- package/dist/kernel/moleculer/runner-heap-flags.d.ts +228 -0
- package/dist/{restore-archive-C4UhQOsg.js → restore-archive-BJp4Llea.js} +1 -1
- package/dist/{restore-archive-GevpOHw8.mjs → restore-archive-CZyUM8Qf.mjs} +1 -1
- package/dist/{retired-settings-keys-0FOsnzIE.js → retired-settings-keys-Dr2AyhsD.js} +1 -1
- package/dist/{retired-settings-keys-CdVFl7m2.mjs → retired-settings-keys-Dut6QWh6.mjs} +1 -1
- package/dist/{wal-maintenance-BtfyfuT5.mjs → wal-maintenance-BZhzoxqA.mjs} +0 -0
- package/dist/{wal-maintenance-DJV1rde2.js → wal-maintenance-Bzm3O2Nj.js} +0 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_chunk = require("./chunk-Cek0wNdY.js");
|
|
3
|
-
const require_dist = require("./dist-
|
|
4
|
-
const require_jwt_signing_key = require("./jwt-signing-key-
|
|
3
|
+
const require_dist = require("./dist-K3WFop9Q.js");
|
|
4
|
+
const require_jwt_signing_key = require("./jwt-signing-key-BdDy9ZJN.js");
|
|
5
5
|
const require_builtins_alerts_alerts_addon = require("./builtins/alerts/alerts.addon.js");
|
|
6
6
|
require("./builtins/alerts/index.js");
|
|
7
7
|
const require_builtins_calendars_calendars_addon = require("./builtins/calendars/calendars.addon.js");
|
|
@@ -22,12 +22,12 @@ require("./builtins/local-auth/index.js");
|
|
|
22
22
|
const require_builtins_loki_logging_index = require("./builtins/loki-logging/index.js");
|
|
23
23
|
const require_builtins_native_metrics_native_metrics_addon = require("./builtins/native-metrics/native-metrics.addon.js");
|
|
24
24
|
const require_builtins_sqlite_storage_filesystem_storage_addon = require("./builtins/sqlite-storage/filesystem-storage.addon.js");
|
|
25
|
-
const require_wal_maintenance = require("./wal-maintenance-
|
|
25
|
+
const require_wal_maintenance = require("./wal-maintenance-Bzm3O2Nj.js");
|
|
26
26
|
const require_builtins_sqlite_storage_sqlite_settings_addon = require("./builtins/sqlite-storage/sqlite-settings.addon.js");
|
|
27
27
|
const require_builtins_sqlite_storage_index = require("./builtins/sqlite-storage/index.js");
|
|
28
28
|
const require_builtins_storage_orchestrator_storage_orchestrator_addon = require("./builtins/storage-orchestrator/storage-orchestrator.addon.js");
|
|
29
29
|
require("./builtins/storage-orchestrator/index.js");
|
|
30
|
-
const require_constants = require("./constants-
|
|
30
|
+
const require_constants = require("./constants-CO5zpJQY.js");
|
|
31
31
|
const require_builtins_system_config_system_config_addon = require("./builtins/system-config/system-config.addon.js");
|
|
32
32
|
require("./builtins/system-config/index.js");
|
|
33
33
|
const require_builtins_winston_logging_index = require("./builtins/winston-logging/index.js");
|
|
@@ -35,7 +35,7 @@ const require_custom_action_registry = require("./custom-action-registry-Bf_Ynag
|
|
|
35
35
|
const require_tls = require("./tls-BgImdxCO.js");
|
|
36
36
|
const require_platform_key_source = require("./platform-key-source-Di7HrlGR.js");
|
|
37
37
|
const require_resource_monitor = require("./resource-monitor-mio9dSCT.js");
|
|
38
|
-
const require_restore_archive = require("./restore-archive-
|
|
38
|
+
const require_restore_archive = require("./restore-archive-BJp4Llea.js");
|
|
39
39
|
let zod = require("zod");
|
|
40
40
|
let node_crypto = require("node:crypto");
|
|
41
41
|
node_crypto = require_chunk.__toESM(node_crypto);
|
|
@@ -25368,6 +25368,273 @@ var CrashSupervisor = class {
|
|
|
25368
25368
|
this.crashes.delete(runnerId);
|
|
25369
25369
|
}
|
|
25370
25370
|
};
|
|
25371
|
+
//#endregion
|
|
25372
|
+
//#region src/kernel/moleculer/runner-heap-flags.ts
|
|
25373
|
+
var EMPTY_HEAP_DECLARATION = {
|
|
25374
|
+
profile: void 0,
|
|
25375
|
+
maxOldSpaceMb: void 0,
|
|
25376
|
+
rssBudgetMb: void 0,
|
|
25377
|
+
semiSpaceMb: void 0,
|
|
25378
|
+
initialOldSpaceMb: void 0
|
|
25379
|
+
};
|
|
25380
|
+
/**
|
|
25381
|
+
* `execution.semiSpaceMb` is accepted in `[1, 256]` MB. V8 itself clamps the flag to its
|
|
25382
|
+
* own limits (the 64-bit default is 64 MB after Node 24's scavenger change, a 256 MB
|
|
25383
|
+
* semi-space is three times anything measured useful here); below 1 MB the flag is
|
|
25384
|
+
* meaningless. Outside the range the declaration is IGNORED (the runner keeps its class
|
|
25385
|
+
* default) rather than clamped: a clamped number silently becomes a different setting from
|
|
25386
|
+
* the one in the file, and the spawn log would then say something the manifest does not.
|
|
25387
|
+
*/
|
|
25388
|
+
var SEMI_SPACE_MB_RANGE = {
|
|
25389
|
+
min: 1,
|
|
25390
|
+
max: 256
|
|
25391
|
+
};
|
|
25392
|
+
/**
|
|
25393
|
+
* `execution.initialOldSpaceMb` is accepted in `[16, 65536]` MB. Below 16 MB V8's default
|
|
25394
|
+
* already starts larger; 64 GB clears any runner in this fleet. The accepted value is then
|
|
25395
|
+
* clamped at spawn to the runner's old-space CEILING (`runnerHeapFlags`) — V8 refuses an
|
|
25396
|
+
* initial size above `--max-old-space-size`, and the ceiling is the one number that belongs
|
|
25397
|
+
* to the runner as a whole.
|
|
25398
|
+
*/
|
|
25399
|
+
var INITIAL_OLD_SPACE_MB_RANGE = {
|
|
25400
|
+
min: 16,
|
|
25401
|
+
max: 65536
|
|
25402
|
+
};
|
|
25403
|
+
/** An integer MB inside `range`, else `undefined` — a non-integer is floored first. */
|
|
25404
|
+
function mbInRange(value, range) {
|
|
25405
|
+
if (typeof value !== "number" || !Number.isFinite(value)) return void 0;
|
|
25406
|
+
const mb = Math.floor(value);
|
|
25407
|
+
return mb >= range.min && mb <= range.max ? mb : void 0;
|
|
25408
|
+
}
|
|
25409
|
+
/**
|
|
25410
|
+
* Resolve the declared heap profile + old-space ceiling for one addon by reading its package.json
|
|
25411
|
+
* manifest (`camstack.addons[].execution.{heapProfile,maxOldSpaceMb}`). `addonDir` points at the
|
|
25412
|
+
* addon's built `dist`, so the manifest is one level up; we also try `addonDir` itself for layouts
|
|
25413
|
+
* where it IS the package root.
|
|
25414
|
+
* Result is cached per (dir,id) — spawns are infrequent but a runner may re-spawn on crash/restart.
|
|
25415
|
+
* Returns an empty declaration when the manifest is unreadable → treated as light, default ceiling.
|
|
25416
|
+
*/
|
|
25417
|
+
var heapProfileCache = /* @__PURE__ */ new Map();
|
|
25418
|
+
function readAddonHeapDeclaration(spec) {
|
|
25419
|
+
const cacheKey = `${spec.addonDir}::${spec.addonId}`;
|
|
25420
|
+
const cached = heapProfileCache.get(cacheKey);
|
|
25421
|
+
if (cached !== void 0) return cached;
|
|
25422
|
+
let declaration = EMPTY_HEAP_DECLARATION;
|
|
25423
|
+
for (const manifestPath of [node_path.join(spec.addonDir, "package.json"), node_path.join(node_path.dirname(spec.addonDir), "package.json")]) try {
|
|
25424
|
+
const raw = node_fs.readFileSync(manifestPath, "utf8");
|
|
25425
|
+
const parsed = JSON.parse(raw);
|
|
25426
|
+
const profile = extractHeapProfile(parsed, spec.addonId);
|
|
25427
|
+
const maxOldSpaceMb = extractMaxOldSpaceMb(parsed, spec.addonId);
|
|
25428
|
+
const rssBudgetMb = extractRssBudgetMb(parsed, spec.addonId);
|
|
25429
|
+
const semiSpaceMb = extractSemiSpaceMb(parsed, spec.addonId);
|
|
25430
|
+
const initialOldSpaceMb = extractInitialOldSpaceMb(parsed, spec.addonId);
|
|
25431
|
+
if (profile !== void 0 || maxOldSpaceMb !== void 0 || rssBudgetMb !== void 0 || semiSpaceMb !== void 0 || initialOldSpaceMb !== void 0) {
|
|
25432
|
+
declaration = {
|
|
25433
|
+
profile,
|
|
25434
|
+
maxOldSpaceMb,
|
|
25435
|
+
rssBudgetMb,
|
|
25436
|
+
semiSpaceMb,
|
|
25437
|
+
initialOldSpaceMb
|
|
25438
|
+
};
|
|
25439
|
+
break;
|
|
25440
|
+
}
|
|
25441
|
+
if (manifestHasAddon(parsed, spec.addonId)) break;
|
|
25442
|
+
} catch {}
|
|
25443
|
+
heapProfileCache.set(cacheKey, declaration);
|
|
25444
|
+
return declaration;
|
|
25445
|
+
}
|
|
25446
|
+
/**
|
|
25447
|
+
* Drop the cached heap declaration for each addon so the NEXT `readAddonHeapDeclaration`
|
|
25448
|
+
* re-reads the manifest from disk. Called on operator-driven spawns / restarts
|
|
25449
|
+
* (a deploy may have changed `execution.heapProfile` / `execution.maxOldSpaceMb`); the
|
|
25450
|
+
* crash-respawn fast path deliberately does NOT invalidate — a crashing runner
|
|
25451
|
+
* has the same on-disk manifest, so re-reading it is pure overhead.
|
|
25452
|
+
*/
|
|
25453
|
+
function invalidateHeapProfiles(addons) {
|
|
25454
|
+
for (const spec of addons) heapProfileCache.delete(`${spec.addonDir}::${spec.addonId}`);
|
|
25455
|
+
}
|
|
25456
|
+
function manifestHasAddon(parsed, addonId) {
|
|
25457
|
+
return readManifestAddons(parsed).some((a) => a.id === addonId);
|
|
25458
|
+
}
|
|
25459
|
+
function extractHeapProfile(parsed, addonId) {
|
|
25460
|
+
const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.heapProfile;
|
|
25461
|
+
return value === "heavy" || value === "light" ? value : void 0;
|
|
25462
|
+
}
|
|
25463
|
+
function extractMaxOldSpaceMb(parsed, addonId) {
|
|
25464
|
+
const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.maxOldSpaceMb;
|
|
25465
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : void 0;
|
|
25466
|
+
}
|
|
25467
|
+
/**
|
|
25468
|
+
* `execution.rssBudgetMb` for one addon. Only a POSITIVE finite number is a budget: unlike
|
|
25469
|
+
* `maxOldSpaceMb`, `0` here is not an escape hatch that lifts a cap, it is an absent budget —
|
|
25470
|
+
* and an absent budget is reported as "unwatched", never rounded into a default.
|
|
25471
|
+
*/
|
|
25472
|
+
function extractRssBudgetMb(parsed, addonId) {
|
|
25473
|
+
const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.rssBudgetMb;
|
|
25474
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : void 0;
|
|
25475
|
+
}
|
|
25476
|
+
/** `execution.semiSpaceMb` for one addon — only a value inside {@link SEMI_SPACE_MB_RANGE}. */
|
|
25477
|
+
function extractSemiSpaceMb(parsed, addonId) {
|
|
25478
|
+
return mbInRange(readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.semiSpaceMb, SEMI_SPACE_MB_RANGE);
|
|
25479
|
+
}
|
|
25480
|
+
/** `execution.initialOldSpaceMb` for one addon — only a value inside {@link INITIAL_OLD_SPACE_MB_RANGE}. */
|
|
25481
|
+
function extractInitialOldSpaceMb(parsed, addonId) {
|
|
25482
|
+
return mbInRange(readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.initialOldSpaceMb, INITIAL_OLD_SPACE_MB_RANGE);
|
|
25483
|
+
}
|
|
25484
|
+
function readManifestAddons(parsed) {
|
|
25485
|
+
if (typeof parsed !== "object" || parsed === null) return [];
|
|
25486
|
+
const camstack = parsed.camstack;
|
|
25487
|
+
if (typeof camstack !== "object" || camstack === null) return [];
|
|
25488
|
+
const addons = camstack.addons;
|
|
25489
|
+
return Array.isArray(addons) ? addons : [];
|
|
25490
|
+
}
|
|
25491
|
+
/**
|
|
25492
|
+
* A runner is heavy if ANY co-located addon declares `heapProfile: 'heavy'` in its manifest.
|
|
25493
|
+
* Everything else (the un-annotated default) is a light control-plane runner. No hard-coded names.
|
|
25494
|
+
*/
|
|
25495
|
+
function isHeavyRunner(addons) {
|
|
25496
|
+
return addons.some((a) => readAddonHeapDeclaration(a).profile === "heavy");
|
|
25497
|
+
}
|
|
25498
|
+
/**
|
|
25499
|
+
* Default old-space ceiling for a HEAVY runner, in MB.
|
|
25500
|
+
*
|
|
25501
|
+
* ## Why a heavy runner needs a ceiling at all
|
|
25502
|
+
*
|
|
25503
|
+
* `heapProfile: 'heavy'` used to mean "V8 defaults", i.e. no `--max-old-space-size` — which on this
|
|
25504
|
+
* host is V8's own memory-derived default (multiple GB). Nothing then bounds a burst, and V8 does
|
|
25505
|
+
* not hand committed pages back (the long argument for that is in `kernel/heap-watch.ts`), so a
|
|
25506
|
+
* one-off burst becomes the runner's permanent RSS.
|
|
25507
|
+
*
|
|
25508
|
+
* Measured on the live hub, 2026-08-17: the `recorder` runner reaches **2122 MB RSS** and stays
|
|
25509
|
+
* there. The same addon on two agent nodes carrying the same code and a smaller archive sits at
|
|
25510
|
+
* **58 MB**.
|
|
25511
|
+
*
|
|
25512
|
+
* ⚠ This paragraph used to attribute that ~2 GB to "walk garbage" from the deferred archive walk,
|
|
25513
|
+
* and to argue from a ~21 MB `RecordingIndex`. Both were wrong, and the error cost a session on
|
|
25514
|
+
* 2026-08-27. The deferred walk had NO CALLER and never ran (it has since been deleted). And the
|
|
25515
|
+
* index is not 21 MB: a 2026-08-27 profile of the live recorder read `heapUsed = 1606 MB` against
|
|
25516
|
+
* `external = 8 MB`, with an `smaps` census putting 1510 MB in V8's 256 KB page ranges against
|
|
25517
|
+
* 54 MB of native heap — LIVE JavaScript retainers, not high-water garbage. The retained set is
|
|
25518
|
+
* ~7.1 M `SegmentRow`s plus their path strings and Map entries (13 cameras x 2 profiles x 8 640
|
|
25519
|
+
* segments/day x ~30 days), reseeded at boot by `SegmentHourLedger.hydrateIndex`.
|
|
25520
|
+
*
|
|
25521
|
+
* The lesson for anyone sizing this flag: a heap number needs a RETAINER, and "garbage the GC has
|
|
25522
|
+
* no reason to collect" is the explanation to reach for last, not first.
|
|
25523
|
+
*
|
|
25524
|
+
* ## Why 1024
|
|
25525
|
+
*
|
|
25526
|
+
* From the same live topology snapshot, every heavy runner's FULL RSS — which is a strict
|
|
25527
|
+
* over-estimate of its old space, since it also contains native allocations, `external` and
|
|
25528
|
+
* `arrayBuffers`, none of which `--max-old-space-size` bounds:
|
|
25529
|
+
*
|
|
25530
|
+
* detection 471 MB · provider-reolink 507 MB · pipeline-orchestrator 238 MB · ml 218 MB ·
|
|
25531
|
+
* snapshot 185 MB · decoder-nodeav 174 MB · audio-analyzer 170 MB
|
|
25532
|
+
*
|
|
25533
|
+
* 1024 MB clears the largest of those by ~2x while cutting the recorder's observed ratchet by
|
|
25534
|
+
* more than half. It is a CEILING, not a target: a runner that fits today keeps behaving exactly
|
|
25535
|
+
* as it does now, because V8 only collects harder as it approaches the limit.
|
|
25536
|
+
*
|
|
25537
|
+
* A runner whose working set genuinely does not fit declares its own number in its manifest
|
|
25538
|
+
* (`execution.maxOldSpaceMb`) — the number belongs next to the addon that knows its working set.
|
|
25539
|
+
* `recorder` (3072 MB; peak 2341 MB heapTotal) is the one shipped addon that does. `stream-broker` does NOT, despite 1585 MB peak RSS: ~1 GB of that is
|
|
25540
|
+
* `arrayBuffers` (47 RTP rings), which this flag does not bound and never will — its actual old
|
|
25541
|
+
* space peaks at 224 MB. Size the declaration from `heapTotal` in the `[mem]` heartbeat, never
|
|
25542
|
+
* from RSS.
|
|
25543
|
+
*
|
|
25544
|
+
* ## What this flag CANNOT do
|
|
25545
|
+
*
|
|
25546
|
+
* It bounds old space and nothing else — not `external`, not `arrayBuffers`, not what a native
|
|
25547
|
+
* addon mallocs. `hub/pipeline-analytics` reached 7334 MB RSS with this flag set to 1024 the
|
|
25548
|
+
* whole time, because 96% of it was native. That half is bounded in `runner-native-allocator.ts`.
|
|
25549
|
+
* Reaching for a bigger number here when RSS is the complaint is the standard wrong move.
|
|
25550
|
+
*/
|
|
25551
|
+
var HEAVY_MAX_OLD_MB_DEFAULT = 1024;
|
|
25552
|
+
/**
|
|
25553
|
+
* The runner's old-space ceiling in MB, or 0 for "uncapped".
|
|
25554
|
+
*
|
|
25555
|
+
* The runner takes the MAXIMUM ceiling any co-located addon declares — co-location must never
|
|
25556
|
+
* silence the hungriest member (D2/D29: a runner is respawned as a whole, so an OOM caused by
|
|
25557
|
+
* under-sizing one member kills every addon in the group). An addon declaring `0` therefore lifts
|
|
25558
|
+
* the cap for the whole runner, which is the intended escape hatch.
|
|
25559
|
+
*/
|
|
25560
|
+
function runnerMaxOldSpaceMb(addons, heavy) {
|
|
25561
|
+
const envDefault = heavy ? HEAVY_MAX_OLD_MB_DEFAULT : 0;
|
|
25562
|
+
const declared = addons.map((a) => readAddonHeapDeclaration(a).maxOldSpaceMb).filter((mb) => mb !== void 0);
|
|
25563
|
+
if (declared.length === 0) return envDefault;
|
|
25564
|
+
if (declared.includes(0)) return 0;
|
|
25565
|
+
return Math.max(...declared);
|
|
25566
|
+
}
|
|
25567
|
+
/**
|
|
25568
|
+
* The runner's declared RSS budget in MB, or `undefined` when NOBODY declared one.
|
|
25569
|
+
*
|
|
25570
|
+
* Same rule as the old-space ceiling, for the same reason (D2/D29): the runner takes the MAXIMUM
|
|
25571
|
+
* any co-located addon declares, because a runner is respawned as a whole and co-location must
|
|
25572
|
+
* never silence the hungriest member. It would be strictly worse here than for a heap ceiling —
|
|
25573
|
+
* an under-sized budget does not kill anything, it just cries wolf, and a watchdog that cries
|
|
25574
|
+
* wolf is a watchdog that gets muted.
|
|
25575
|
+
*
|
|
25576
|
+
* `undefined` propagates as an ABSENT env var, and the child then announces itself unwatched.
|
|
25577
|
+
* There is deliberately no class default: see `AddonExecution.rssBudgetMb`.
|
|
25578
|
+
*/
|
|
25579
|
+
function runnerRssBudgetMb(addons) {
|
|
25580
|
+
const declared = addons.map((a) => readAddonHeapDeclaration(a).rssBudgetMb).filter((mb) => mb !== void 0);
|
|
25581
|
+
return declared.length === 0 ? void 0 : Math.max(...declared);
|
|
25582
|
+
}
|
|
25583
|
+
/**
|
|
25584
|
+
* Per-placement V8 flags (Block D). LIGHT runners cap their young generation
|
|
25585
|
+
* (`--max-semi-space-size`) to reclaim V8's over-reservation — a measured ~15 MB/runner win — bound
|
|
25586
|
+
* V8's platform thread pool (`--v8-pool-size`, see {@link LIGHT_V8_POOL_SIZE_DEFAULT}: a measured
|
|
25587
|
+
* ~58 threads across the hub's 29 light runners, against a cgroup `pids.max` this host has already
|
|
25588
|
+
* hit) and carry no old-space cap unless a manifest declares one. HEAVY
|
|
25589
|
+
* runners keep V8's default young generation but ALWAYS carry an old-space ceiling; see
|
|
25590
|
+
* {@link HEAVY_MAX_OLD_MB_DEFAULT} for why, and how a runner declares its own.
|
|
25591
|
+
*
|
|
25592
|
+
* The per-addon overrides are the manifest's declared `maxOldSpaceMb` (`0` = uncapped),
|
|
25593
|
+
* `semiSpaceMb` (`--max-semi-space-size`, replacing the class default) and
|
|
25594
|
+
* `initialOldSpaceMb` (`--initial-old-space-size`, clamped to the ceiling) — each the MAXIMUM
|
|
25595
|
+
* across a runner's co-located addons. Shipped for `stream-broker` alone (8 / 512): with the
|
|
25596
|
+
* heavy class default the hub's broker ran ~166 major GCs a minute and not one scavenge, at
|
|
25597
|
+
* 45–60 % of a core. The environment overrides and the `CAMSTACK_RUNNER_HEAP_TUNING=off` kill
|
|
25598
|
+
* switch were removed on 2026-10-06 — nothing had set them since the rollout. The flags must
|
|
25599
|
+
* be on the spawn command line: `v8.setFlagsFromString` at runtime changes neither space.
|
|
25600
|
+
*/
|
|
25601
|
+
function runnerHeapFlags(addons) {
|
|
25602
|
+
const heavy = isHeavyRunner(addons);
|
|
25603
|
+
const flags = [];
|
|
25604
|
+
const semiMb = runnerSemiSpaceMb(addons) ?? (heavy ? void 0 : 2);
|
|
25605
|
+
if (semiMb !== void 0) flags.push(`--max-semi-space-size=${semiMb}`);
|
|
25606
|
+
if (!heavy) flags.push(`--v8-pool-size=2`);
|
|
25607
|
+
const oldMb = runnerMaxOldSpaceMb(addons, heavy);
|
|
25608
|
+
if (oldMb > 0) flags.push(`--max-old-space-size=${oldMb}`);
|
|
25609
|
+
const initialOldMb = runnerInitialOldSpaceMb(addons);
|
|
25610
|
+
if (initialOldMb !== void 0) {
|
|
25611
|
+
const bounded = oldMb > 0 ? Math.min(initialOldMb, oldMb) : initialOldMb;
|
|
25612
|
+
flags.push(`--initial-old-space-size=${bounded}`);
|
|
25613
|
+
}
|
|
25614
|
+
return flags;
|
|
25615
|
+
}
|
|
25616
|
+
/**
|
|
25617
|
+
* The runner's declared young-generation semi-space in MB, or `undefined` when no
|
|
25618
|
+
* co-located addon declares one (→ the class default applies).
|
|
25619
|
+
*
|
|
25620
|
+
* The MAXIMUM across the group, like every other declaration here (D2/D29): the
|
|
25621
|
+
* member that measured its churn away must not be handed a smaller young
|
|
25622
|
+
* generation because a quieter co-tenant declared one.
|
|
25623
|
+
*/
|
|
25624
|
+
function runnerSemiSpaceMb(addons) {
|
|
25625
|
+
const declared = addons.map((a) => readAddonHeapDeclaration(a).semiSpaceMb).filter((mb) => mb !== void 0);
|
|
25626
|
+
return declared.length === 0 ? void 0 : Math.max(...declared);
|
|
25627
|
+
}
|
|
25628
|
+
/**
|
|
25629
|
+
* The runner's declared initial old space in MB, or `undefined` when nobody declares
|
|
25630
|
+
* one (→ V8 grows it from the live set). MAXIMUM across the group, same rule and same
|
|
25631
|
+
* reason as {@link runnerSemiSpaceMb}; the clamp to the runner's ceiling happens in
|
|
25632
|
+
* {@link runnerHeapFlags}, where the ceiling is known.
|
|
25633
|
+
*/
|
|
25634
|
+
function runnerInitialOldSpaceMb(addons) {
|
|
25635
|
+
const declared = addons.map((a) => readAddonHeapDeclaration(a).initialOldSpaceMb).filter((mb) => mb !== void 0);
|
|
25636
|
+
return declared.length === 0 ? void 0 : Math.max(...declared);
|
|
25637
|
+
}
|
|
25371
25638
|
/**
|
|
25372
25639
|
* Resolve the native-allocator env for a child runner from the parent's env.
|
|
25373
25640
|
*
|
|
@@ -25697,199 +25964,6 @@ async function getPidStats$1(pids) {
|
|
|
25697
25964
|
});
|
|
25698
25965
|
return result;
|
|
25699
25966
|
}
|
|
25700
|
-
var EMPTY_HEAP_DECLARATION = {
|
|
25701
|
-
profile: void 0,
|
|
25702
|
-
maxOldSpaceMb: void 0,
|
|
25703
|
-
rssBudgetMb: void 0
|
|
25704
|
-
};
|
|
25705
|
-
/**
|
|
25706
|
-
* Resolve the declared heap profile + old-space ceiling for one addon by reading its package.json
|
|
25707
|
-
* manifest (`camstack.addons[].execution.{heapProfile,maxOldSpaceMb}`). `addonDir` points at the
|
|
25708
|
-
* addon's built `dist`, so the manifest is one level up; we also try `addonDir` itself for layouts
|
|
25709
|
-
* where it IS the package root.
|
|
25710
|
-
* Result is cached per (dir,id) — spawns are infrequent but a runner may re-spawn on crash/restart.
|
|
25711
|
-
* Returns an empty declaration when the manifest is unreadable → treated as light, default ceiling.
|
|
25712
|
-
*/
|
|
25713
|
-
var heapProfileCache = /* @__PURE__ */ new Map();
|
|
25714
|
-
function readAddonHeapDeclaration(spec) {
|
|
25715
|
-
const cacheKey = `${spec.addonDir}::${spec.addonId}`;
|
|
25716
|
-
const cached = heapProfileCache.get(cacheKey);
|
|
25717
|
-
if (cached !== void 0) return cached;
|
|
25718
|
-
let declaration = EMPTY_HEAP_DECLARATION;
|
|
25719
|
-
for (const manifestPath of [node_path.join(spec.addonDir, "package.json"), node_path.join(node_path.dirname(spec.addonDir), "package.json")]) try {
|
|
25720
|
-
const raw = node_fs.readFileSync(manifestPath, "utf8");
|
|
25721
|
-
const parsed = JSON.parse(raw);
|
|
25722
|
-
const profile = extractHeapProfile(parsed, spec.addonId);
|
|
25723
|
-
const maxOldSpaceMb = extractMaxOldSpaceMb(parsed, spec.addonId);
|
|
25724
|
-
const rssBudgetMb = extractRssBudgetMb(parsed, spec.addonId);
|
|
25725
|
-
if (profile !== void 0 || maxOldSpaceMb !== void 0 || rssBudgetMb !== void 0) {
|
|
25726
|
-
declaration = {
|
|
25727
|
-
profile,
|
|
25728
|
-
maxOldSpaceMb,
|
|
25729
|
-
rssBudgetMb
|
|
25730
|
-
};
|
|
25731
|
-
break;
|
|
25732
|
-
}
|
|
25733
|
-
if (manifestHasAddon(parsed, spec.addonId)) break;
|
|
25734
|
-
} catch {}
|
|
25735
|
-
heapProfileCache.set(cacheKey, declaration);
|
|
25736
|
-
return declaration;
|
|
25737
|
-
}
|
|
25738
|
-
/**
|
|
25739
|
-
* Drop the cached heap declaration for each addon so the NEXT `readAddonHeapDeclaration`
|
|
25740
|
-
* re-reads the manifest from disk. Called on operator-driven spawns / restarts
|
|
25741
|
-
* (a deploy may have changed `execution.heapProfile` / `execution.maxOldSpaceMb`); the
|
|
25742
|
-
* crash-respawn fast path deliberately does NOT invalidate — a crashing runner
|
|
25743
|
-
* has the same on-disk manifest, so re-reading it is pure overhead.
|
|
25744
|
-
*/
|
|
25745
|
-
function invalidateHeapProfiles(addons) {
|
|
25746
|
-
for (const spec of addons) heapProfileCache.delete(`${spec.addonDir}::${spec.addonId}`);
|
|
25747
|
-
}
|
|
25748
|
-
function manifestHasAddon(parsed, addonId) {
|
|
25749
|
-
return readManifestAddons(parsed).some((a) => a.id === addonId);
|
|
25750
|
-
}
|
|
25751
|
-
function extractHeapProfile(parsed, addonId) {
|
|
25752
|
-
const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.heapProfile;
|
|
25753
|
-
return value === "heavy" || value === "light" ? value : void 0;
|
|
25754
|
-
}
|
|
25755
|
-
function extractMaxOldSpaceMb(parsed, addonId) {
|
|
25756
|
-
const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.maxOldSpaceMb;
|
|
25757
|
-
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : void 0;
|
|
25758
|
-
}
|
|
25759
|
-
/**
|
|
25760
|
-
* `execution.rssBudgetMb` for one addon. Only a POSITIVE finite number is a budget: unlike
|
|
25761
|
-
* `maxOldSpaceMb`, `0` here is not an escape hatch that lifts a cap, it is an absent budget —
|
|
25762
|
-
* and an absent budget is reported as "unwatched", never rounded into a default.
|
|
25763
|
-
*/
|
|
25764
|
-
function extractRssBudgetMb(parsed, addonId) {
|
|
25765
|
-
const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.rssBudgetMb;
|
|
25766
|
-
return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : void 0;
|
|
25767
|
-
}
|
|
25768
|
-
function readManifestAddons(parsed) {
|
|
25769
|
-
if (typeof parsed !== "object" || parsed === null) return [];
|
|
25770
|
-
const camstack = parsed.camstack;
|
|
25771
|
-
if (typeof camstack !== "object" || camstack === null) return [];
|
|
25772
|
-
const addons = camstack.addons;
|
|
25773
|
-
return Array.isArray(addons) ? addons : [];
|
|
25774
|
-
}
|
|
25775
|
-
/**
|
|
25776
|
-
* A runner is heavy if ANY co-located addon declares `heapProfile: 'heavy'` in its manifest.
|
|
25777
|
-
* Everything else (the un-annotated default) is a light control-plane runner. No hard-coded names.
|
|
25778
|
-
*/
|
|
25779
|
-
function isHeavyRunner(addons) {
|
|
25780
|
-
return addons.some((a) => readAddonHeapDeclaration(a).profile === "heavy");
|
|
25781
|
-
}
|
|
25782
|
-
/**
|
|
25783
|
-
* Default old-space ceiling for a HEAVY runner, in MB.
|
|
25784
|
-
*
|
|
25785
|
-
* ## Why a heavy runner needs a ceiling at all
|
|
25786
|
-
*
|
|
25787
|
-
* `heapProfile: 'heavy'` used to mean "V8 defaults", i.e. no `--max-old-space-size` — which on this
|
|
25788
|
-
* host is V8's own memory-derived default (multiple GB). Nothing then bounds a burst, and V8 does
|
|
25789
|
-
* not hand committed pages back (the long argument for that is in `kernel/heap-watch.ts`), so a
|
|
25790
|
-
* one-off burst becomes the runner's permanent RSS.
|
|
25791
|
-
*
|
|
25792
|
-
* Measured on the live hub, 2026-08-17: the `recorder` runner reaches **2122 MB RSS** and stays
|
|
25793
|
-
* there. The same addon on two agent nodes carrying the same code and a smaller archive sits at
|
|
25794
|
-
* **58 MB**.
|
|
25795
|
-
*
|
|
25796
|
-
* ⚠ This paragraph used to attribute that ~2 GB to "walk garbage" from the deferred archive walk,
|
|
25797
|
-
* and to argue from a ~21 MB `RecordingIndex`. Both were wrong, and the error cost a session on
|
|
25798
|
-
* 2026-08-27. The deferred walk had NO CALLER and never ran (it has since been deleted). And the
|
|
25799
|
-
* index is not 21 MB: a 2026-08-27 profile of the live recorder read `heapUsed = 1606 MB` against
|
|
25800
|
-
* `external = 8 MB`, with an `smaps` census putting 1510 MB in V8's 256 KB page ranges against
|
|
25801
|
-
* 54 MB of native heap — LIVE JavaScript retainers, not high-water garbage. The retained set is
|
|
25802
|
-
* ~7.1 M `SegmentRow`s plus their path strings and Map entries (13 cameras x 2 profiles x 8 640
|
|
25803
|
-
* segments/day x ~30 days), reseeded at boot by `SegmentHourLedger.hydrateIndex`.
|
|
25804
|
-
*
|
|
25805
|
-
* The lesson for anyone sizing this flag: a heap number needs a RETAINER, and "garbage the GC has
|
|
25806
|
-
* no reason to collect" is the explanation to reach for last, not first.
|
|
25807
|
-
*
|
|
25808
|
-
* ## Why 1024
|
|
25809
|
-
*
|
|
25810
|
-
* From the same live topology snapshot, every heavy runner's FULL RSS — which is a strict
|
|
25811
|
-
* over-estimate of its old space, since it also contains native allocations, `external` and
|
|
25812
|
-
* `arrayBuffers`, none of which `--max-old-space-size` bounds:
|
|
25813
|
-
*
|
|
25814
|
-
* detection 471 MB · provider-reolink 507 MB · pipeline-orchestrator 238 MB · ml 218 MB ·
|
|
25815
|
-
* snapshot 185 MB · decoder-nodeav 174 MB · audio-analyzer 170 MB
|
|
25816
|
-
*
|
|
25817
|
-
* 1024 MB clears the largest of those by ~2x while cutting the recorder's observed ratchet by
|
|
25818
|
-
* more than half. It is a CEILING, not a target: a runner that fits today keeps behaving exactly
|
|
25819
|
-
* as it does now, because V8 only collects harder as it approaches the limit.
|
|
25820
|
-
*
|
|
25821
|
-
* A runner whose working set genuinely does not fit declares its own number in its manifest
|
|
25822
|
-
* (`execution.maxOldSpaceMb`) — the number belongs next to the addon that knows its working set.
|
|
25823
|
-
* `recorder` (3072 MB; peak 2341 MB heapTotal) is the one shipped addon that does. `stream-broker` does NOT, despite 1585 MB peak RSS: ~1 GB of that is
|
|
25824
|
-
* `arrayBuffers` (47 RTP rings), which this flag does not bound and never will — its actual old
|
|
25825
|
-
* space peaks at 224 MB. Size the declaration from `heapTotal` in the `[mem]` heartbeat, never
|
|
25826
|
-
* from RSS.
|
|
25827
|
-
*
|
|
25828
|
-
* ## What this flag CANNOT do
|
|
25829
|
-
*
|
|
25830
|
-
* It bounds old space and nothing else — not `external`, not `arrayBuffers`, not what a native
|
|
25831
|
-
* addon mallocs. `hub/pipeline-analytics` reached 7334 MB RSS with this flag set to 1024 the
|
|
25832
|
-
* whole time, because 96% of it was native. That half is bounded in `runner-native-allocator.ts`.
|
|
25833
|
-
* Reaching for a bigger number here when RSS is the complaint is the standard wrong move.
|
|
25834
|
-
*/
|
|
25835
|
-
var HEAVY_MAX_OLD_MB_DEFAULT = 1024;
|
|
25836
|
-
/** Light runners' young-generation semi-space, in MB. */
|
|
25837
|
-
var LIGHT_SEMI_SPACE_MB = 2;
|
|
25838
|
-
/**
|
|
25839
|
-
* The runner's old-space ceiling in MB, or 0 for "uncapped".
|
|
25840
|
-
*
|
|
25841
|
-
* The runner takes the MAXIMUM ceiling any co-located addon declares — co-location must never
|
|
25842
|
-
* silence the hungriest member (D2/D29: a runner is respawned as a whole, so an OOM caused by
|
|
25843
|
-
* under-sizing one member kills every addon in the group). An addon declaring `0` therefore lifts
|
|
25844
|
-
* the cap for the whole runner, which is the intended escape hatch.
|
|
25845
|
-
*/
|
|
25846
|
-
function runnerMaxOldSpaceMb(addons, heavy) {
|
|
25847
|
-
const envDefault = heavy ? HEAVY_MAX_OLD_MB_DEFAULT : 0;
|
|
25848
|
-
const declared = addons.map((a) => readAddonHeapDeclaration(a).maxOldSpaceMb).filter((mb) => mb !== void 0);
|
|
25849
|
-
if (declared.length === 0) return envDefault;
|
|
25850
|
-
if (declared.includes(0)) return 0;
|
|
25851
|
-
return Math.max(...declared);
|
|
25852
|
-
}
|
|
25853
|
-
/**
|
|
25854
|
-
* The runner's declared RSS budget in MB, or `undefined` when NOBODY declared one.
|
|
25855
|
-
*
|
|
25856
|
-
* Same rule as the old-space ceiling, for the same reason (D2/D29): the runner takes the MAXIMUM
|
|
25857
|
-
* any co-located addon declares, because a runner is respawned as a whole and co-location must
|
|
25858
|
-
* never silence the hungriest member. It would be strictly worse here than for a heap ceiling —
|
|
25859
|
-
* an under-sized budget does not kill anything, it just cries wolf, and a watchdog that cries
|
|
25860
|
-
* wolf is a watchdog that gets muted.
|
|
25861
|
-
*
|
|
25862
|
-
* `undefined` propagates as an ABSENT env var, and the child then announces itself unwatched.
|
|
25863
|
-
* There is deliberately no class default: see `AddonExecution.rssBudgetMb`.
|
|
25864
|
-
*/
|
|
25865
|
-
function runnerRssBudgetMb(addons) {
|
|
25866
|
-
const declared = addons.map((a) => readAddonHeapDeclaration(a).rssBudgetMb).filter((mb) => mb !== void 0);
|
|
25867
|
-
return declared.length === 0 ? void 0 : Math.max(...declared);
|
|
25868
|
-
}
|
|
25869
|
-
/**
|
|
25870
|
-
* Per-placement V8 flags (Block D). LIGHT runners cap their young generation
|
|
25871
|
-
* (`--max-semi-space-size`) to reclaim V8's over-reservation — a measured ~15 MB/runner win — bound
|
|
25872
|
-
* V8's platform thread pool (`--v8-pool-size`, see {@link LIGHT_V8_POOL_SIZE_DEFAULT}: a measured
|
|
25873
|
-
* ~58 threads across the hub's 29 light runners, against a cgroup `pids.max` this host has already
|
|
25874
|
-
* hit) and carry no old-space cap unless a manifest declares one. HEAVY
|
|
25875
|
-
* runners keep V8's default young generation but ALWAYS carry an old-space ceiling; see
|
|
25876
|
-
* {@link HEAVY_MAX_OLD_MB_DEFAULT} for why, and how a runner declares its own.
|
|
25877
|
-
*
|
|
25878
|
-
* The per-addon override is the manifest's declared `maxOldSpaceMb` (`0` = uncapped); the
|
|
25879
|
-
* environment overrides and the `CAMSTACK_RUNNER_HEAP_TUNING=off` kill switch were removed on
|
|
25880
|
-
* 2026-10-06 — nothing had set them since the rollout.
|
|
25881
|
-
*/
|
|
25882
|
-
function runnerHeapFlags(addons) {
|
|
25883
|
-
const heavy = isHeavyRunner(addons);
|
|
25884
|
-
const flags = [];
|
|
25885
|
-
if (!heavy) {
|
|
25886
|
-
flags.push(`--max-semi-space-size=${LIGHT_SEMI_SPACE_MB}`);
|
|
25887
|
-
flags.push(`--v8-pool-size=2`);
|
|
25888
|
-
}
|
|
25889
|
-
const oldMb = runnerMaxOldSpaceMb(addons, heavy);
|
|
25890
|
-
if (oldMb > 0) flags.push(`--max-old-space-size=${oldMb}`);
|
|
25891
|
-
return flags;
|
|
25892
|
-
}
|
|
25893
25967
|
var MAX_BACKOFF_MS = 3e4;
|
|
25894
25968
|
var CRASH_WINDOW_MS = 6e4;
|
|
25895
25969
|
var MAX_CRASHES_IN_WINDOW = 5;
|