@camstack/system 1.2.101 → 1.2.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +1 -1
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
- package/dist/builtins/device-manager/device-manager.addon.js +1 -1
- package/dist/builtins/device-manager/device-manager.addon.mjs +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +1 -1
- package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
- package/dist/builtins/local-network/local-network.addon.js +1 -1
- package/dist/builtins/local-network/local-network.addon.mjs +1 -1
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.js +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1 -1
- package/dist/builtins/platform-probe/index.js +1 -1
- package/dist/builtins/platform-probe/index.mjs +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +182 -25
- package/dist/builtins/snapshot/index.mjs +182 -25
- package/dist/builtins/snapshot/snapshot-cache.d.ts +63 -0
- package/dist/builtins/snapshot/snapshot-grab-pick.d.ts +20 -0
- package/dist/builtins/snapshot/snapshot-resize.d.ts +33 -1
- package/dist/builtins/snapshot/snapshot.addon.d.ts +6 -0
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +1 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +1 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +1 -1
- package/dist/builtins/system-config/system-config.addon.js +1 -1
- package/dist/builtins/system-config/system-config.addon.mjs +1 -1
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{dist-RbFV8cX7.mjs → dist-DHKXob1m.mjs} +179 -29
- package/dist/{dist-tMrPAfpF.js → dist-qnwy_iWM.js} +179 -29
- package/dist/index.js +264 -62
- package/dist/index.mjs +264 -62
- package/dist/kernel/config-schema.d.ts +14 -10
- package/dist/kernel/heap-watch.d.ts +39 -7
- package/dist/kernel/moleculer/process-service.d.ts +34 -0
- package/dist/logging/log-manager.d.ts +7 -5
- package/dist/logging/partitioned-log-buffer.d.ts +95 -16
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_chunk = require("./chunk-Cek0wNdY.js");
|
|
3
|
-
const require_dist = require("./dist-
|
|
3
|
+
const require_dist = require("./dist-qnwy_iWM.js");
|
|
4
4
|
const require_builtins_alerts_alerts_addon = require("./builtins/alerts/alerts.addon.js");
|
|
5
5
|
require("./builtins/alerts/index.js");
|
|
6
6
|
const require_formatter = require("./formatter-DqAKDlvN.js");
|
|
@@ -501,9 +501,10 @@ function proxyToUpstream(opts) {
|
|
|
501
501
|
* ask V8 for the pages back when the process is visibly holding more than it is
|
|
502
502
|
* using — which is what `shouldReclaim` decides and what the heartbeat now does.
|
|
503
503
|
*
|
|
504
|
-
* The cost is a
|
|
505
|
-
*
|
|
506
|
-
*
|
|
504
|
+
* The cost is a stop-the-world on the hub's router process — 250-450ms when
|
|
505
|
+
* first measured, 654ms on the largest pass seen since — bounded to once per
|
|
506
|
+
* `HEAP_RECLAIM_MIN_INTERVAL_MS`. The alternative, measured, is a ratchet to
|
|
507
|
+
* 7.8GB and the OOM that ends it.
|
|
507
508
|
*/
|
|
508
509
|
/** Heartbeat period. Slow enough to be free, fast enough that the sample before
|
|
509
510
|
* a fatal is recent. */
|
|
@@ -541,14 +542,46 @@ function nextMode(current, usedRatio, escalateRatio = HEAP_WATCH_ESCALATE_RATIO,
|
|
|
541
542
|
* Live calibration, both readings from the same hub minutes apart: 2491MB
|
|
542
543
|
* stranded before the reclaim, 438MB after. 438MB is this process's irreducible
|
|
543
544
|
* native floor (sqlite pages, libvips, sharp, thread stacks, node's own text) —
|
|
544
|
-
* collecting against it would buy nothing and cost a stall every minute.
|
|
545
|
-
*
|
|
546
|
-
*
|
|
545
|
+
* collecting against it would buy nothing and cost a stall every minute.
|
|
546
|
+
*
|
|
547
|
+
* ## Why this came down from 1536 (2026-08-17)
|
|
548
|
+
*
|
|
549
|
+
* The trigger was originally set to fire only on an unmistakable strand. It
|
|
550
|
+
* does, and the pass on the record works exactly as designed:
|
|
551
|
+
*
|
|
552
|
+
* [mem] reclaim hub-main stranded=2365MB rss=4016MB→2368MB freed=1648MB took=654ms
|
|
553
|
+
*
|
|
554
|
+
* But the number it was protecting against changed. The host now runs with a
|
|
555
|
+
* 24GiB cgroup ceiling and **no swap**, and is OOM-killing a neighbouring VM.
|
|
556
|
+
* RSS this process is holding and not using is no longer merely untidy — it is
|
|
557
|
+
* the headroom something else needed, and the OOM is triggered by the PEAK, not
|
|
558
|
+
* by the average. A trigger of 1536 means the peak is allowed to stand 1.5GiB
|
|
559
|
+
* above the floor before anything happens.
|
|
560
|
+
*
|
|
561
|
+
* 1024 is 2.3x the measured 438MB floor, so ordinary movement around the floor
|
|
562
|
+
* can never trip it, and every pass that does fire still returns roughly
|
|
563
|
+
* `1024 - 438` ≈ **580MB minimum** — a real return for a 250-650ms stall,
|
|
564
|
+
* not a stall spent collecting noise.
|
|
547
565
|
*/
|
|
548
|
-
var HEAP_RECLAIM_TRIGGER_MB =
|
|
549
|
-
/**
|
|
550
|
-
*
|
|
551
|
-
|
|
566
|
+
var HEAP_RECLAIM_TRIGGER_MB = 1024;
|
|
567
|
+
/**
|
|
568
|
+
* Floor on how often the stop-the-world may run.
|
|
569
|
+
*
|
|
570
|
+
* Ten minutes was chosen to keep the worst case under 0.1% of the router's
|
|
571
|
+
* time. The worst case is not what governs the cost, though: the reclaim only
|
|
572
|
+
* runs when {@link shouldReclaim} says the process is genuinely stranded, and
|
|
573
|
+
* after a pass it is at the ~438MB floor. Live off-heap growth was measured at
|
|
574
|
+
* roughly 3GB/2h ≈ 25MB/min, so re-stranding the ~580MB needed to qualify again
|
|
575
|
+
* takes on the order of **20 minutes** — well outside any floor considered
|
|
576
|
+
* here. In steady state this constant therefore costs nothing at all.
|
|
577
|
+
*
|
|
578
|
+
* What it does govern is a BURST, which is the only case that matters on a host
|
|
579
|
+
* with no swap: the ten-minute floor let a peak that re-formed in seconds stand
|
|
580
|
+
* for ten minutes. Two minutes bounds that, and even the pathological case of a
|
|
581
|
+
* process that re-strands instantly costs 654ms per 120s — 0.55% of the
|
|
582
|
+
* router's time, against a ~1.5GiB peak it removes each pass.
|
|
583
|
+
*/
|
|
584
|
+
var HEAP_RECLAIM_MIN_INTERVAL_MS = 12e4;
|
|
552
585
|
var MB = (bytes) => Math.round(bytes / 1048576);
|
|
553
586
|
/**
|
|
554
587
|
* Build a sample from raw readings. Pure so the thresholds are testable without
|
|
@@ -624,8 +657,8 @@ function format$1(label, s) {
|
|
|
624
657
|
function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions) {
|
|
625
658
|
const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
|
|
626
659
|
const now = reclaimOptions?.now ?? (() => Date.now());
|
|
627
|
-
const triggerMb = reclaimOptions?.triggerMb ??
|
|
628
|
-
const minIntervalMs = reclaimOptions?.minIntervalMs ??
|
|
660
|
+
const triggerMb = reclaimOptions?.triggerMb ?? 1024;
|
|
661
|
+
const minIntervalMs = reclaimOptions?.minIntervalMs ?? 12e4;
|
|
629
662
|
const fastIntervalMs = reclaimOptions?.fastIntervalMs ?? 2e3;
|
|
630
663
|
const escalateRatio = reclaimOptions?.escalateRatio ?? .7;
|
|
631
664
|
const deescalateRatio = reclaimOptions?.deescalateRatio ?? .6;
|
|
@@ -968,6 +1001,26 @@ function addonBucketKey(addonId) {
|
|
|
968
1001
|
*/
|
|
969
1002
|
var PRUNE_TARGET_RATIO = .9;
|
|
970
1003
|
/**
|
|
1004
|
+
* Default ceiling on retained entries across every bucket.
|
|
1005
|
+
*
|
|
1006
|
+
* This used to be `null` — no aggregate bound at all — and the only thing
|
|
1007
|
+
* holding hub-main's log memory was `perAddonCapacity x buckets`, a product
|
|
1008
|
+
* that grows with the roster. Measured on the live cluster: 37 forked runners
|
|
1009
|
+
* on the hub plus its in-process builtins, and hub-main additionally ingests
|
|
1010
|
+
* every line from both agents, so ~60 buckets is the ordinary case and
|
|
1011
|
+
* 60 x 5000 = 300k retained entries.
|
|
1012
|
+
*
|
|
1013
|
+
* A realistic entry (timestamp, level, scope, ~70-char message, three tags, no
|
|
1014
|
+
* `meta`) measures **~306 B retained** in V8. So the old unbounded case is
|
|
1015
|
+
* ~92MB of a hub process that is being asked to fit inside a cgroup with a
|
|
1016
|
+
* neighbour to protect, and 50_000 is **~15MB**.
|
|
1017
|
+
*
|
|
1018
|
+
* 50_000 across a ~60-bucket roster is ~830 entries per addon on average while
|
|
1019
|
+
* still letting a single chatty addon hold its full 5000-entry ring — the
|
|
1020
|
+
* partitioning is not weakened, only the total is held.
|
|
1021
|
+
*/
|
|
1022
|
+
var DEFAULT_MAX_TOTAL_LOG_ENTRIES = 5e4;
|
|
1023
|
+
/**
|
|
971
1024
|
* A log buffer partitioned into one fixed-capacity {@link LogRingBuffer} per
|
|
972
1025
|
* `addonId`. A chatty addon (e.g. detection-pipeline) only ever evicts its OWN
|
|
973
1026
|
* bucket, so a quiet addon's sparse lines (e.g. a HomeAssistant `image` entity)
|
|
@@ -976,38 +1029,55 @@ var PRUNE_TARGET_RATIO = .9;
|
|
|
976
1029
|
* Drop-in for {@link LogRingBuffer} from {@link LogManager}'s perspective —
|
|
977
1030
|
* exposes the same `push` / `getAll` / `query` / `clear` surface.
|
|
978
1031
|
*
|
|
979
|
-
* ##
|
|
1032
|
+
* ## Three bounds, doing different jobs
|
|
980
1033
|
*
|
|
981
1034
|
* **`perAddonCapacity`** is a HARD bound: each bucket is a fixed-size ring, so
|
|
982
1035
|
* total memory can never exceed `perAddonCapacity × buckets` whatever else
|
|
983
1036
|
* happens. This is the safety net.
|
|
984
1037
|
*
|
|
985
|
-
* **`maxTotalEntries`**
|
|
986
|
-
* the hard bound scales with addon count —
|
|
987
|
-
* entries on
|
|
988
|
-
* entries at or below `pruneLevel` (default `debug`), oldest first, from the
|
|
1038
|
+
* **`maxTotalEntries`** targets the total across all buckets, and it exists because
|
|
1039
|
+
* the hard bound scales with addon count — 60 buckets × 5000 was 300k retained
|
|
1040
|
+
* entries on the live hub. When the total is exceeded, the sweep first discards
|
|
1041
|
+
* only entries at or below `pruneLevel` (default `debug`), oldest first, from the
|
|
989
1042
|
* largest buckets first.
|
|
990
1043
|
*
|
|
991
|
-
* It is deliberately soft. Meeting a total cap by discarding
|
|
992
|
-
*
|
|
993
|
-
*
|
|
994
|
-
*
|
|
995
|
-
* that can evict an error is not a feature.
|
|
1044
|
+
* It is deliberately soft ON A LIVE BUCKET. Meeting a total cap by discarding
|
|
1045
|
+
* `warn` and `error` from an addon that is still logging would trade the entries
|
|
1046
|
+
* that explain an incident for entries about frame timings. A cap that can evict
|
|
1047
|
+
* a live addon's error is not a feature.
|
|
996
1048
|
*
|
|
997
1049
|
* Largest-bucket-first because the noisy addon is the one creating the pressure —
|
|
998
1050
|
* the same principle that motivated partitioning in the first place.
|
|
1051
|
+
*
|
|
1052
|
+
* **`maxBuckets`, and bucket eviction, are the hard backstop.** Pruning alone
|
|
1053
|
+
* cannot bound this structure: it may only discard at or below `pruneLevel`, so
|
|
1054
|
+
* a bucket full of `info` from an addon that no longer exists was immortal, and
|
|
1055
|
+
* the roster the product scales with is not fixed. When pruning cannot reach the
|
|
1056
|
+
* target — or the bucket count exceeds `maxBuckets` — whole buckets are dropped
|
|
1057
|
+
* **least-recently-WRITTEN first**, and never the bucket currently being written
|
|
1058
|
+
* to. That is precisely the dead addon's bucket, and it is precisely not the
|
|
1059
|
+
* bucket of the addon whose incident is in progress.
|
|
999
1060
|
*/
|
|
1000
1061
|
var PartitionedLogBuffer = class {
|
|
1001
1062
|
buffers = /* @__PURE__ */ new Map();
|
|
1063
|
+
/**
|
|
1064
|
+
* Bucket key → the write ordinal of its last `push`. A monotonic counter, not
|
|
1065
|
+
* a clock: eviction order must follow the order writes actually happened, and
|
|
1066
|
+
* a wall clock that steps backwards would pick the wrong victim.
|
|
1067
|
+
*/
|
|
1068
|
+
lastWriteSeq = /* @__PURE__ */ new Map();
|
|
1069
|
+
writeSeq = 0;
|
|
1002
1070
|
perAddonCapacity;
|
|
1003
1071
|
maxTotalEntries;
|
|
1004
1072
|
pruneLevel;
|
|
1073
|
+
maxBuckets;
|
|
1005
1074
|
/** Running total, so `push` does not walk every bucket to decide. */
|
|
1006
1075
|
totalEntries = 0;
|
|
1007
1076
|
constructor(perAddonCapacity = 5e3, options = {}) {
|
|
1008
1077
|
this.perAddonCapacity = options.perAddonCapacity ?? perAddonCapacity;
|
|
1009
|
-
this.maxTotalEntries = options.maxTotalEntries
|
|
1078
|
+
this.maxTotalEntries = options.maxTotalEntries === void 0 ? DEFAULT_MAX_TOTAL_LOG_ENTRIES : options.maxTotalEntries;
|
|
1010
1079
|
this.pruneLevel = options.pruneLevel ?? "debug";
|
|
1080
|
+
this.maxBuckets = options.maxBuckets === void 0 ? 128 : options.maxBuckets;
|
|
1011
1081
|
}
|
|
1012
1082
|
/**
|
|
1013
1083
|
* Apply new limits at runtime. Shrinking takes effect as entries are pushed
|
|
@@ -1020,6 +1090,7 @@ var PartitionedLogBuffer = class {
|
|
|
1020
1090
|
if (options.perAddonCapacity !== void 0 && options.perAddonCapacity > 0) this.perAddonCapacity = options.perAddonCapacity;
|
|
1021
1091
|
if (options.maxTotalEntries !== void 0) this.maxTotalEntries = options.maxTotalEntries;
|
|
1022
1092
|
if (options.pruneLevel !== void 0) this.pruneLevel = options.pruneLevel;
|
|
1093
|
+
if (options.maxBuckets !== void 0) this.maxBuckets = options.maxBuckets;
|
|
1023
1094
|
}
|
|
1024
1095
|
bufferFor(key) {
|
|
1025
1096
|
let buf = this.buffers.get(key);
|
|
@@ -1030,34 +1101,82 @@ var PartitionedLogBuffer = class {
|
|
|
1030
1101
|
return buf;
|
|
1031
1102
|
}
|
|
1032
1103
|
push(entry) {
|
|
1033
|
-
const
|
|
1104
|
+
const key = addonBucketKey(entry.tags?.addonId);
|
|
1105
|
+
const buf = this.bufferFor(key);
|
|
1034
1106
|
const before = buf.size();
|
|
1035
1107
|
buf.push(entry);
|
|
1036
1108
|
this.totalEntries += buf.size() - before;
|
|
1037
|
-
|
|
1109
|
+
this.lastWriteSeq.set(key, ++this.writeSeq);
|
|
1110
|
+
if (this.maxBuckets !== null && this.buffers.size > this.maxBuckets) this.evictBuckets(this.buffers.size - this.maxBuckets, key);
|
|
1111
|
+
if (this.maxTotalEntries !== null && this.totalEntries > this.maxTotalEntries) this.enforceTotalCap(this.maxTotalEntries, key);
|
|
1038
1112
|
}
|
|
1039
1113
|
/**
|
|
1040
|
-
* Bring the total down to `PRUNE_TARGET_RATIO × cap
|
|
1041
|
-
*
|
|
1042
|
-
*
|
|
1043
|
-
*
|
|
1044
|
-
|
|
1045
|
-
|
|
1114
|
+
* Bring the total down to `PRUNE_TARGET_RATIO × cap`.
|
|
1115
|
+
*
|
|
1116
|
+
* Two passes, in this order and no other: first discard prunable entries,
|
|
1117
|
+
* largest bucket first, which is free of information cost. Only if that
|
|
1118
|
+
* cannot reach the target does the second pass drop WHOLE buckets,
|
|
1119
|
+
* least-recently-written first — the bound has to hold even when every
|
|
1120
|
+
* retained entry is above `pruneLevel`, and a bucket nobody has written to is
|
|
1121
|
+
* the cheapest thing in the structure to lose.
|
|
1122
|
+
*/
|
|
1123
|
+
enforceTotalCap(cap, protectedKey) {
|
|
1046
1124
|
const target = Math.max(1, Math.floor(cap * PRUNE_TARGET_RATIO));
|
|
1047
1125
|
let deficit = this.totalEntries - target;
|
|
1048
1126
|
if (deficit <= 0) return;
|
|
1049
|
-
const bySizeDesc = [...this.buffers.
|
|
1050
|
-
for (const buf of bySizeDesc) {
|
|
1127
|
+
const bySizeDesc = [...this.buffers.entries()].toSorted((a, b) => b[1].size() - a[1].size());
|
|
1128
|
+
for (const [key, buf] of bySizeDesc) {
|
|
1051
1129
|
if (deficit <= 0) break;
|
|
1052
1130
|
const removed = buf.pruneOldestAtOrBelow(this.pruneLevel, deficit);
|
|
1053
1131
|
this.totalEntries -= removed;
|
|
1054
1132
|
deficit -= removed;
|
|
1133
|
+
if (buf.size() === 0) this.dropBucket(key);
|
|
1055
1134
|
}
|
|
1135
|
+
if (deficit > 0) this.evictBucketsUntil(target, protectedKey);
|
|
1136
|
+
}
|
|
1137
|
+
/**
|
|
1138
|
+
* Drop whole buckets, least-recently-WRITTEN first, until the total is at or
|
|
1139
|
+
* below `target`. Never touches `protectedKey` (the bucket being written to
|
|
1140
|
+
* right now) and always leaves at least one bucket standing.
|
|
1141
|
+
*/
|
|
1142
|
+
evictBucketsUntil(target, protectedKey) {
|
|
1143
|
+
for (const key of this.evictionOrder(protectedKey)) {
|
|
1144
|
+
if (this.totalEntries <= target || this.buffers.size <= 1) return;
|
|
1145
|
+
this.dropBucket(key);
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
/** Drop exactly `count` buckets, least-recently-written first. */
|
|
1149
|
+
evictBuckets(count, protectedKey) {
|
|
1150
|
+
let remaining = count;
|
|
1151
|
+
for (const key of this.evictionOrder(protectedKey)) {
|
|
1152
|
+
if (remaining <= 0 || this.buffers.size <= 1) return;
|
|
1153
|
+
this.dropBucket(key);
|
|
1154
|
+
remaining -= 1;
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
/** Bucket keys ordered least-recently-written first, excluding `protectedKey`.
|
|
1158
|
+
* A bucket with no recorded write sorts first — it can only be a leftover. */
|
|
1159
|
+
evictionOrder(protectedKey) {
|
|
1160
|
+
return [...this.buffers.keys()].filter((key) => key !== protectedKey).toSorted((a, b) => (this.lastWriteSeq.get(a) ?? 0) - (this.lastWriteSeq.get(b) ?? 0));
|
|
1161
|
+
}
|
|
1162
|
+
/** Remove a bucket and everything that indexes it. An empty ring left behind
|
|
1163
|
+
* is the leak this class shipped with — the map only ever grew. */
|
|
1164
|
+
dropBucket(key) {
|
|
1165
|
+
const buf = this.buffers.get(key);
|
|
1166
|
+
if (buf === void 0) return;
|
|
1167
|
+
this.totalEntries -= buf.size();
|
|
1168
|
+
this.buffers.delete(key);
|
|
1169
|
+
this.lastWriteSeq.delete(key);
|
|
1056
1170
|
}
|
|
1057
1171
|
/** Entries retained across every bucket. */
|
|
1058
1172
|
size() {
|
|
1059
1173
|
return this.totalEntries;
|
|
1060
1174
|
}
|
|
1175
|
+
/** Buckets currently retained. Observability for the aggregate bound — an
|
|
1176
|
+
* addon roster that only ever grows is what this class had to stop doing. */
|
|
1177
|
+
bucketCount() {
|
|
1178
|
+
return this.buffers.size;
|
|
1179
|
+
}
|
|
1061
1180
|
/** Every retained entry across all buckets, newest-first (mirrors
|
|
1062
1181
|
* {@link LogRingBuffer.getAll}). Used to replay history to a destination. */
|
|
1063
1182
|
getAll() {
|
|
@@ -1085,13 +1204,25 @@ var PartitionedLogBuffer = class {
|
|
|
1085
1204
|
if (pinned !== null) {
|
|
1086
1205
|
const removed = this.buffers.get(pinned)?.clear(filter) ?? 0;
|
|
1087
1206
|
this.totalEntries -= removed;
|
|
1207
|
+
this.dropIfEmpty(pinned);
|
|
1088
1208
|
return removed;
|
|
1089
1209
|
}
|
|
1090
1210
|
let removed = 0;
|
|
1091
|
-
for (const buf of this.buffers.
|
|
1211
|
+
for (const [key, buf] of [...this.buffers.entries()]) {
|
|
1212
|
+
removed += buf.clear(filter);
|
|
1213
|
+
this.dropIfEmpty(key);
|
|
1214
|
+
}
|
|
1092
1215
|
this.totalEntries -= removed;
|
|
1093
1216
|
return removed;
|
|
1094
1217
|
}
|
|
1218
|
+
/** An emptied bucket is removed rather than left as an empty ring: the map
|
|
1219
|
+
* used to grow with every addon id that ever logged and never shrink. */
|
|
1220
|
+
dropIfEmpty(key) {
|
|
1221
|
+
if (this.buffers.get(key)?.size() === 0) {
|
|
1222
|
+
this.buffers.delete(key);
|
|
1223
|
+
this.lastWriteSeq.delete(key);
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1095
1226
|
/** The bucket key a filter pins to via `tags.addonId`, or null when the
|
|
1096
1227
|
* filter doesn't constrain the addon (→ scan all buckets). */
|
|
1097
1228
|
pinnedAddonKey(filter) {
|
|
@@ -1189,11 +1320,13 @@ var LogManager = class {
|
|
|
1189
1320
|
* `perAddonCapacity` bounds EACH addon's bucket, not the total — a chatty addon
|
|
1190
1321
|
* evicts only its own lines, so quiet addons keep their sparse history.
|
|
1191
1322
|
*
|
|
1192
|
-
* `options.maxTotalEntries`
|
|
1193
|
-
* per-bucket bound scales with addon count
|
|
1194
|
-
*
|
|
1195
|
-
* `
|
|
1196
|
-
*
|
|
1323
|
+
* `options.maxTotalEntries` ceilings the total across all buckets, because the
|
|
1324
|
+
* per-bucket bound scales with addon count, and `options.maxBuckets` ceilings the
|
|
1325
|
+
* bucket count itself. When the total is exceeded, entries at or below
|
|
1326
|
+
* `options.pruneLevel` (default `debug`) are discarded first — a live addon's
|
|
1327
|
+
* `warn`/`error` are never evicted to satisfy it — and only if that cannot reach
|
|
1328
|
+
* the target are whole least-recently-written buckets dropped. Both bounds are ON
|
|
1329
|
+
* by default; pass an explicit `null` to opt out. See {@link PartitionedLogBuffer}.
|
|
1197
1330
|
*/
|
|
1198
1331
|
constructor(perAddonCapacity = 5e3, options = {}) {
|
|
1199
1332
|
this.ringBuffer = new PartitionedLogBuffer(perAddonCapacity, options);
|
|
@@ -94304,33 +94437,45 @@ async function getPidStats$1(pids) {
|
|
|
94304
94437
|
});
|
|
94305
94438
|
});
|
|
94306
94439
|
}
|
|
94440
|
+
var EMPTY_HEAP_DECLARATION = {
|
|
94441
|
+
profile: void 0,
|
|
94442
|
+
maxOldSpaceMb: void 0
|
|
94443
|
+
};
|
|
94307
94444
|
/**
|
|
94308
|
-
* Resolve the declared heap profile for one addon by reading its package.json
|
|
94309
|
-
* (`camstack.addons[].execution.heapProfile`). `addonDir` points at the
|
|
94310
|
-
* manifest is one level up; we also try `addonDir` itself for layouts
|
|
94445
|
+
* Resolve the declared heap profile + old-space ceiling for one addon by reading its package.json
|
|
94446
|
+
* manifest (`camstack.addons[].execution.{heapProfile,maxOldSpaceMb}`). `addonDir` points at the
|
|
94447
|
+
* addon's built `dist`, so the manifest is one level up; we also try `addonDir` itself for layouts
|
|
94448
|
+
* where it IS the package root.
|
|
94311
94449
|
* Result is cached per (dir,id) — spawns are infrequent but a runner may re-spawn on crash/restart.
|
|
94312
|
-
* Returns
|
|
94450
|
+
* Returns an empty declaration when the manifest is unreadable → treated as light, default ceiling.
|
|
94313
94451
|
*/
|
|
94314
94452
|
var heapProfileCache = /* @__PURE__ */ new Map();
|
|
94315
|
-
function
|
|
94453
|
+
function readAddonHeapDeclaration(spec) {
|
|
94316
94454
|
const cacheKey = `${spec.addonDir}::${spec.addonId}`;
|
|
94317
94455
|
const cached = heapProfileCache.get(cacheKey);
|
|
94318
|
-
if (cached !== void 0
|
|
94319
|
-
let
|
|
94456
|
+
if (cached !== void 0) return cached;
|
|
94457
|
+
let declaration = EMPTY_HEAP_DECLARATION;
|
|
94320
94458
|
for (const manifestPath of [node_path.join(spec.addonDir, "package.json"), node_path.join(node_path.dirname(spec.addonDir), "package.json")]) try {
|
|
94321
94459
|
const raw = node_fs.readFileSync(manifestPath, "utf8");
|
|
94322
94460
|
const parsed = JSON.parse(raw);
|
|
94323
|
-
profile = extractHeapProfile(parsed, spec.addonId);
|
|
94324
|
-
|
|
94461
|
+
const profile = extractHeapProfile(parsed, spec.addonId);
|
|
94462
|
+
const maxOldSpaceMb = extractMaxOldSpaceMb(parsed, spec.addonId);
|
|
94463
|
+
if (profile !== void 0 || maxOldSpaceMb !== void 0) {
|
|
94464
|
+
declaration = {
|
|
94465
|
+
profile,
|
|
94466
|
+
maxOldSpaceMb
|
|
94467
|
+
};
|
|
94468
|
+
break;
|
|
94469
|
+
}
|
|
94325
94470
|
if (manifestHasAddon(parsed, spec.addonId)) break;
|
|
94326
94471
|
} catch {}
|
|
94327
|
-
heapProfileCache.set(cacheKey,
|
|
94328
|
-
return
|
|
94472
|
+
heapProfileCache.set(cacheKey, declaration);
|
|
94473
|
+
return declaration;
|
|
94329
94474
|
}
|
|
94330
94475
|
/**
|
|
94331
|
-
* Drop the cached heap
|
|
94476
|
+
* Drop the cached heap declaration for each addon so the NEXT `readAddonHeapDeclaration`
|
|
94332
94477
|
* re-reads the manifest from disk. Called on operator-driven spawns / restarts
|
|
94333
|
-
* (a deploy may have changed
|
|
94478
|
+
* (a deploy may have changed `execution.heapProfile` / `execution.maxOldSpaceMb`); the
|
|
94334
94479
|
* crash-respawn fast path deliberately does NOT invalidate — a crashing runner
|
|
94335
94480
|
* has the same on-disk manifest, so re-reading it is pure overhead.
|
|
94336
94481
|
*/
|
|
@@ -94344,6 +94489,10 @@ function extractHeapProfile(parsed, addonId) {
|
|
|
94344
94489
|
const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.heapProfile;
|
|
94345
94490
|
return value === "heavy" || value === "light" ? value : void 0;
|
|
94346
94491
|
}
|
|
94492
|
+
function extractMaxOldSpaceMb(parsed, addonId) {
|
|
94493
|
+
const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.maxOldSpaceMb;
|
|
94494
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : void 0;
|
|
94495
|
+
}
|
|
94347
94496
|
function readManifestAddons(parsed) {
|
|
94348
94497
|
if (typeof parsed !== "object" || parsed === null) return [];
|
|
94349
94498
|
const camstack = parsed.camstack;
|
|
@@ -94356,25 +94505,78 @@ function readManifestAddons(parsed) {
|
|
|
94356
94505
|
* Everything else (the un-annotated default) is a light control-plane runner. No hard-coded names.
|
|
94357
94506
|
*/
|
|
94358
94507
|
function isHeavyRunner(addons) {
|
|
94359
|
-
return addons.some((a) =>
|
|
94508
|
+
return addons.some((a) => readAddonHeapDeclaration(a).profile === "heavy");
|
|
94509
|
+
}
|
|
94510
|
+
/**
|
|
94511
|
+
* Default old-space ceiling for a HEAVY runner, in MB.
|
|
94512
|
+
*
|
|
94513
|
+
* ## Why a heavy runner needs a ceiling at all
|
|
94514
|
+
*
|
|
94515
|
+
* `heapProfile: 'heavy'` used to mean "V8 defaults", i.e. no `--max-old-space-size` — which on this
|
|
94516
|
+
* host is V8's own memory-derived default (multiple GB). Nothing then bounds a burst, and V8 does
|
|
94517
|
+
* not hand committed pages back (the long argument for that is in `kernel/heap-watch.ts`), so a
|
|
94518
|
+
* one-off burst becomes the runner's permanent RSS.
|
|
94519
|
+
*
|
|
94520
|
+
* Measured on the live hub, 2026-08-17: the `recorder` runner reaches **2122 MB RSS** and stays
|
|
94521
|
+
* there after the deferred archive walk (99,809 segments in 84s), while the `RecordingIndex` it
|
|
94522
|
+
* actually retains is arithmetically ~21 MB (a `SegmentRow` is 5 numbers + 3 strings ≈ 207 B). The
|
|
94523
|
+
* same addon on two agent nodes carrying the same code and a smaller archive sits at **58 MB**.
|
|
94524
|
+
* The ~2 GB is walk garbage held as high-water in a heap that had no reason to collect it.
|
|
94525
|
+
*
|
|
94526
|
+
* ## Why 1024
|
|
94527
|
+
*
|
|
94528
|
+
* From the same live topology snapshot, every heavy runner's FULL RSS — which is a strict
|
|
94529
|
+
* over-estimate of its old space, since it also contains native allocations, `external` and
|
|
94530
|
+
* `arrayBuffers`, none of which `--max-old-space-size` bounds:
|
|
94531
|
+
*
|
|
94532
|
+
* detection 471 MB · provider-reolink 507 MB · pipeline-orchestrator 238 MB · ml 218 MB ·
|
|
94533
|
+
* snapshot 185 MB · decoder-nodeav 174 MB · audio-analyzer 170 MB
|
|
94534
|
+
*
|
|
94535
|
+
* 1024 MB clears the largest of those by ~2x while cutting the recorder's observed ratchet by
|
|
94536
|
+
* more than half. It is a CEILING, not a target: a runner that fits today keeps behaving exactly
|
|
94537
|
+
* as it does now, because V8 only collects harder as it approaches the limit.
|
|
94538
|
+
*
|
|
94539
|
+
* A runner whose working set genuinely does not fit declares its own number in its manifest
|
|
94540
|
+
* (`execution.maxOldSpaceMb`) — the number belongs next to the addon that knows its working set.
|
|
94541
|
+
* `stream-broker` (1323 MB measured, frame buffers) is the one shipped addon that does.
|
|
94542
|
+
*/
|
|
94543
|
+
var HEAVY_MAX_OLD_MB_DEFAULT = 1024;
|
|
94544
|
+
/**
|
|
94545
|
+
* The runner's old-space ceiling in MB, or 0 for "uncapped".
|
|
94546
|
+
*
|
|
94547
|
+
* The runner takes the MAXIMUM ceiling any co-located addon declares — co-location must never
|
|
94548
|
+
* silence the hungriest member (D2/D29: a runner is respawned as a whole, so an OOM caused by
|
|
94549
|
+
* under-sizing one member kills every addon in the group). An addon declaring `0` therefore lifts
|
|
94550
|
+
* the cap for the whole runner, which is the intended escape hatch.
|
|
94551
|
+
*/
|
|
94552
|
+
function runnerMaxOldSpaceMb(addons, heavy) {
|
|
94553
|
+
const envDefault = heavy ? positiveIntEnv("CAMSTACK_RUNNER_HEAVY_MAX_OLD_MB", HEAVY_MAX_OLD_MB_DEFAULT) : positiveIntEnv("CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB", 0);
|
|
94554
|
+
const declared = addons.map((a) => readAddonHeapDeclaration(a).maxOldSpaceMb).filter((mb) => mb !== void 0);
|
|
94555
|
+
if (declared.length === 0) return envDefault;
|
|
94556
|
+
if (declared.includes(0)) return 0;
|
|
94557
|
+
return Math.max(...declared);
|
|
94360
94558
|
}
|
|
94361
94559
|
/**
|
|
94362
94560
|
* Per-placement V8 heap flags (Block D). LIGHT runners cap their young generation
|
|
94363
|
-
* (`--max-semi-space-size`) to reclaim V8's over-reservation — a measured ~15 MB/runner win
|
|
94364
|
-
*
|
|
94365
|
-
*
|
|
94561
|
+
* (`--max-semi-space-size`) to reclaim V8's over-reservation — a measured ~15 MB/runner win — and
|
|
94562
|
+
* keep an opt-in old-space cap (`CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB`, unset = uncapped). HEAVY
|
|
94563
|
+
* runners keep V8's default young generation but ALWAYS carry an old-space ceiling; see
|
|
94564
|
+
* {@link HEAVY_MAX_OLD_MB_DEFAULT} for why, and how a runner declares its own.
|
|
94366
94565
|
*
|
|
94367
94566
|
* Kill-switch: `CAMSTACK_RUNNER_HEAP_TUNING=off` disables all flags (recover without a rebuild).
|
|
94368
94567
|
* Overrides: `CAMSTACK_RUNNER_LIGHT_SEMI_SPACE_MB` (default 2), `CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB`
|
|
94369
|
-
* (default unset = no old-space cap)
|
|
94568
|
+
* (default unset = no old-space cap), `CAMSTACK_RUNNER_HEAVY_MAX_OLD_MB`
|
|
94569
|
+
* (default {@link HEAVY_MAX_OLD_MB_DEFAULT}; `0` restores the old uncapped behaviour).
|
|
94370
94570
|
*/
|
|
94371
94571
|
function runnerHeapFlags(addons) {
|
|
94372
94572
|
if (process.env["CAMSTACK_RUNNER_HEAP_TUNING"] === "off") return [];
|
|
94373
|
-
|
|
94374
|
-
const semiMb = positiveIntEnv("CAMSTACK_RUNNER_LIGHT_SEMI_SPACE_MB", 2);
|
|
94375
|
-
const oldMb = positiveIntEnv("CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB", 0);
|
|
94573
|
+
const heavy = isHeavyRunner(addons);
|
|
94376
94574
|
const flags = [];
|
|
94377
|
-
if (
|
|
94575
|
+
if (!heavy) {
|
|
94576
|
+
const semiMb = positiveIntEnv("CAMSTACK_RUNNER_LIGHT_SEMI_SPACE_MB", 2);
|
|
94577
|
+
if (semiMb > 0) flags.push(`--max-semi-space-size=${semiMb}`);
|
|
94578
|
+
}
|
|
94579
|
+
const oldMb = runnerMaxOldSpaceMb(addons, heavy);
|
|
94378
94580
|
if (oldMb > 0) flags.push(`--max-old-space-size=${oldMb}`);
|
|
94379
94581
|
return flags;
|
|
94380
94582
|
}
|