@camstack/system 1.2.101 → 1.2.103

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
  2. package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
  3. package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
  4. package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
  5. package/dist/builtins/alerts/alerts.addon.js +1 -1
  6. package/dist/builtins/alerts/alerts.addon.mjs +1 -1
  7. package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +1 -1
  8. package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +1 -1
  9. package/dist/builtins/console-logging/index.js +1 -1
  10. package/dist/builtins/console-logging/index.mjs +1 -1
  11. package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
  12. package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
  13. package/dist/builtins/device-manager/device-manager.addon.js +1 -1
  14. package/dist/builtins/device-manager/device-manager.addon.mjs +1 -1
  15. package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
  16. package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
  17. package/dist/builtins/hub-forwarder/index.js +1 -1
  18. package/dist/builtins/hub-forwarder/index.mjs +1 -1
  19. package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
  20. package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
  21. package/dist/builtins/local-auth/local-auth.addon.js +1 -1
  22. package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
  23. package/dist/builtins/local-network/local-network.addon.js +1 -1
  24. package/dist/builtins/local-network/local-network.addon.mjs +1 -1
  25. package/dist/builtins/loki-logging/index.js +1 -1
  26. package/dist/builtins/loki-logging/index.mjs +1 -1
  27. package/dist/builtins/native-metrics/native-metrics.addon.js +1 -1
  28. package/dist/builtins/native-metrics/native-metrics.addon.mjs +1 -1
  29. package/dist/builtins/platform-probe/index.js +1 -1
  30. package/dist/builtins/platform-probe/index.mjs +1 -1
  31. package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
  32. package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
  33. package/dist/builtins/snapshot/index.js +182 -25
  34. package/dist/builtins/snapshot/index.mjs +182 -25
  35. package/dist/builtins/snapshot/snapshot-cache.d.ts +63 -0
  36. package/dist/builtins/snapshot/snapshot-grab-pick.d.ts +20 -0
  37. package/dist/builtins/snapshot/snapshot-resize.d.ts +33 -1
  38. package/dist/builtins/snapshot/snapshot.addon.d.ts +6 -0
  39. package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +1 -1
  40. package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +1 -1
  41. package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +1 -1
  42. package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +1 -1
  43. package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +1 -1
  44. package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +1 -1
  45. package/dist/builtins/system-config/system-config.addon.js +1 -1
  46. package/dist/builtins/system-config/system-config.addon.mjs +1 -1
  47. package/dist/builtins/winston-logging/index.js +1 -1
  48. package/dist/builtins/winston-logging/index.mjs +1 -1
  49. package/dist/{dist-RbFV8cX7.mjs → dist-DHKXob1m.mjs} +179 -29
  50. package/dist/{dist-tMrPAfpF.js → dist-qnwy_iWM.js} +179 -29
  51. package/dist/index.js +264 -62
  52. package/dist/index.mjs +264 -62
  53. package/dist/kernel/config-schema.d.ts +14 -10
  54. package/dist/kernel/heap-watch.d.ts +39 -7
  55. package/dist/kernel/moleculer/process-service.d.ts +34 -0
  56. package/dist/logging/log-manager.d.ts +7 -5
  57. package/dist/logging/partitioned-log-buffer.d.ts +95 -16
  58. package/package.json +1 -1
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
2
  const require_chunk = require("./chunk-Cek0wNdY.js");
3
- const require_dist = require("./dist-tMrPAfpF.js");
3
+ const require_dist = require("./dist-qnwy_iWM.js");
4
4
  const require_builtins_alerts_alerts_addon = require("./builtins/alerts/alerts.addon.js");
5
5
  require("./builtins/alerts/index.js");
6
6
  const require_formatter = require("./formatter-DqAKDlvN.js");
@@ -501,9 +501,10 @@ function proxyToUpstream(opts) {
501
501
  * ask V8 for the pages back when the process is visibly holding more than it is
502
502
  * using — which is what `shouldReclaim` decides and what the heartbeat now does.
503
503
  *
504
- * The cost is a 250-450ms stop-the-world on the hub's router process, bounded to
505
- * once per `HEAP_RECLAIM_MIN_INTERVAL_MS`. The alternative, measured, is a
506
- * ratchet to 7.8GB and the OOM that ends it.
504
+ * The cost is a stop-the-world on the hub's router process 250-450ms when
505
+ * first measured, 654ms on the largest pass seen since — bounded to once per
506
+ * `HEAP_RECLAIM_MIN_INTERVAL_MS`. The alternative, measured, is a ratchet to
507
+ * 7.8GB and the OOM that ends it.
507
508
  */
508
509
  /** Heartbeat period. Slow enough to be free, fast enough that the sample before
509
510
  * a fatal is recent. */
@@ -541,14 +542,46 @@ function nextMode(current, usedRatio, escalateRatio = HEAP_WATCH_ESCALATE_RATIO,
541
542
  * Live calibration, both readings from the same hub minutes apart: 2491MB
542
543
  * stranded before the reclaim, 438MB after. 438MB is this process's irreducible
543
544
  * native floor (sqlite pages, libvips, sharp, thread stacks, node's own text) —
544
- * collecting against it would buy nothing and cost a stall every minute. 1536
545
- * sits clear of that floor and still fires ~1.1GB before the ratchet is worth
546
- * anything, so it triggers on the real condition and stops as soon as it is fixed.
545
+ * collecting against it would buy nothing and cost a stall every minute.
546
+ *
547
+ * ## Why this came down from 1536 (2026-08-17)
548
+ *
549
+ * The trigger was originally set to fire only on an unmistakable strand. It
550
+ * does, and the pass on the record works exactly as designed:
551
+ *
552
+ * [mem] reclaim hub-main stranded=2365MB rss=4016MB→2368MB freed=1648MB took=654ms
553
+ *
554
+ * But the number it was protecting against changed. The host now runs with a
555
+ * 24GiB cgroup ceiling and **no swap**, and is OOM-killing a neighbouring VM.
556
+ * RSS this process is holding and not using is no longer merely untidy — it is
557
+ * the headroom something else needed, and the OOM is triggered by the PEAK, not
558
+ * by the average. A trigger of 1536 means the peak is allowed to stand 1.5GiB
559
+ * above the floor before anything happens.
560
+ *
561
+ * 1024 is 2.3x the measured 438MB floor, so ordinary movement around the floor
562
+ * can never trip it, and every pass that does fire still returns roughly
563
+ * `1024 - 438` ≈ **580MB minimum** — a real return for a 250-650ms stall,
564
+ * not a stall spent collecting noise.
547
565
  */
548
- var HEAP_RECLAIM_TRIGGER_MB = 1536;
549
- /** Floor on how often the stop-the-world may run. 250-450ms measured, so ten
550
- * minutes keeps the worst case under 0.1% of the router's time. */
551
- var HEAP_RECLAIM_MIN_INTERVAL_MS = 6e5;
566
+ var HEAP_RECLAIM_TRIGGER_MB = 1024;
567
+ /**
568
+ * Floor on how often the stop-the-world may run.
569
+ *
570
+ * Ten minutes was chosen to keep the worst case under 0.1% of the router's
571
+ * time. The worst case is not what governs the cost, though: the reclaim only
572
+ * runs when {@link shouldReclaim} says the process is genuinely stranded, and
573
+ * after a pass it is at the ~438MB floor. Live off-heap growth was measured at
574
+ * roughly 3GB/2h ≈ 25MB/min, so re-stranding the ~580MB needed to qualify again
575
+ * takes on the order of **20 minutes** — well outside any floor considered
576
+ * here. In steady state this constant therefore costs nothing at all.
577
+ *
578
+ * What it does govern is a BURST, which is the only case that matters on a host
579
+ * with no swap: the ten-minute floor let a peak that re-formed in seconds stand
580
+ * for ten minutes. Two minutes bounds that, and even the pathological case of a
581
+ * process that re-strands instantly costs 654ms per 120s — 0.55% of the
582
+ * router's time, against a ~1.5GiB peak it removes each pass.
583
+ */
584
+ var HEAP_RECLAIM_MIN_INTERVAL_MS = 12e4;
552
585
  var MB = (bytes) => Math.round(bytes / 1048576);
553
586
  /**
554
587
  * Build a sample from raw readings. Pure so the thresholds are testable without
@@ -624,8 +657,8 @@ function format$1(label, s) {
624
657
  function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions) {
625
658
  const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
626
659
  const now = reclaimOptions?.now ?? (() => Date.now());
627
- const triggerMb = reclaimOptions?.triggerMb ?? 1536;
628
- const minIntervalMs = reclaimOptions?.minIntervalMs ?? 6e5;
660
+ const triggerMb = reclaimOptions?.triggerMb ?? 1024;
661
+ const minIntervalMs = reclaimOptions?.minIntervalMs ?? 12e4;
629
662
  const fastIntervalMs = reclaimOptions?.fastIntervalMs ?? 2e3;
630
663
  const escalateRatio = reclaimOptions?.escalateRatio ?? .7;
631
664
  const deescalateRatio = reclaimOptions?.deescalateRatio ?? .6;
@@ -968,6 +1001,26 @@ function addonBucketKey(addonId) {
968
1001
  */
969
1002
  var PRUNE_TARGET_RATIO = .9;
970
1003
  /**
1004
+ * Default ceiling on retained entries across every bucket.
1005
+ *
1006
+ * This used to be `null` — no aggregate bound at all — and the only thing
1007
+ * holding hub-main's log memory was `perAddonCapacity x buckets`, a product
1008
+ * that grows with the roster. Measured on the live cluster: 37 forked runners
1009
+ * on the hub plus its in-process builtins, and hub-main additionally ingests
1010
+ * every line from both agents, so ~60 buckets is the ordinary case and
1011
+ * 60 x 5000 = 300k retained entries.
1012
+ *
1013
+ * A realistic entry (timestamp, level, scope, ~70-char message, three tags, no
1014
+ * `meta`) measures **~306 B retained** in V8. So the old unbounded case is
1015
+ * ~92MB of a hub process that is being asked to fit inside a cgroup with a
1016
+ * neighbour to protect, and 50_000 is **~15MB**.
1017
+ *
1018
+ * 50_000 across a ~60-bucket roster is ~830 entries per addon on average while
1019
+ * still letting a single chatty addon hold its full 5000-entry ring — the
1020
+ * partitioning is not weakened, only the total is held.
1021
+ */
1022
+ var DEFAULT_MAX_TOTAL_LOG_ENTRIES = 5e4;
1023
+ /**
971
1024
  * A log buffer partitioned into one fixed-capacity {@link LogRingBuffer} per
972
1025
  * `addonId`. A chatty addon (e.g. detection-pipeline) only ever evicts its OWN
973
1026
  * bucket, so a quiet addon's sparse lines (e.g. a HomeAssistant `image` entity)
@@ -976,38 +1029,55 @@ var PRUNE_TARGET_RATIO = .9;
976
1029
  * Drop-in for {@link LogRingBuffer} from {@link LogManager}'s perspective —
977
1030
  * exposes the same `push` / `getAll` / `query` / `clear` surface.
978
1031
  *
979
- * ## Two bounds, doing different jobs
1032
+ * ## Three bounds, doing different jobs
980
1033
  *
981
1034
  * **`perAddonCapacity`** is a HARD bound: each bucket is a fixed-size ring, so
982
1035
  * total memory can never exceed `perAddonCapacity × buckets` whatever else
983
1036
  * happens. This is the safety net.
984
1037
  *
985
- * **`maxTotalEntries`** is a SOFT target across all buckets, and it exists because
986
- * the hard bound scales with addon count — 70 addons × 5000 is a lot of retained
987
- * entries on a small box. When the total is exceeded, the sweep discards only
988
- * entries at or below `pruneLevel` (default `debug`), oldest first, from the
1038
+ * **`maxTotalEntries`** targets the total across all buckets, and it exists because
1039
+ * the hard bound scales with addon count — 60 buckets × 5000 was 300k retained
1040
+ * entries on the live hub. When the total is exceeded, the sweep first discards
1041
+ * only entries at or below `pruneLevel` (default `debug`), oldest first, from the
989
1042
  * largest buckets first.
990
1043
  *
991
- * It is deliberately soft. Meeting a total cap by discarding `warn` and `error`
992
- * would trade the entries that explain an incident for entries about frame
993
- * timings; if pruning every prunable entry still leaves the total above the cap,
994
- * the buffer stays above it and the hard per-bucket bound is what holds. A cap
995
- * that can evict an error is not a feature.
1044
+ * It is deliberately soft ON A LIVE BUCKET. Meeting a total cap by discarding
1045
+ * `warn` and `error` from an addon that is still logging would trade the entries
1046
+ * that explain an incident for entries about frame timings. A cap that can evict
1047
+ * a live addon's error is not a feature.
996
1048
  *
997
1049
  * Largest-bucket-first because the noisy addon is the one creating the pressure —
998
1050
  * the same principle that motivated partitioning in the first place.
1051
+ *
1052
+ * **`maxBuckets`, and bucket eviction, are the hard backstop.** Pruning alone
1053
+ * cannot bound this structure: it may only discard at or below `pruneLevel`, so
1054
+ * a bucket full of `info` from an addon that no longer exists was immortal, and
1055
+ * the roster the product scales with is not fixed. When pruning cannot reach the
1056
+ * target — or the bucket count exceeds `maxBuckets` — whole buckets are dropped
1057
+ * **least-recently-WRITTEN first**, and never the bucket currently being written
1058
+ * to. That is precisely the dead addon's bucket, and it is precisely not the
1059
+ * bucket of the addon whose incident is in progress.
999
1060
  */
1000
1061
  var PartitionedLogBuffer = class {
1001
1062
  buffers = /* @__PURE__ */ new Map();
1063
+ /**
1064
+ * Bucket key → the write ordinal of its last `push`. A monotonic counter, not
1065
+ * a clock: eviction order must follow the order writes actually happened, and
1066
+ * a wall clock that steps backwards would pick the wrong victim.
1067
+ */
1068
+ lastWriteSeq = /* @__PURE__ */ new Map();
1069
+ writeSeq = 0;
1002
1070
  perAddonCapacity;
1003
1071
  maxTotalEntries;
1004
1072
  pruneLevel;
1073
+ maxBuckets;
1005
1074
  /** Running total, so `push` does not walk every bucket to decide. */
1006
1075
  totalEntries = 0;
1007
1076
  constructor(perAddonCapacity = 5e3, options = {}) {
1008
1077
  this.perAddonCapacity = options.perAddonCapacity ?? perAddonCapacity;
1009
- this.maxTotalEntries = options.maxTotalEntries ?? null;
1078
+ this.maxTotalEntries = options.maxTotalEntries === void 0 ? DEFAULT_MAX_TOTAL_LOG_ENTRIES : options.maxTotalEntries;
1010
1079
  this.pruneLevel = options.pruneLevel ?? "debug";
1080
+ this.maxBuckets = options.maxBuckets === void 0 ? 128 : options.maxBuckets;
1011
1081
  }
1012
1082
  /**
1013
1083
  * Apply new limits at runtime. Shrinking takes effect as entries are pushed
@@ -1020,6 +1090,7 @@ var PartitionedLogBuffer = class {
1020
1090
  if (options.perAddonCapacity !== void 0 && options.perAddonCapacity > 0) this.perAddonCapacity = options.perAddonCapacity;
1021
1091
  if (options.maxTotalEntries !== void 0) this.maxTotalEntries = options.maxTotalEntries;
1022
1092
  if (options.pruneLevel !== void 0) this.pruneLevel = options.pruneLevel;
1093
+ if (options.maxBuckets !== void 0) this.maxBuckets = options.maxBuckets;
1023
1094
  }
1024
1095
  bufferFor(key) {
1025
1096
  let buf = this.buffers.get(key);
@@ -1030,34 +1101,82 @@ var PartitionedLogBuffer = class {
1030
1101
  return buf;
1031
1102
  }
1032
1103
  push(entry) {
1033
- const buf = this.bufferFor(addonBucketKey(entry.tags?.addonId));
1104
+ const key = addonBucketKey(entry.tags?.addonId);
1105
+ const buf = this.bufferFor(key);
1034
1106
  const before = buf.size();
1035
1107
  buf.push(entry);
1036
1108
  this.totalEntries += buf.size() - before;
1037
- if (this.maxTotalEntries !== null && this.totalEntries > this.maxTotalEntries) this.enforceTotalCap(this.maxTotalEntries);
1109
+ this.lastWriteSeq.set(key, ++this.writeSeq);
1110
+ if (this.maxBuckets !== null && this.buffers.size > this.maxBuckets) this.evictBuckets(this.buffers.size - this.maxBuckets, key);
1111
+ if (this.maxTotalEntries !== null && this.totalEntries > this.maxTotalEntries) this.enforceTotalCap(this.maxTotalEntries, key);
1038
1112
  }
1039
1113
  /**
1040
- * Bring the total down to `PRUNE_TARGET_RATIO × cap` by discarding prunable
1041
- * entries, largest bucket first. Stops early when nothing prunable is left —
1042
- * see the class docblock on why that is the correct outcome rather than a
1043
- * failure to enforce.
1044
- */
1045
- enforceTotalCap(cap) {
1114
+ * Bring the total down to `PRUNE_TARGET_RATIO × cap`.
1115
+ *
1116
+ * Two passes, in this order and no other: first discard prunable entries,
1117
+ * largest bucket first, which is free of information cost. Only if that
1118
+ * cannot reach the target does the second pass drop WHOLE buckets,
1119
+ * least-recently-written first — the bound has to hold even when every
1120
+ * retained entry is above `pruneLevel`, and a bucket nobody has written to is
1121
+ * the cheapest thing in the structure to lose.
1122
+ */
1123
+ enforceTotalCap(cap, protectedKey) {
1046
1124
  const target = Math.max(1, Math.floor(cap * PRUNE_TARGET_RATIO));
1047
1125
  let deficit = this.totalEntries - target;
1048
1126
  if (deficit <= 0) return;
1049
- const bySizeDesc = [...this.buffers.values()].sort((a, b) => b.size() - a.size());
1050
- for (const buf of bySizeDesc) {
1127
+ const bySizeDesc = [...this.buffers.entries()].toSorted((a, b) => b[1].size() - a[1].size());
1128
+ for (const [key, buf] of bySizeDesc) {
1051
1129
  if (deficit <= 0) break;
1052
1130
  const removed = buf.pruneOldestAtOrBelow(this.pruneLevel, deficit);
1053
1131
  this.totalEntries -= removed;
1054
1132
  deficit -= removed;
1133
+ if (buf.size() === 0) this.dropBucket(key);
1055
1134
  }
1135
+ if (deficit > 0) this.evictBucketsUntil(target, protectedKey);
1136
+ }
1137
+ /**
1138
+ * Drop whole buckets, least-recently-WRITTEN first, until the total is at or
1139
+ * below `target`. Never touches `protectedKey` (the bucket being written to
1140
+ * right now) and always leaves at least one bucket standing.
1141
+ */
1142
+ evictBucketsUntil(target, protectedKey) {
1143
+ for (const key of this.evictionOrder(protectedKey)) {
1144
+ if (this.totalEntries <= target || this.buffers.size <= 1) return;
1145
+ this.dropBucket(key);
1146
+ }
1147
+ }
1148
+ /** Drop exactly `count` buckets, least-recently-written first. */
1149
+ evictBuckets(count, protectedKey) {
1150
+ let remaining = count;
1151
+ for (const key of this.evictionOrder(protectedKey)) {
1152
+ if (remaining <= 0 || this.buffers.size <= 1) return;
1153
+ this.dropBucket(key);
1154
+ remaining -= 1;
1155
+ }
1156
+ }
1157
+ /** Bucket keys ordered least-recently-written first, excluding `protectedKey`.
1158
+ * A bucket with no recorded write sorts first — it can only be a leftover. */
1159
+ evictionOrder(protectedKey) {
1160
+ return [...this.buffers.keys()].filter((key) => key !== protectedKey).toSorted((a, b) => (this.lastWriteSeq.get(a) ?? 0) - (this.lastWriteSeq.get(b) ?? 0));
1161
+ }
1162
+ /** Remove a bucket and everything that indexes it. An empty ring left behind
1163
+ * is the leak this class shipped with — the map only ever grew. */
1164
+ dropBucket(key) {
1165
+ const buf = this.buffers.get(key);
1166
+ if (buf === void 0) return;
1167
+ this.totalEntries -= buf.size();
1168
+ this.buffers.delete(key);
1169
+ this.lastWriteSeq.delete(key);
1056
1170
  }
1057
1171
  /** Entries retained across every bucket. */
1058
1172
  size() {
1059
1173
  return this.totalEntries;
1060
1174
  }
1175
+ /** Buckets currently retained. Observability for the aggregate bound — an
1176
+ * addon roster that only ever grows is what this class had to stop doing. */
1177
+ bucketCount() {
1178
+ return this.buffers.size;
1179
+ }
1061
1180
  /** Every retained entry across all buckets, newest-first (mirrors
1062
1181
  * {@link LogRingBuffer.getAll}). Used to replay history to a destination. */
1063
1182
  getAll() {
@@ -1085,13 +1204,25 @@ var PartitionedLogBuffer = class {
1085
1204
  if (pinned !== null) {
1086
1205
  const removed = this.buffers.get(pinned)?.clear(filter) ?? 0;
1087
1206
  this.totalEntries -= removed;
1207
+ this.dropIfEmpty(pinned);
1088
1208
  return removed;
1089
1209
  }
1090
1210
  let removed = 0;
1091
- for (const buf of this.buffers.values()) removed += buf.clear(filter);
1211
+ for (const [key, buf] of [...this.buffers.entries()]) {
1212
+ removed += buf.clear(filter);
1213
+ this.dropIfEmpty(key);
1214
+ }
1092
1215
  this.totalEntries -= removed;
1093
1216
  return removed;
1094
1217
  }
1218
+ /** An emptied bucket is removed rather than left as an empty ring: the map
1219
+ * used to grow with every addon id that ever logged and never shrink. */
1220
+ dropIfEmpty(key) {
1221
+ if (this.buffers.get(key)?.size() === 0) {
1222
+ this.buffers.delete(key);
1223
+ this.lastWriteSeq.delete(key);
1224
+ }
1225
+ }
1095
1226
  /** The bucket key a filter pins to via `tags.addonId`, or null when the
1096
1227
  * filter doesn't constrain the addon (→ scan all buckets). */
1097
1228
  pinnedAddonKey(filter) {
@@ -1189,11 +1320,13 @@ var LogManager = class {
1189
1320
  * `perAddonCapacity` bounds EACH addon's bucket, not the total — a chatty addon
1190
1321
  * evicts only its own lines, so quiet addons keep their sparse history.
1191
1322
  *
1192
- * `options.maxTotalEntries` adds a soft ceiling across all buckets, because the
1193
- * per-bucket bound scales with addon count. When it is exceeded, only entries at
1194
- * or below `options.pruneLevel` (default `debug`) are discarded `warn` and
1195
- * `error` are never evicted to satisfy it. Both default to the previous
1196
- * behaviour: no total cap. See {@link PartitionedLogBuffer}.
1323
+ * `options.maxTotalEntries` ceilings the total across all buckets, because the
1324
+ * per-bucket bound scales with addon count, and `options.maxBuckets` ceilings the
1325
+ * bucket count itself. When the total is exceeded, entries at or below
1326
+ * `options.pruneLevel` (default `debug`) are discarded first a live addon's
1327
+ * `warn`/`error` are never evicted to satisfy it — and only if that cannot reach
1328
+ * the target are whole least-recently-written buckets dropped. Both bounds are ON
1329
+ * by default; pass an explicit `null` to opt out. See {@link PartitionedLogBuffer}.
1197
1330
  */
1198
1331
  constructor(perAddonCapacity = 5e3, options = {}) {
1199
1332
  this.ringBuffer = new PartitionedLogBuffer(perAddonCapacity, options);
@@ -94304,33 +94437,45 @@ async function getPidStats$1(pids) {
94304
94437
  });
94305
94438
  });
94306
94439
  }
94440
+ var EMPTY_HEAP_DECLARATION = {
94441
+ profile: void 0,
94442
+ maxOldSpaceMb: void 0
94443
+ };
94307
94444
  /**
94308
- * Resolve the declared heap profile for one addon by reading its package.json manifest
94309
- * (`camstack.addons[].execution.heapProfile`). `addonDir` points at the addon's built `dist`, so the
94310
- * manifest is one level up; we also try `addonDir` itself for layouts where it IS the package root.
94445
+ * Resolve the declared heap profile + old-space ceiling for one addon by reading its package.json
94446
+ * manifest (`camstack.addons[].execution.{heapProfile,maxOldSpaceMb}`). `addonDir` points at the
94447
+ * addon's built `dist`, so the manifest is one level up; we also try `addonDir` itself for layouts
94448
+ * where it IS the package root.
94311
94449
  * Result is cached per (dir,id) — spawns are infrequent but a runner may re-spawn on crash/restart.
94312
- * Returns `undefined` when the manifest is unreadable or declares no profile → treated as light.
94450
+ * Returns an empty declaration when the manifest is unreadable → treated as light, default ceiling.
94313
94451
  */
94314
94452
  var heapProfileCache = /* @__PURE__ */ new Map();
94315
- function readAddonHeapProfile(spec) {
94453
+ function readAddonHeapDeclaration(spec) {
94316
94454
  const cacheKey = `${spec.addonDir}::${spec.addonId}`;
94317
94455
  const cached = heapProfileCache.get(cacheKey);
94318
- if (cached !== void 0 || heapProfileCache.has(cacheKey)) return cached;
94319
- let profile;
94456
+ if (cached !== void 0) return cached;
94457
+ let declaration = EMPTY_HEAP_DECLARATION;
94320
94458
  for (const manifestPath of [node_path.join(spec.addonDir, "package.json"), node_path.join(node_path.dirname(spec.addonDir), "package.json")]) try {
94321
94459
  const raw = node_fs.readFileSync(manifestPath, "utf8");
94322
94460
  const parsed = JSON.parse(raw);
94323
- profile = extractHeapProfile(parsed, spec.addonId);
94324
- if (profile !== void 0) break;
94461
+ const profile = extractHeapProfile(parsed, spec.addonId);
94462
+ const maxOldSpaceMb = extractMaxOldSpaceMb(parsed, spec.addonId);
94463
+ if (profile !== void 0 || maxOldSpaceMb !== void 0) {
94464
+ declaration = {
94465
+ profile,
94466
+ maxOldSpaceMb
94467
+ };
94468
+ break;
94469
+ }
94325
94470
  if (manifestHasAddon(parsed, spec.addonId)) break;
94326
94471
  } catch {}
94327
- heapProfileCache.set(cacheKey, profile);
94328
- return profile;
94472
+ heapProfileCache.set(cacheKey, declaration);
94473
+ return declaration;
94329
94474
  }
94330
94475
  /**
94331
- * Drop the cached heap profile for each addon so the NEXT `readAddonHeapProfile`
94476
+ * Drop the cached heap declaration for each addon so the NEXT `readAddonHeapDeclaration`
94332
94477
  * re-reads the manifest from disk. Called on operator-driven spawns / restarts
94333
- * (a deploy may have changed the addon's `execution.heapProfile`); the
94478
+ * (a deploy may have changed `execution.heapProfile` / `execution.maxOldSpaceMb`); the
94334
94479
  * crash-respawn fast path deliberately does NOT invalidate — a crashing runner
94335
94480
  * has the same on-disk manifest, so re-reading it is pure overhead.
94336
94481
  */
@@ -94344,6 +94489,10 @@ function extractHeapProfile(parsed, addonId) {
94344
94489
  const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.heapProfile;
94345
94490
  return value === "heavy" || value === "light" ? value : void 0;
94346
94491
  }
94492
+ function extractMaxOldSpaceMb(parsed, addonId) {
94493
+ const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.maxOldSpaceMb;
94494
+ return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : void 0;
94495
+ }
94347
94496
  function readManifestAddons(parsed) {
94348
94497
  if (typeof parsed !== "object" || parsed === null) return [];
94349
94498
  const camstack = parsed.camstack;
@@ -94356,25 +94505,78 @@ function readManifestAddons(parsed) {
94356
94505
  * Everything else (the un-annotated default) is a light control-plane runner. No hard-coded names.
94357
94506
  */
94358
94507
  function isHeavyRunner(addons) {
94359
- return addons.some((a) => readAddonHeapProfile(a) === "heavy");
94508
+ return addons.some((a) => readAddonHeapDeclaration(a).profile === "heavy");
94509
+ }
94510
+ /**
94511
+ * Default old-space ceiling for a HEAVY runner, in MB.
94512
+ *
94513
+ * ## Why a heavy runner needs a ceiling at all
94514
+ *
94515
+ * `heapProfile: 'heavy'` used to mean "V8 defaults", i.e. no `--max-old-space-size` — which on this
94516
+ * host is V8's own memory-derived default (multiple GB). Nothing then bounds a burst, and V8 does
94517
+ * not hand committed pages back (the long argument for that is in `kernel/heap-watch.ts`), so a
94518
+ * one-off burst becomes the runner's permanent RSS.
94519
+ *
94520
+ * Measured on the live hub, 2026-08-17: the `recorder` runner reaches **2122 MB RSS** and stays
94521
+ * there after the deferred archive walk (99,809 segments in 84s), while the `RecordingIndex` it
94522
+ * actually retains is arithmetically ~21 MB (a `SegmentRow` is 5 numbers + 3 strings ≈ 207 B). The
94523
+ * same addon on two agent nodes carrying the same code and a smaller archive sits at **58 MB**.
94524
+ * The ~2 GB is walk garbage held as high-water in a heap that had no reason to collect it.
94525
+ *
94526
+ * ## Why 1024
94527
+ *
94528
+ * From the same live topology snapshot, every heavy runner's FULL RSS — which is a strict
94529
+ * over-estimate of its old space, since it also contains native allocations, `external` and
94530
+ * `arrayBuffers`, none of which `--max-old-space-size` bounds:
94531
+ *
94532
+ * detection 471 MB · provider-reolink 507 MB · pipeline-orchestrator 238 MB · ml 218 MB ·
94533
+ * snapshot 185 MB · decoder-nodeav 174 MB · audio-analyzer 170 MB
94534
+ *
94535
+ * 1024 MB clears the largest of those by ~2x while cutting the recorder's observed ratchet by
94536
+ * more than half. It is a CEILING, not a target: a runner that fits today keeps behaving exactly
94537
+ * as it does now, because V8 only collects harder as it approaches the limit.
94538
+ *
94539
+ * A runner whose working set genuinely does not fit declares its own number in its manifest
94540
+ * (`execution.maxOldSpaceMb`) — the number belongs next to the addon that knows its working set.
94541
+ * `stream-broker` (1323 MB measured, frame buffers) is the one shipped addon that does.
94542
+ */
94543
+ var HEAVY_MAX_OLD_MB_DEFAULT = 1024;
94544
+ /**
94545
+ * The runner's old-space ceiling in MB, or 0 for "uncapped".
94546
+ *
94547
+ * The runner takes the MAXIMUM ceiling any co-located addon declares — co-location must never
94548
+ * silence the hungriest member (D2/D29: a runner is respawned as a whole, so an OOM caused by
94549
+ * under-sizing one member kills every addon in the group). An addon declaring `0` therefore lifts
94550
+ * the cap for the whole runner, which is the intended escape hatch.
94551
+ */
94552
+ function runnerMaxOldSpaceMb(addons, heavy) {
94553
+ const envDefault = heavy ? positiveIntEnv("CAMSTACK_RUNNER_HEAVY_MAX_OLD_MB", HEAVY_MAX_OLD_MB_DEFAULT) : positiveIntEnv("CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB", 0);
94554
+ const declared = addons.map((a) => readAddonHeapDeclaration(a).maxOldSpaceMb).filter((mb) => mb !== void 0);
94555
+ if (declared.length === 0) return envDefault;
94556
+ if (declared.includes(0)) return 0;
94557
+ return Math.max(...declared);
94360
94558
  }
94361
94559
  /**
94362
94560
  * Per-placement V8 heap flags (Block D). LIGHT runners cap their young generation
94363
- * (`--max-semi-space-size`) to reclaim V8's over-reservation — a measured ~15 MB/runner win.
94364
- * HEAVY runners keep V8 defaults. The old-space cap is opt-in via env (unset = uncapped = no OOM
94365
- * risk) so it can be enabled after before/after measurement on the real cluster.
94561
+ * (`--max-semi-space-size`) to reclaim V8's over-reservation — a measured ~15 MB/runner win — and
94562
+ * keep an opt-in old-space cap (`CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB`, unset = uncapped). HEAVY
94563
+ * runners keep V8's default young generation but ALWAYS carry an old-space ceiling; see
94564
+ * {@link HEAVY_MAX_OLD_MB_DEFAULT} for why, and how a runner declares its own.
94366
94565
  *
94367
94566
  * Kill-switch: `CAMSTACK_RUNNER_HEAP_TUNING=off` disables all flags (recover without a rebuild).
94368
94567
  * Overrides: `CAMSTACK_RUNNER_LIGHT_SEMI_SPACE_MB` (default 2), `CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB`
94369
- * (default unset = no old-space cap).
94568
+ * (default unset = no old-space cap), `CAMSTACK_RUNNER_HEAVY_MAX_OLD_MB`
94569
+ * (default {@link HEAVY_MAX_OLD_MB_DEFAULT}; `0` restores the old uncapped behaviour).
94370
94570
  */
94371
94571
  function runnerHeapFlags(addons) {
94372
94572
  if (process.env["CAMSTACK_RUNNER_HEAP_TUNING"] === "off") return [];
94373
- if (isHeavyRunner(addons)) return [];
94374
- const semiMb = positiveIntEnv("CAMSTACK_RUNNER_LIGHT_SEMI_SPACE_MB", 2);
94375
- const oldMb = positiveIntEnv("CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB", 0);
94573
+ const heavy = isHeavyRunner(addons);
94376
94574
  const flags = [];
94377
- if (semiMb > 0) flags.push(`--max-semi-space-size=${semiMb}`);
94575
+ if (!heavy) {
94576
+ const semiMb = positiveIntEnv("CAMSTACK_RUNNER_LIGHT_SEMI_SPACE_MB", 2);
94577
+ if (semiMb > 0) flags.push(`--max-semi-space-size=${semiMb}`);
94578
+ }
94579
+ const oldMb = runnerMaxOldSpaceMb(addons, heavy);
94378
94580
  if (oldMb > 0) flags.push(`--max-old-space-size=${oldMb}`);
94379
94581
  return flags;
94380
94582
  }