@camstack/system 1.2.102 → 1.2.104

package/dist/index.js

@@ -1,6 +1,6 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 const require_chunk = require("./chunk-Cek0wNdY.js");
-const require_dist = require("./dist-9sC0wZw6.js");
+const require_dist = require("./dist-DR1VmGz6.js");
 const require_builtins_alerts_alerts_addon = require("./builtins/alerts/alerts.addon.js");
 require("./builtins/alerts/index.js");
 const require_formatter = require("./formatter-DqAKDlvN.js");
@@ -25,7 +25,7 @@ const require_builtins_system_config_system_config_addon = require("./builtins/s
 require("./builtins/system-config/index.js");
 const require_builtins_winston_logging_index = require("./builtins/winston-logging/index.js");
 const require_file_data_plane = require("./file-data-plane-DUHPHa-Y.js");
-const require_manifest_python_deps = require("./manifest-python-deps-BtVjvfvj.js");
+const require_manifest_python_deps = require("./manifest-python-deps-GejnH--L.js");
 const require_resource_monitor = require("./resource-monitor-CdnzxBLP.js");
 const require_custom_action_registry = require("./custom-action-registry-jY0NOZK8.js");
 let zod = require("zod");
@@ -43,8 +43,6 @@ let node_fs = require("node:fs");
 let node_fs$1 = require_chunk.__toESM(node_fs, 1);
 node_fs = require_chunk.__toESM(node_fs);
 let node_http = require("node:http");
-let node_v8 = require("node:v8");
-node_v8 = require_chunk.__toESM(node_v8);
 let node_vm = require("node:vm");
 node_vm = require_chunk.__toESM(node_vm);
 let _camstack_types_addon = require("@camstack/types/addon");
@@ -431,271 +429,6 @@ function proxyToUpstream(opts) {
 	else clientReq.pipe(upstream);
 }
 //#endregion
-//#region src/kernel/heap-watch.ts
-/**
-* heap-watch — hub-main's own memory, on the record, and returned to the OS.
-*
-* ## Why this exists
-*
-* hub-main used to be launched with `--max-old-space-size=1536` (docker-entrypoint;
-* the cap was removed on 2026-08-02, so the live ceiling is now V8's own default —
-* 4288MB in this container — and this module reads it at runtime rather than
-* assuming a number). It logged NOTHING about its own heap. When it died on
-* 2026-08-01/02 the only trace was V8's own stderr line:
-*
-*     FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory
-*
-* which carries no process name, no heap number and no history — the whole
-* server restarts and the evidence is gone. Four of those in ~15h, and the one
-* at 06:57:57 landed nine seconds after a viewer opened a WebRTC session, which
-* is why the app sat in a permanent "loading": the hub was dying as it
-* connected.
-*
-* `[mem]` reports already exist for forked addon runners (SIGUSR2 in
-* `addon-runner.ts`) — hub-main, the one process that actually OOMs, was the
-* one process with no telemetry at all.
-*
-* ## What it gives
-*
-* A heartbeat carrying the numbers that separate the two possible stories, and
-* a WARN as the heap approaches the ceiling — the line that will exist BEFORE
-* the next fatal instead of after it:
-*
-*  - `heapUsedMb` climbing steadily between restarts → a leak;
-*  - `heapUsedMb` flat with a sharp jump under load → a burst allocation, and
-*    the timestamp says what was being served.
-*
-* `external`/`arrayBuffers` ride along because they are NOT bounded by
-* `--max-old-space-size` and are most of the gap between a 1.5GB cap and the
-* ~2.05GB RSS observed live: a diagnosis that only looks at the JS heap
-* misreads that process by 500MB.
-*
-* ## And then the heartbeat explained the "off-heap leak", which was not one
-*
-* 2026-08-08, measured on the live hub over 1019 consecutive samples and seven
-* separate boots: **hub-main's RSS is a HIGH-WATER MARK, not a leak.** In every
-* one of those 1019 samples RSS was >= the running maximum of
-* `heapTotal + external`, never once below it, and
-* `RSS_peak - peak(heapTotal + external)` landed between 874MB and 1639MB on
-* every boot — a roughly constant native floor on top of a number that only
-* ever goes up. RSS tracked the highest point the process ever reached and
-* stayed there: 3.3GB -> 4.5GB -> 5.2GB, and 7.8GB on the worst boot.
-*
-* Two things push that peak, and neither is a retained object:
-*
-*  - the JS heap sawtooths to 2.0-2.8GB because the ceiling is 4288MB and
-*    nothing creates pressure before then;
-*  - `arrayBuffers` accumulates DEAD Buffers — up to 3.65GB — because V8 only
-*    sweeps them on a major GC. Eleven collapses were captured, and every
-*    single one coincides with `heapUsed` dropping to its ~650MB floor. No code
-*    released a reference at 13:23:03; a mark-compact ran.
-*
-* At the peak V8 has committed the pages, and **an ordinary major GC does not
-* give them back.** Proven on the production hub: `smaps` showed 10,819 mapped
-* 256KB pages (V8's `kRegularPageSize`) holding 2704MB while V8 itself reported
-* `total_physical_size` of only 846MB — ~7,800 pages committed, resident, and
-* outside V8's own accounting. One `gc({ flavor: 'last-resort' })` took 417ms
-* and returned RSS 3355MB -> 1237MB, pages 10,819 -> 2,963.
-*
-* So the fix for the "~3GB/2h off-heap growth" is not a retainer hunt. It is to
-* ask V8 for the pages back when the process is visibly holding more than it is
-* using — which is what `shouldReclaim` decides and what the heartbeat now does.
-*
-* The cost is a 250-450ms stop-the-world on the hub's router process, bounded to
-* once per `HEAP_RECLAIM_MIN_INTERVAL_MS`. The alternative, measured, is a
-* ratchet to 7.8GB and the OOM that ends it.
-*/
-/** Heartbeat period. Slow enough to be free, fast enough that the sample before
-*  a fatal is recent. */
-var HEAP_WATCH_INTERVAL_MS = 6e4;
-/** Warn above this share of the heap ceiling. Chosen so the warning precedes
-*  the fatal by more than one interval under realistic growth. */
-var HEAP_WATCH_WARN_RATIO = .8;
-/**
-* Escalate above this share of the ceiling.
-*
-* 0.70 is 3002MB: clear of the measured routine max (2847MB) so steady state
-* never escalates, and below {@link HEAP_WATCH_WARN_RATIO} so the fast cadence
-* is already running by the time the warn is reachable.
-*/
-var HEAP_WATCH_ESCALATE_RATIO = .7;
-/**
-* Release escalation below this share.
-*
-* Hysteresis, not a second threshold: hub-main's heap sawtooths continuously,
-* so a single ratio would flap the mode on every GC. 0.60 is 2573MB — a real
-* retreat rather than the trough of one collection.
-*/
-var HEAP_WATCH_DEESCALATE_RATIO = .6;
-/**
-* The cadence state machine, pure so the thresholds are testable without
-* allocating three gigabytes to reach them.
-*/
-function nextMode(current, usedRatio, escalateRatio = HEAP_WATCH_ESCALATE_RATIO, deescalateRatio = HEAP_WATCH_DEESCALATE_RATIO) {
-	if (current === "escalated") return usedRatio < deescalateRatio ? "steady" : "escalated";
-	return usedRatio >= escalateRatio ? "escalated" : "steady";
-}
-/**
-* Reclaim once the process holds this many MB beyond what it is USING.
-*
-* Live calibration, both readings from the same hub minutes apart: 2491MB
-* stranded before the reclaim, 438MB after. 438MB is this process's irreducible
-* native floor (sqlite pages, libvips, sharp, thread stacks, node's own text) —
-* collecting against it would buy nothing and cost a stall every minute. 1536
-* sits clear of that floor and still fires ~1.1GB before the ratchet is worth
-* anything, so it triggers on the real condition and stops as soon as it is fixed.
-*/
-var HEAP_RECLAIM_TRIGGER_MB = 1536;
-/** Floor on how often the stop-the-world may run. 250-450ms measured, so ten
-*  minutes keeps the worst case under 0.1% of the router's time. */
-var HEAP_RECLAIM_MIN_INTERVAL_MS = 6e5;
-var MB = (bytes) => Math.round(bytes / 1048576);
-/**
-* Build a sample from raw readings. Pure so the thresholds are testable without
-* allocating a gigabyte to reach them.
-*/
-function buildHeapSample(mem, heapLimitBytes, warnRatio = HEAP_WATCH_WARN_RATIO) {
-	const usedRatio = heapLimitBytes > 0 ? mem.heapUsed / heapLimitBytes : 0;
-	return {
-		rssMb: MB(mem.rss),
-		heapUsedMb: MB(mem.heapUsed),
-		heapTotalMb: MB(mem.heapTotal),
-		heapLimitMb: MB(heapLimitBytes),
-		externalMb: MB(mem.external),
-		arrayBuffersMb: MB(mem.arrayBuffers),
-		usedRatio: Math.round(usedRatio * 100) / 100,
-		nearLimit: usedRatio >= warnRatio
-	};
-}
-var consoleSink = {
-	info: (line) => console.log(line),
-	warn: (line) => console.warn(line)
-};
-/**
-* RSS the process is holding but not using.
-*
-* The subtrahend is heapUSED, not heapTotal, deliberately: the pages V8
-* committed at a sawtooth peak and no longer needs are counted inside heapTotal,
-* and they are precisely what this number exists to see.
-*/
-function strandedMb(s) {
-	return s.rssMb - s.heapUsedMb - s.externalMb;
-}
-/** Whether it is worth paying a stop-the-world to get those pages back. */
-function shouldReclaim(s, triggerMb = HEAP_RECLAIM_TRIGGER_MB) {
-	return strandedMb(s) > triggerMb;
-}
-function isGcFunction(value) {
-	return typeof value === "function";
-}
-/**
-* Build the reclaimer without requiring `--expose-gc` on argv.
-*
-* The flag is turned on, the function captured, and the flag turned straight
-* back off — a global `gc` left reachable by application code is a footgun, and
-* requiring an argv change would mean this cannot ship without a new image.
-* Verified against the live hub before being written: 266ms, RSS 1453 -> 1237.
-*
-* Returns undefined if V8 declines, so the caller keeps a heartbeat either way.
-*/
-function createV8Reclaimer() {
-	try {
-		node_v8.setFlagsFromString("--expose-gc");
-		const gc = node_vm.runInNewContext("gc");
-		node_v8.setFlagsFromString("--no-expose-gc");
-		if (!isGcFunction(gc)) return void 0;
-		return () => gc({
-			execution: "sync",
-			flavor: "last-resort"
-		});
-	} catch {
-		return;
-	}
-}
-function format$1(label, s) {
-	return `[mem] ${label} rss=${s.rssMb}MB heapUsed=${s.heapUsedMb}MB heapTotal=${s.heapTotalMb}MB heapLimit=${s.heapLimitMb}MB used=${Math.round(s.usedRatio * 100)}% external=${s.externalMb}MB arrayBuffers=${s.arrayBuffersMb}MB`;
-}
-/**
-* Start the heartbeat. Returns a stop function.
-*
-* `unref`ed so it can never hold the process open, and every read is wrapped:
-* a diagnostic that can crash the process it watches is worse than none.
-*/
-function startHeapWatch(label = "hub-main", sink = consoleSink, intervalMs = HEAP_WATCH_INTERVAL_MS, reclaimOptions) {
-	const readMemory = reclaimOptions?.readMemory ?? (() => process.memoryUsage());
-	const now = reclaimOptions?.now ?? (() => Date.now());
-	const triggerMb = reclaimOptions?.triggerMb ?? 1536;
-	const minIntervalMs = reclaimOptions?.minIntervalMs ?? 6e5;
-	const fastIntervalMs = reclaimOptions?.fastIntervalMs ?? 2e3;
-	const escalateRatio = reclaimOptions?.escalateRatio ?? .7;
-	const deescalateRatio = reclaimOptions?.deescalateRatio ?? .6;
-	let lastReclaimAt = Number.NEGATIVE_INFINITY;
-	const read = () => {
-		const limit = reclaimOptions?.heapLimitBytes ?? node_v8.getHeapStatistics().heap_size_limit;
-		return buildHeapSample(readMemory(), limit);
-	};
-	const maybeReclaim = (sample) => {
-		if (reclaimOptions === void 0) return;
-		if (!shouldReclaim(sample, triggerMb)) return;
-		if (now() - lastReclaimAt < minIntervalMs) return;
-		lastReclaimAt = now();
-		const startedAt = now();
-		try {
-			reclaimOptions.reclaim();
-		} catch (error) {
-			sink.warn(`[mem] reclaim failed — ${error instanceof Error ? error.message : String(error)}`);
-			return;
-		}
-		const after = read();
-		sink.info(`[mem] reclaim ${label} stranded=${strandedMb(sample)}MB rss=${sample.rssMb}MB→${after.rssMb}MB freed=${sample.rssMb - after.rssMb}MB arrayBuffers=${sample.arrayBuffersMb}MB→${after.arrayBuffersMb}MB took=${now() - startedAt}ms`);
-	};
-	let mode = "steady";
-	let lastLoggedAt = Number.NEGATIVE_INFINITY;
-	/**
-	* PROBING and LOGGING are separate cadences, and that separation is the whole
-	* point.
-	*
-	* The 2026-08-08 fatal escaped a 60s heartbeat because the burst began and
-	* ended between two samples. Deciding "is the heap elevated?" only when a
-	* line is due would reproduce that exactly: the process would still be read
-	* once a minute, and a 62-second climb would still be invisible. So the timer
-	* runs at the FAST period always — `process.memoryUsage()` is tens of
-	* microseconds, i.e. free at this rate — and the slow period governs only how
-	* often a steady process is allowed to say so.
-	*
-	* Never probe slower than the log cadence, or a caller passing a short
-	* `intervalMs` would silently get the long one.
-	*/
-	const probeIntervalMs = Math.min(fastIntervalMs, intervalMs);
-	const tick = () => {
-		try {
-			const sample = read();
-			const previous = mode;
-			mode = nextMode(previous, sample.usedRatio, escalateRatio, deescalateRatio);
-			const at = now();
-			const due = at - lastLoggedAt >= intervalMs;
-			if (mode === "escalated" || due) {
-				lastLoggedAt = at;
-				const line = format$1(label, sample);
-				if (sample.nearLimit) sink.warn(`${line} — APPROACHING HEAP LIMIT`);
-				else if (mode === "escalated") sink.warn(`${line} — heap elevated, sampling every ${probeIntervalMs}ms`);
-				else sink.info(line);
-			}
-			if (previous === "escalated" && mode === "steady") sink.info(`[mem] ${label} heap back to routine — logging every ${intervalMs}ms`);
-			maybeReclaim(sample);
-		} catch {}
-	};
-	const timer = setInterval(tick, probeIntervalMs);
-	timer.unref?.();
-	tick();
-	let stopped = false;
-	return () => {
-		if (stopped) return;
-		stopped = true;
-		clearInterval(timer);
-	};
-}
-//#endregion
 //#region src/lifecycle/lifecycle-state-machine.ts
 var VALID_TRANSITIONS = {
 	stopped: ["starting", "disabled"],
@@ -968,6 +701,26 @@ function addonBucketKey(addonId) {
 */
 var PRUNE_TARGET_RATIO = .9;
 /**
+* Default ceiling on retained entries across every bucket.
+*
+* This used to be `null` — no aggregate bound at all — and the only thing
+* holding hub-main's log memory was `perAddonCapacity x buckets`, a product
+* that grows with the roster. Measured on the live cluster: 37 forked runners
+* on the hub plus its in-process builtins, and hub-main additionally ingests
+* every line from both agents, so ~60 buckets is the ordinary case and
+* 60 x 5000 = 300k retained entries.
+*
+* A realistic entry (timestamp, level, scope, ~70-char message, three tags, no
+* `meta`) measures **~306 B retained** in V8. So the old unbounded case is
+* ~92MB of a hub process that is being asked to fit inside a cgroup with a
+* neighbour to protect, and 50_000 is **~15MB**.
+*
+* 50_000 across a ~60-bucket roster is ~830 entries per addon on average while
+* still letting a single chatty addon hold its full 5000-entry ring — the
+* partitioning is not weakened, only the total is held.
+*/
+var DEFAULT_MAX_TOTAL_LOG_ENTRIES = 5e4;
+/**
 * A log buffer partitioned into one fixed-capacity {@link LogRingBuffer} per
 * `addonId`. A chatty addon (e.g. detection-pipeline) only ever evicts its OWN
 * bucket, so a quiet addon's sparse lines (e.g. a HomeAssistant `image` entity)
@@ -976,38 +729,55 @@ var PRUNE_TARGET_RATIO = .9;
 * Drop-in for {@link LogRingBuffer} from {@link LogManager}'s perspective —
 * exposes the same `push` / `getAll` / `query` / `clear` surface.
 *
-* ## Two bounds, doing different jobs
+* ## Three bounds, doing different jobs
 *
 * **`perAddonCapacity`** is a HARD bound: each bucket is a fixed-size ring, so
 * total memory can never exceed `perAddonCapacity × buckets` whatever else
 * happens. This is the safety net.
 *
-* **`maxTotalEntries`** is a SOFT target across all buckets, and it exists because
-* the hard bound scales with addon count — 70 addons × 5000 is a lot of retained
-* entries on a small box. When the total is exceeded, the sweep discards only
-* entries at or below `pruneLevel` (default `debug`), oldest first, from the
+* **`maxTotalEntries`** targets the total across all buckets, and it exists because
+* the hard bound scales with addon count — 60 buckets × 5000 was 300k retained
+* entries on the live hub. When the total is exceeded, the sweep first discards
+* only entries at or below `pruneLevel` (default `debug`), oldest first, from the
 * largest buckets first.
 *
-* It is deliberately soft. Meeting a total cap by discarding `warn` and `error`
-* would trade the entries that explain an incident for entries about frame
-* timings; if pruning every prunable entry still leaves the total above the cap,
-* the buffer stays above it and the hard per-bucket bound is what holds. A cap
-* that can evict an error is not a feature.
+* It is deliberately soft ON A LIVE BUCKET. Meeting a total cap by discarding
+* `warn` and `error` from an addon that is still logging would trade the entries
+* that explain an incident for entries about frame timings. A cap that can evict
+* a live addon's error is not a feature.
 *
 * Largest-bucket-first because the noisy addon is the one creating the pressure —
 * the same principle that motivated partitioning in the first place.
+*
+* **`maxBuckets`, and bucket eviction, are the hard backstop.** Pruning alone
+* cannot bound this structure: it may only discard at or below `pruneLevel`, so
+* a bucket full of `info` from an addon that no longer exists was immortal, and
+* the roster the product scales with is not fixed. When pruning cannot reach the
+* target — or the bucket count exceeds `maxBuckets` — whole buckets are dropped
+* **least-recently-WRITTEN first**, and never the bucket currently being written
+* to. That is precisely the dead addon's bucket, and it is precisely not the
+* bucket of the addon whose incident is in progress.
 */
 var PartitionedLogBuffer = class {
 	buffers = /* @__PURE__ */ new Map();
+	/**
+	* Bucket key → the write ordinal of its last `push`. A monotonic counter, not
+	* a clock: eviction order must follow the order writes actually happened, and
+	* a wall clock that steps backwards would pick the wrong victim.
+	*/
+	lastWriteSeq = /* @__PURE__ */ new Map();
+	writeSeq = 0;
 	perAddonCapacity;
 	maxTotalEntries;
 	pruneLevel;
+	maxBuckets;
 	/** Running total, so `push` does not walk every bucket to decide. */
 	totalEntries = 0;
 	constructor(perAddonCapacity = 5e3, options = {}) {
 		this.perAddonCapacity = options.perAddonCapacity ?? perAddonCapacity;
-		this.maxTotalEntries = options.maxTotalEntries ?? null;
+		this.maxTotalEntries = options.maxTotalEntries === void 0 ? DEFAULT_MAX_TOTAL_LOG_ENTRIES : options.maxTotalEntries;
 		this.pruneLevel = options.pruneLevel ?? "debug";
+		this.maxBuckets = options.maxBuckets === void 0 ? 128 : options.maxBuckets;
 	}
 	/**
 	* Apply new limits at runtime. Shrinking takes effect as entries are pushed
@@ -1020,6 +790,7 @@ var PartitionedLogBuffer = class {
 		if (options.perAddonCapacity !== void 0 && options.perAddonCapacity > 0) this.perAddonCapacity = options.perAddonCapacity;
 		if (options.maxTotalEntries !== void 0) this.maxTotalEntries = options.maxTotalEntries;
 		if (options.pruneLevel !== void 0) this.pruneLevel = options.pruneLevel;
+		if (options.maxBuckets !== void 0) this.maxBuckets = options.maxBuckets;
 	}
 	bufferFor(key) {
 		let buf = this.buffers.get(key);
@@ -1030,34 +801,82 @@ var PartitionedLogBuffer = class {
 		return buf;
 	}
 	push(entry) {
-		const buf = this.bufferFor(addonBucketKey(entry.tags?.addonId));
+		const key = addonBucketKey(entry.tags?.addonId);
+		const buf = this.bufferFor(key);
 		const before = buf.size();
 		buf.push(entry);
 		this.totalEntries += buf.size() - before;
-		if (this.maxTotalEntries !== null && this.totalEntries > this.maxTotalEntries) this.enforceTotalCap(this.maxTotalEntries);
+		this.lastWriteSeq.set(key, ++this.writeSeq);
+		if (this.maxBuckets !== null && this.buffers.size > this.maxBuckets) this.evictBuckets(this.buffers.size - this.maxBuckets, key);
+		if (this.maxTotalEntries !== null && this.totalEntries > this.maxTotalEntries) this.enforceTotalCap(this.maxTotalEntries, key);
 	}
 	/**
-	* Bring the total down to `PRUNE_TARGET_RATIO × cap` by discarding prunable
-	* entries, largest bucket first. Stops early when nothing prunable is left —
-	* see the class docblock on why that is the correct outcome rather than a
-	* failure to enforce.
-	*/
-	enforceTotalCap(cap) {
+	* Bring the total down to `PRUNE_TARGET_RATIO × cap`.
+	*
+	* Two passes, in this order and no other: first discard prunable entries,
+	* largest bucket first, which is free of information cost. Only if that
+	* cannot reach the target does the second pass drop WHOLE buckets,
+	* least-recently-written first — the bound has to hold even when every
+	* retained entry is above `pruneLevel`, and a bucket nobody has written to is
+	* the cheapest thing in the structure to lose.
+	*/
+	enforceTotalCap(cap, protectedKey) {
 		const target = Math.max(1, Math.floor(cap * PRUNE_TARGET_RATIO));
 		let deficit = this.totalEntries - target;
 		if (deficit <= 0) return;
-		const bySizeDesc = [...this.buffers.values()].sort((a, b) => b.size() - a.size());
-		for (const buf of bySizeDesc) {
+		const bySizeDesc = [...this.buffers.entries()].toSorted((a, b) => b[1].size() - a[1].size());
+		for (const [key, buf] of bySizeDesc) {
 			if (deficit <= 0) break;
 			const removed = buf.pruneOldestAtOrBelow(this.pruneLevel, deficit);
 			this.totalEntries -= removed;
 			deficit -= removed;
+			if (buf.size() === 0) this.dropBucket(key);
 		}
+		if (deficit > 0) this.evictBucketsUntil(target, protectedKey);
+	}
+	/**
+	* Drop whole buckets, least-recently-WRITTEN first, until the total is at or
+	* below `target`. Never touches `protectedKey` (the bucket being written to
+	* right now) and always leaves at least one bucket standing.
+	*/
+	evictBucketsUntil(target, protectedKey) {
+		for (const key of this.evictionOrder(protectedKey)) {
+			if (this.totalEntries <= target || this.buffers.size <= 1) return;
+			this.dropBucket(key);
+		}
+	}
+	/** Drop exactly `count` buckets, least-recently-written first. */
+	evictBuckets(count, protectedKey) {
+		let remaining = count;
+		for (const key of this.evictionOrder(protectedKey)) {
+			if (remaining <= 0 || this.buffers.size <= 1) return;
+			this.dropBucket(key);
+			remaining -= 1;
+		}
+	}
+	/** Bucket keys ordered least-recently-written first, excluding `protectedKey`.
+	*  A bucket with no recorded write sorts first — it can only be a leftover. */
+	evictionOrder(protectedKey) {
+		return [...this.buffers.keys()].filter((key) => key !== protectedKey).toSorted((a, b) => (this.lastWriteSeq.get(a) ?? 0) - (this.lastWriteSeq.get(b) ?? 0));
+	}
+	/** Remove a bucket and everything that indexes it. An empty ring left behind
+	*  is the leak this class shipped with — the map only ever grew. */
+	dropBucket(key) {
+		const buf = this.buffers.get(key);
+		if (buf === void 0) return;
+		this.totalEntries -= buf.size();
+		this.buffers.delete(key);
+		this.lastWriteSeq.delete(key);
 	}
 	/** Entries retained across every bucket. */
 	size() {
 		return this.totalEntries;
 	}
+	/** Buckets currently retained. Observability for the aggregate bound — an
+	*  addon roster that only ever grows is what this class had to stop doing. */
+	bucketCount() {
+		return this.buffers.size;
+	}
 	/** Every retained entry across all buckets, newest-first (mirrors
 	*  {@link LogRingBuffer.getAll}). Used to replay history to a destination. */
 	getAll() {
@@ -1085,13 +904,25 @@ var PartitionedLogBuffer = class {
 		if (pinned !== null) {
 			const removed = this.buffers.get(pinned)?.clear(filter) ?? 0;
 			this.totalEntries -= removed;
+			this.dropIfEmpty(pinned);
 			return removed;
 		}
 		let removed = 0;
-		for (const buf of this.buffers.values()) removed += buf.clear(filter);
+		for (const [key, buf] of [...this.buffers.entries()]) {
+			removed += buf.clear(filter);
+			this.dropIfEmpty(key);
+		}
 		this.totalEntries -= removed;
 		return removed;
 	}
+	/** An emptied bucket is removed rather than left as an empty ring: the map
+	*  used to grow with every addon id that ever logged and never shrink. */
+	dropIfEmpty(key) {
+		if (this.buffers.get(key)?.size() === 0) {
+			this.buffers.delete(key);
+			this.lastWriteSeq.delete(key);
+		}
+	}
 	/** The bucket key a filter pins to via `tags.addonId`, or null when the
 	*  filter doesn't constrain the addon (→ scan all buckets). */
 	pinnedAddonKey(filter) {
@@ -1189,11 +1020,13 @@ var LogManager = class {
 	* `perAddonCapacity` bounds EACH addon's bucket, not the total — a chatty addon
 	* evicts only its own lines, so quiet addons keep their sparse history.
 	*
-	* `options.maxTotalEntries` adds a soft ceiling across all buckets, because the
-	* per-bucket bound scales with addon count. When it is exceeded, only entries at
-	* or below `options.pruneLevel` (default `debug`) are discarded — `warn` and
-	* `error` are never evicted to satisfy it. Both default to the previous
-	* behaviour: no total cap. See {@link PartitionedLogBuffer}.
+	* `options.maxTotalEntries` ceilings the total across all buckets, because the
+	* per-bucket bound scales with addon count, and `options.maxBuckets` ceilings the
+	* bucket count itself. When the total is exceeded, entries at or below
+	* `options.pruneLevel` (default `debug`) are discarded first — a live addon's
+	* `warn`/`error` are never evicted to satisfy it — and only if that cannot reach
+	* the target are whole least-recently-written buckets dropped. Both bounds are ON
+	* by default; pass an explicit `null` to opt out. See {@link PartitionedLogBuffer}.
 	*/
 	constructor(perAddonCapacity = 5e3, options = {}) {
 		this.ringBuffer = new PartitionedLogBuffer(perAddonCapacity, options);
@@ -94304,33 +94137,45 @@ async function getPidStats$1(pids) {
 		});
 	});
 }
+var EMPTY_HEAP_DECLARATION = {
+	profile: void 0,
+	maxOldSpaceMb: void 0
+};
 /**
-* Resolve the declared heap profile for one addon by reading its package.json manifest
-* (`camstack.addons[].execution.heapProfile`). `addonDir` points at the addon's built `dist`, so the
-* manifest is one level up; we also try `addonDir` itself for layouts where it IS the package root.
+* Resolve the declared heap profile + old-space ceiling for one addon by reading its package.json
+* manifest (`camstack.addons[].execution.{heapProfile,maxOldSpaceMb}`). `addonDir` points at the
+* addon's built `dist`, so the manifest is one level up; we also try `addonDir` itself for layouts
+* where it IS the package root.
 * Result is cached per (dir,id) — spawns are infrequent but a runner may re-spawn on crash/restart.
-* Returns `undefined` when the manifest is unreadable or declares no profile → treated as light.
+* Returns an empty declaration when the manifest is unreadable → treated as light, default ceiling.
 */
 var heapProfileCache = /* @__PURE__ */ new Map();
-function readAddonHeapProfile(spec) {
+function readAddonHeapDeclaration(spec) {
 	const cacheKey = `${spec.addonDir}::${spec.addonId}`;
 	const cached = heapProfileCache.get(cacheKey);
-	if (cached !== void 0 || heapProfileCache.has(cacheKey)) return cached;
-	let profile;
+	if (cached !== void 0) return cached;
+	let declaration = EMPTY_HEAP_DECLARATION;
 	for (const manifestPath of [node_path.join(spec.addonDir, "package.json"), node_path.join(node_path.dirname(spec.addonDir), "package.json")]) try {
 		const raw = node_fs.readFileSync(manifestPath, "utf8");
 		const parsed = JSON.parse(raw);
-		profile = extractHeapProfile(parsed, spec.addonId);
-		if (profile !== void 0) break;
+		const profile = extractHeapProfile(parsed, spec.addonId);
+		const maxOldSpaceMb = extractMaxOldSpaceMb(parsed, spec.addonId);
+		if (profile !== void 0 || maxOldSpaceMb !== void 0) {
+			declaration = {
+				profile,
+				maxOldSpaceMb
+			};
+			break;
+		}
 		if (manifestHasAddon(parsed, spec.addonId)) break;
 	} catch {}
-	heapProfileCache.set(cacheKey, profile);
-	return profile;
+	heapProfileCache.set(cacheKey, declaration);
+	return declaration;
 }
 /**
-* Drop the cached heap profile for each addon so the NEXT `readAddonHeapProfile`
+* Drop the cached heap declaration for each addon so the NEXT `readAddonHeapDeclaration`
 * re-reads the manifest from disk. Called on operator-driven spawns / restarts
-* (a deploy may have changed the addon's `execution.heapProfile`); the
+* (a deploy may have changed `execution.heapProfile` / `execution.maxOldSpaceMb`); the
 * crash-respawn fast path deliberately does NOT invalidate — a crashing runner
 * has the same on-disk manifest, so re-reading it is pure overhead.
 */
@@ -94344,6 +94189,10 @@ function extractHeapProfile(parsed, addonId) {
 	const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.heapProfile;
 	return value === "heavy" || value === "light" ? value : void 0;
 }
+function extractMaxOldSpaceMb(parsed, addonId) {
+	const value = readManifestAddons(parsed).find((a) => a.id === addonId)?.execution?.maxOldSpaceMb;
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : void 0;
+}
 function readManifestAddons(parsed) {
 	if (typeof parsed !== "object" || parsed === null) return [];
 	const camstack = parsed.camstack;
@@ -94356,25 +94205,78 @@ function readManifestAddons(parsed) {
 * Everything else (the un-annotated default) is a light control-plane runner. No hard-coded names.
 */
 function isHeavyRunner(addons) {
-	return addons.some((a) => readAddonHeapProfile(a) === "heavy");
+	return addons.some((a) => readAddonHeapDeclaration(a).profile === "heavy");
+}
+/**
+* Default old-space ceiling for a HEAVY runner, in MB.
+*
+* ## Why a heavy runner needs a ceiling at all
+*
+* `heapProfile: 'heavy'` used to mean "V8 defaults", i.e. no `--max-old-space-size` — which on this
+* host is V8's own memory-derived default (multiple GB). Nothing then bounds a burst, and V8 does
+* not hand committed pages back (the long argument for that is in `kernel/heap-watch.ts`), so a
+* one-off burst becomes the runner's permanent RSS.
+*
+* Measured on the live hub, 2026-08-17: the `recorder` runner reaches **2122 MB RSS** and stays
+* there after the deferred archive walk (99,809 segments in 84s), while the `RecordingIndex` it
+* actually retains is arithmetically ~21 MB (a `SegmentRow` is 5 numbers + 3 strings ≈ 207 B). The
+* same addon on two agent nodes carrying the same code and a smaller archive sits at **58 MB**.
+* The ~2 GB is walk garbage held as high-water in a heap that had no reason to collect it.
+*
+* ## Why 1024
+*
+* From the same live topology snapshot, every heavy runner's FULL RSS — which is a strict
+* over-estimate of its old space, since it also contains native allocations, `external` and
+* `arrayBuffers`, none of which `--max-old-space-size` bounds:
+*
+*   detection 471 MB · provider-reolink 507 MB · pipeline-orchestrator 238 MB · ml 218 MB ·
+*   snapshot 185 MB · decoder-nodeav 174 MB · audio-analyzer 170 MB
+*
+* 1024 MB clears the largest of those by ~2x while cutting the recorder's observed ratchet by
+* more than half. It is a CEILING, not a target: a runner that fits today keeps behaving exactly
+* as it does now, because V8 only collects harder as it approaches the limit.
+*
+* A runner whose working set genuinely does not fit declares its own number in its manifest
+* (`execution.maxOldSpaceMb`) — the number belongs next to the addon that knows its working set.
+* `stream-broker` (1323 MB measured, frame buffers) is the one shipped addon that does.
+*/
+var HEAVY_MAX_OLD_MB_DEFAULT = 1024;
+/**
+* The runner's old-space ceiling in MB, or 0 for "uncapped".
+*
+* The runner takes the MAXIMUM ceiling any co-located addon declares — co-location must never
+* silence the hungriest member (D2/D29: a runner is respawned as a whole, so an OOM caused by
+* under-sizing one member kills every addon in the group). An addon declaring `0` therefore lifts
+* the cap for the whole runner, which is the intended escape hatch.
+*/
+function runnerMaxOldSpaceMb(addons, heavy) {
+	const envDefault = heavy ? positiveIntEnv("CAMSTACK_RUNNER_HEAVY_MAX_OLD_MB", HEAVY_MAX_OLD_MB_DEFAULT) : positiveIntEnv("CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB", 0);
+	const declared = addons.map((a) => readAddonHeapDeclaration(a).maxOldSpaceMb).filter((mb) => mb !== void 0);
+	if (declared.length === 0) return envDefault;
+	if (declared.includes(0)) return 0;
+	return Math.max(...declared);
 }
 /**
 * Per-placement V8 heap flags (Block D). LIGHT runners cap their young generation
-* (`--max-semi-space-size`) to reclaim V8's over-reservation — a measured ~15 MB/runner win.
-* HEAVY runners keep V8 defaults. The old-space cap is opt-in via env (unset = uncapped = no OOM
-* risk) so it can be enabled after before/after measurement on the real cluster.
+* (`--max-semi-space-size`) to reclaim V8's over-reservation — a measured ~15 MB/runner win — and
+* keep an opt-in old-space cap (`CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB`, unset = uncapped). HEAVY
+* runners keep V8's default young generation but ALWAYS carry an old-space ceiling; see
+* {@link HEAVY_MAX_OLD_MB_DEFAULT} for why, and how a runner declares its own.
 *
 * Kill-switch: `CAMSTACK_RUNNER_HEAP_TUNING=off` disables all flags (recover without a rebuild).
 * Overrides: `CAMSTACK_RUNNER_LIGHT_SEMI_SPACE_MB` (default 2), `CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB`
-* (default unset = no old-space cap).
+* (default unset = no old-space cap), `CAMSTACK_RUNNER_HEAVY_MAX_OLD_MB`
+* (default {@link HEAVY_MAX_OLD_MB_DEFAULT}; `0` restores the old uncapped behaviour).
 */
 function runnerHeapFlags(addons) {
 	if (process.env["CAMSTACK_RUNNER_HEAP_TUNING"] === "off") return [];
-	if (isHeavyRunner(addons)) return [];
-	const semiMb = positiveIntEnv("CAMSTACK_RUNNER_LIGHT_SEMI_SPACE_MB", 2);
-	const oldMb = positiveIntEnv("CAMSTACK_RUNNER_LIGHT_MAX_OLD_MB", 0);
+	const heavy = isHeavyRunner(addons);
 	const flags = [];
-	if (semiMb > 0) flags.push(`--max-semi-space-size=${semiMb}`);
+	if (!heavy) {
+		const semiMb = positiveIntEnv("CAMSTACK_RUNNER_LIGHT_SEMI_SPACE_MB", 2);
+		if (semiMb > 0) flags.push(`--max-semi-space-size=${semiMb}`);
+	}
+	const oldMb = runnerMaxOldSpaceMb(addons, heavy);
 	if (oldMb > 0) flags.push(`--max-old-space-size=${oldMb}`);
 	return flags;
 }
@@ -94451,19 +94353,21 @@ function createProcessService(parentNodeId, dataDir, deps, parentTcpPort, parent
 	function spawnRunner(runnerId, addons, env) {
 		const nodeId = buildNodeId(runnerId);
 		const runnerPath = node_path.resolve(__dirname, "addon-runner.js");
+		const heavy = isHeavyRunner(addons);
 		const childEnv = {
 			...process.env,
 			CAMSTACK_RUNNER_ID: runnerId,
 			CAMSTACK_RUNNER_ADDONS: JSON.stringify(addons),
 			CAMSTACK_PARENT_NODE_ID: parentNodeId,
 			CAMSTACK_DATA_DIR: dataDir,
+			CAMSTACK_RUNNER_HEAP_PROFILE: heavy ? "heavy" : "light",
 			CAMSTACK_LOG_LEVEL: process.env["CAMSTACK_LOG_LEVEL"] ?? "info",
 			...parentTcpPort !== void 0 ? { CAMSTACK_PARENT_TCP_PORT: String(parentTcpPort) } : {},
 			...parentUdsPath !== void 0 ? { CAMSTACK_PARENT_UDS_PATH: parentUdsPath } : {},
 			...env
 		};
 		const heapFlags = runnerHeapFlags(addons);
-		capturedBroker?.logger.info(`[${runnerId}] heap profile: ${isHeavyRunner(addons) ? "heavy" : "light"} flags=[${heapFlags.join(" ")}]`);
+		capturedBroker?.logger.info(`[${runnerId}] heap profile: ${heavy ? "heavy" : "light"} flags=[${heapFlags.join(" ")}]`);
 		const child = spawnFn(process.execPath, [...heapFlags, runnerPath], {
 			env: childEnv,
 			stdio: [
@@ -94507,6 +94411,15 @@ function createProcessService(parentNodeId, dataDir, deps, parentTcpPort, parent
 				capturedBroker?.logger.error(`[${runnerId}] Crash circuit-breaker tripped — ${decision.crashesInWindow} crashes in ${CRASH_WINDOW_MS}ms. Not respawning.`);
 				processes.set(runnerId, entry);
 				if (capturedBroker) forceMoleculerDisconnect(capturedBroker, entry.nodeId);
+				emitRunnerEvent(require_dist.EventCategory.AddonRunnerFailed, runnerId, {
+					runnerId,
+					nodeId: parentNodeId,
+					addonIds: entry.runnerAddons.map((addon) => addon.addonId),
+					crashesInWindow: decision.crashesInWindow,
+					windowMs: CRASH_WINDOW_MS,
+					...code !== null ? { exitCode: code } : {},
+					...signal !== null ? { signal } : {}
+				});
 				for (const addon of entry.runnerAddons) emitRunnerEvent(require_dist.EventCategory.AddonCrashed, addon.addonId, {
 					addonId: addon.addonId,
 					error: `Crash circuit-breaker tripped after ${decision.crashesInWindow} crashes in ${CRASH_WINDOW_MS}ms`
@@ -95233,10 +95146,10 @@ exports.FsStorageBackend = FsStorageBackend;
 exports.HEALTH_MONITOR_GRACE_PERIOD_MS = HEALTH_MONITOR_GRACE_PERIOD_MS;
 exports.HEALTH_MONITOR_RETRY_INTERVALS_MS = HEALTH_MONITOR_RETRY_INTERVALS_MS;
 exports.HEALTH_MONITOR_TICK_MS = HEALTH_MONITOR_TICK_MS;
-exports.HEAP_RECLAIM_MIN_INTERVAL_MS = HEAP_RECLAIM_MIN_INTERVAL_MS;
-exports.HEAP_RECLAIM_TRIGGER_MB = HEAP_RECLAIM_TRIGGER_MB;
-exports.HEAP_WATCH_INTERVAL_MS = HEAP_WATCH_INTERVAL_MS;
-exports.HEAP_WATCH_WARN_RATIO = HEAP_WATCH_WARN_RATIO;
+exports.HEAP_RECLAIM_MIN_INTERVAL_MS = require_manifest_python_deps.HEAP_RECLAIM_MIN_INTERVAL_MS;
+exports.HEAP_RECLAIM_TRIGGER_MB = require_manifest_python_deps.HEAP_RECLAIM_TRIGGER_MB;
+exports.HEAP_WATCH_INTERVAL_MS = require_manifest_python_deps.HEAP_WATCH_INTERVAL_MS;
+exports.HEAP_WATCH_WARN_RATIO = require_manifest_python_deps.HEAP_WATCH_WARN_RATIO;
 exports.HUB_CAP_FWD_ACTION = require_manifest_python_deps.HUB_CAP_FWD_ACTION;
 exports.HUB_CAP_FWD_SERVICE = require_manifest_python_deps.HUB_CAP_FWD_SERVICE;
 exports.HubForwarderAddon = require_builtins_hub_forwarder_index.HubForwarderAddon$1;
@@ -95275,6 +95188,7 @@ exports.PipelineValidator = PipelineValidator;
 exports.PythonEnvManager = PythonEnvManager;
 exports.QUARANTINE_DIRNAME = QUARANTINE_DIRNAME;
 exports.RESTART_MARKER_FILE = RESTART_MARKER_FILE;
+exports.RUNNER_HEAP_WATCH_INTERVAL_MS = require_manifest_python_deps.RUNNER_HEAP_WATCH_INTERVAL_MS;
 exports.RUNTIME_DEFAULTS = require_dist.RUNTIME_DEFAULTS;
 exports.ReadinessRegistry = require_dist.ReadinessRegistry;
 exports.ReadinessTimeoutError = require_dist.ReadinessTimeoutError;
@@ -95313,7 +95227,7 @@ Object.defineProperty(exports, "buildBinaryPath", {
 	}
 });
 exports.buildCapRouters = buildCapRouters;
-exports.buildHeapSample = buildHeapSample;
+exports.buildHeapSample = require_manifest_python_deps.buildHeapSample;
 exports.buildLinkChain = require_manifest_python_deps.buildLinkChain;
 exports.buildNativeCapProxy = require_manifest_python_deps.buildNativeCapProxy;
 exports.buildNodeManifest = buildNodeManifest;
@@ -95358,7 +95272,7 @@ exports.createUdsEventBridge = require_manifest_python_deps.createUdsEventBridge
 exports.createUdsEventBus = require_manifest_python_deps.createUdsEventBus;
 exports.createUdsLogger = require_manifest_python_deps.createUdsLogger;
 exports.createUdsLoggerWithControl = require_manifest_python_deps.createUdsLoggerWithControl;
-exports.createV8Reclaimer = createV8Reclaimer;
+exports.createV8Reclaimer = require_manifest_python_deps.createV8Reclaimer;
 exports.deleteModelFromDisk = require_file_data_plane.deleteModelFromDisk;
 exports.deriveAgentListenPort = deriveAgentListenPort;
 exports.describeProviderKindDrift = describeProviderKindDrift;
@@ -95484,9 +95398,10 @@ exports.selectAddonResidue = selectAddonResidue;
 exports.serializeTypedArrays = require_manifest_python_deps.serializeTypedArrays;
 exports.setHubConnected = require_manifest_python_deps.setHubConnected;
 exports.setNodeEventInterest = require_manifest_python_deps.setNodeEventInterest;
-exports.shouldReclaim = shouldReclaim;
-exports.startHeapWatch = startHeapWatch;
-exports.strandedMb = strandedMb;
+exports.shouldReclaim = require_manifest_python_deps.shouldReclaim;
+exports.startHeapWatch = require_manifest_python_deps.startHeapWatch;
+exports.startRunnerHeapWatch = require_manifest_python_deps.startRunnerHeapWatch;
+exports.strandedMb = require_manifest_python_deps.strandedMb;
 exports.stripCamstackDeps = stripCamstackDeps;
 exports.subscribePassthrough = require_manifest_python_deps.subscribePassthrough;
 exports.udsChildLogToWorkerEntry = require_manifest_python_deps.udsChildLogToWorkerEntry;