@camstack/addon-pipeline 1.2.368 → 1.2.370

package/dist/stream-broker/index.js

@@ -3,12 +3,12 @@ Object.defineProperties(exports, {
 	[Symbol.toStringTag]: { value: "Module" }
 });
 const require_chunk = require("../chunk-emK7D4bc.js");
-const require_dist = require("../dist-DDXsZaRH.js");
-const require_node = require("../node-CiciZWts.js");
+const require_dist = require("../dist-jkwxbcbK.js");
+const require_node = require("../node-C_1U1uzn.js");
 const require_restream_intent = require("../restream-intent-7uDNqMKs.js");
 const require_remote_restream = require("../remote-restream-CO36Sr30.js");
 const require_retire_root_keys = require("../retire-root-keys-Csb8z2f3.js");
-const require_segment_demux_js = require("../segment-demux-js-0CRf7GAb.js");
+const require_segment_demux_js = require("../segment-demux-js-uxBcJy15.js");
 let node_os = require("node:os");
 node_os = require_chunk.__toESM(node_os);
 let node_crypto = require("node:crypto");
@@ -2367,6 +2367,10 @@ var PacedRtpReplay = class {
 		this.enqueuedLive++;
 		return true;
 	}
+	/** Visit every still-queued item (probe time only; no allocation). */
+	forEachQueued(visit) {
+		for (const item of this.queue) visit(item);
+	}
 	/** Cancel the replay: pending packets are dropped, `onDone` fires with
 	*  `aborted: true`. Safe to call multiple times / before `start`. */
 	abort() {
@@ -3086,6 +3090,133 @@ var RtpPacketizer = class {
 	}
 };
 //#endregion
+//#region src/stream-broker/stream-broker/owned-buffer.ts
+/**
+* Buffers that enter a LONG-LIVED ring own their backing store.
+*
+* Node serves `Buffer.from(...)`, `Buffer.allocUnsafe(n)` and `Buffer.concat`
+* below `Buffer.poolSize >>> 1` (4 KiB) as views of shared 8 KiB pool slabs.
+* That is the right choice for the ingest path — a start code, a 4-byte
+* interleave header, a 1.3 KB RTP packet that is fanned out and forgotten
+* within a tick — and the wrong one for anything RETAINED: a 600-byte NAL kept
+* for 60 s in a clip ring pins its whole slab, and with it every short-lived
+* packet that happened to be allocated beside it. On the hub that held
+* 550–980 MB of ArrayBuffers for ~150–250 MB of payload, and the full GCs the
+* pinned slabs caused ran every 0.2–0.5 s at 40–65 % of a core.
+*
+* The ring byte caps counted payload bytes, so they never saw it. Both rings
+* now pass every entry through {@link detachFromPool} on the way in and
+* account it with {@link backingStoreBytes}, so the cap measures what the
+* entry costs. Short-lived traffic stays pooled.
+*/
+/** True when `buf` is the whole of its ArrayBuffer — nothing else shares it. */
+function ownsBackingStore(buf) {
+	return buf.byteOffset === 0 && buf.buffer.byteLength === buf.length;
+}
+/**
+* What retaining `buf` really costs: its own length when it owns its
+* ArrayBuffer, the whole shared slab when it is a view of one.
+*/
+function backingStoreBytes(buf) {
+	return ownsBackingStore(buf) ? buf.length : buf.buffer.byteLength;
+}
+/**
+* `buf` itself when it already owns its ArrayBuffer (decided by inspecting the
+* buffer, never by its size: whether a large `allocUnsafe` is pooled is a Node
+* runtime detail), else a copy that does. One copy, never two: a buffer that is
+* already unpooled is returned as-is.
+*/
+function detachFromPool(buf) {
+	if (ownsBackingStore(buf)) return buf;
+	const owned = Buffer.allocUnsafeSlow(buf.length);
+	buf.copy(owned);
+	return owned;
+}
+/**
+* A copy of `src[start, end)` that OWNS its backing store — the shape a reader
+* hands out when the packet it cuts from its socket buffer is going to be
+* retained anyway.
+*
+* Every source video packet is: `captureRtpForPreBuffer` keeps it for the life
+* of a GOP, and the restreamer's keyframe cache keeps the keyframe ones. With
+* the readers cutting packets as `Buffer.from(subarray)` — a pooled view — the
+* ring then had to copy each one a second time to own it: two copies of every
+* packet on the ingest path (measured 15.1 + 14.8 MB/s on the hub's load).
+* Cutting the packet owned up front makes the ring's and the cache's
+* `detachFromPool` an identity, so the packet is copied once and shared by the
+* reader's fan-out, the ring, the cache and every session's write queue.
+*/
+function copyOwned(src, start, end) {
+	const owned = Buffer.allocUnsafeSlow(end - start);
+	src.copy(owned, 0, start, end);
+	recordCopyOwned(end - start);
+	return owned;
+}
+/** Accumulates entries; each distinct backing ArrayBuffer is summed once. */
+var BackingTally = class {
+	seen = /* @__PURE__ */ new Set();
+	entries = 0;
+	payload = 0;
+	naive = 0;
+	distinct = 0;
+	capped = false;
+	add(buf) {
+		if (this.entries >= 2e5) {
+			this.capped = true;
+			return;
+		}
+		this.entries += 1;
+		this.payload += buf.length;
+		const backing = backingStoreBytes(buf);
+		this.naive += backing;
+		if (!this.seen.has(buf.buffer)) {
+			this.seen.add(buf.buffer);
+			this.distinct += backing;
+		}
+	}
+	totals() {
+		return {
+			entries: this.entries,
+			payloadBytes: this.payload,
+			naiveBackingBytes: this.naive,
+			distinctBackingBytes: this.distinct,
+			capped: this.capped
+		};
+	}
+};
+/** Add every populated slot of a sender's `rtpCache` (≤ RTP_HISTORY_SIZE). */
+function tallyRtpCache(tally, cache) {
+	if (cache === void 0) return;
+	for (let i = 0; i < cache.length; i++) {
+		const slot = cache[i];
+		if (slot !== void 0) tally.add(slot.payload);
+	}
+}
+/** Distinct backing bytes a session pins across all four structures. */
+function sessionBackingBytes(r) {
+	return r.rtpCache.distinctBackingBytes + r.pushQueue.distinctBackingBytes + r.rtpReplay.distinctBackingBytes + r.scrub.distinctBackingBytes;
+}
+function mbOf(bytes) {
+	return Number((bytes / (1024 * 1024)).toFixed(1));
+}
+/** Flat log meta for the session open / close lines. */
+function sessionMemMeta(r) {
+	return {
+		rtpCacheEntries: r.rtpCache.entries,
+		rtpCachePayloadMb: mbOf(r.rtpCache.payloadBytes),
+		rtpCacheNaiveMb: mbOf(r.rtpCache.naiveBackingBytes),
+		rtpCacheBackingMb: mbOf(r.rtpCache.distinctBackingBytes),
+		pushQueueFrames: r.pushQueue.entries,
+		pushQueueBackingMb: mbOf(r.pushQueue.distinctBackingBytes),
+		rtpReplayPackets: r.rtpReplay.entries,
+		rtpReplayBackingMb: mbOf(r.rtpReplay.distinctBackingBytes),
+		scrubPayloadMb: mbOf(r.scrub.payloadBytes),
+		scrubBackingMb: mbOf(r.scrub.distinctBackingBytes),
+		sessionBackingMb: mbOf(sessionBackingBytes(r)),
+		memCapped: r.rtpCache.capped || r.pushQueue.capped || r.rtpReplay.capped || r.scrub.capped
+	};
+}
+//#endregion
 //#region src/stream-broker/stream-broker/broker-mem-probe.ts
 /**
 * broker-mem-probe — what owns the broker runner's NATIVE memory.
@@ -3329,8 +3460,58 @@ function formatBrokerMem(input) {
 	const w = input.worker;
 	if (w !== void 0 && w !== null) parts.push(`workers=${w.workers}`, `workerHeapMb=${mb(w.heapBytes)}`, `workerPhysicalMb=${mb(w.physicalBytes)}`, `workerUsedMb=${mb(w.usedBytes)}`, `workerAgeS=${Math.round(w.maxAgeMs / 1e3)}`, `workerIdleExit=${w.idleTerminates ? "yes" : "never"}`);
 	parts.push(`brokers=${samples.length}`, `ingesting=${ingesting}`, `annexBPlanes=${planes}`, `rtsp=${rtsp}`, `webrtc=${input.webrtcSessions}`, `encoded=${kindList(consumers)}`, `audioSubs=${audio}`, `pipeClients=${pipe}`, `egress=${input.egressTranscodes}`, `children=${input.childProcesses}`, `opened=${churn.opened}`, `closed=${churn.closed}`, `openedKinds=${kindList(churn.openedByKind)}`, `closedKinds=${kindList(churn.closedByKind)}`, `preBufferMb=${mb(pre)}`, `clipMb=${mb(clip)}`, `rtpRingMb=${mb(rtp)}`, `keyframeMb=${mb(kf)}`, `writeQueueSumMb=${mb(qSum)}`, `writeQueueMaxMb=${mb(qMax)}`, `copiedPackets=${copied.packets}`, `copiedMb=${mb(copied.bytes)}`, `topRingMb=${top === "" ? "none" : top}`);
+	const sessions = input.sessions;
+	if (sessions !== void 0) parts.push(...formatSessionMem(sessions, input));
 	return parts.join(" ");
 }
+/**
+* Sums over every WebRTC session, each distinct backing store counted once PER
+* SESSION, and the three views of the NACK history so pinning shows:
+* `rtpCachePayloadMb` (what the packets say), `rtpCacheNaiveMb` (each entry's
+* own backing, a shared slab repeated) and `rtpCacheBackingMb` (each slab once).
+* `fragMb` is the brk heap NOT explained by v8 external memory.
+*/
+function formatSessionMem(sessions, input) {
+	let rtpEntries = 0;
+	let rtpPayload = 0;
+	let rtpNaive = 0;
+	let rtpBacking = 0;
+	let pushFrames = 0;
+	let pushBacking = 0;
+	let replayPackets = 0;
+	let replayBacking = 0;
+	let scrubPayload = 0;
+	let scrubBacking = 0;
+	for (const r of sessions) {
+		rtpEntries += r.rtpCache.entries;
+		rtpPayload += r.rtpCache.payloadBytes;
+		rtpNaive += r.rtpCache.naiveBackingBytes;
+		rtpBacking += r.rtpCache.distinctBackingBytes;
+		pushFrames += r.pushQueue.entries;
+		pushBacking += r.pushQueue.distinctBackingBytes;
+		replayPackets += r.rtpReplay.entries;
+		replayBacking += r.rtpReplay.distinctBackingBytes;
+		scrubPayload += r.scrub.payloadBytes;
+		scrubBacking += r.scrub.distinctBackingBytes;
+	}
+	const top = [...sessions].sort((a, b) => sessionBackingBytes(b) - sessionBackingBytes(a)).slice(0, 3).map((r) => `${r.deviceId ?? "x"}:${mb(sessionBackingBytes(r))}`).join(",");
+	const parts = [
+		`rtpCacheEntries=${rtpEntries}`,
+		`rtpCachePayloadMb=${mb(rtpPayload)}`,
+		`rtpCacheNaiveMb=${mb(rtpNaive)}`,
+		`rtpCacheBackingMb=${mb(rtpBacking)}`,
+		`pushQueueFrames=${pushFrames}`,
+		`pushQueueBackingMb=${mb(pushBacking)}`,
+		`rtpReplayPackets=${replayPackets}`,
+		`rtpReplayBackingMb=${mb(replayBacking)}`,
+		`scrubPayloadMb=${mb(scrubPayload)}`,
+		`scrubBackingMb=${mb(scrubBacking)}`
+	];
+	if (input.arrayBuffersBytes !== void 0) parts.push(`arrayBuffersMb=${mb(input.arrayBuffersBytes)}`);
+	if (input.proc.maps !== null && input.v8 !== void 0 && input.v8 !== null) parts.push(`fragMb=${mb(input.proc.maps.heapBytes - input.v8.externalBytes)}`);
+	parts.push(`topSessionMb=${top === "" ? "none" : top}`);
+	return parts;
+}
 /** Count entries named `name` in `process.getActiveResourcesInfo()`. */
 function countActiveResources(resources, name) {
 	let n = 0;
@@ -3338,69 +3519,6 @@ function countActiveResources(resources, name) {
 	return n;
 }
 //#endregion
-//#region src/stream-broker/stream-broker/owned-buffer.ts
-/**
-* Buffers that enter a LONG-LIVED ring own their backing store.
-*
-* Node serves `Buffer.from(...)`, `Buffer.allocUnsafe(n)` and `Buffer.concat`
-* below `Buffer.poolSize >>> 1` (4 KiB) as views of shared 8 KiB pool slabs.
-* That is the right choice for the ingest path — a start code, a 4-byte
-* interleave header, a 1.3 KB RTP packet that is fanned out and forgotten
-* within a tick — and the wrong one for anything RETAINED: a 600-byte NAL kept
-* for 60 s in a clip ring pins its whole slab, and with it every short-lived
-* packet that happened to be allocated beside it. On the hub that held
-* 550–980 MB of ArrayBuffers for ~150–250 MB of payload, and the full GCs the
-* pinned slabs caused ran every 0.2–0.5 s at 40–65 % of a core.
-*
-* The ring byte caps counted payload bytes, so they never saw it. Both rings
-* now pass every entry through {@link detachFromPool} on the way in and
-* account it with {@link backingStoreBytes}, so the cap measures what the
-* entry costs. Short-lived traffic stays pooled.
-*/
-/** True when `buf` is the whole of its ArrayBuffer — nothing else shares it. */
-function ownsBackingStore(buf) {
-	return buf.byteOffset === 0 && buf.buffer.byteLength === buf.length;
-}
-/**
-* What retaining `buf` really costs: its own length when it owns its
-* ArrayBuffer, the whole shared slab when it is a view of one.
-*/
-function backingStoreBytes(buf) {
-	return ownsBackingStore(buf) ? buf.length : buf.buffer.byteLength;
-}
-/**
-* `buf` itself when it already owns its ArrayBuffer (decided by inspecting the
-* buffer, never by its size: whether a large `allocUnsafe` is pooled is a Node
-* runtime detail), else a copy that does. One copy, never two: a buffer that is
-* already unpooled is returned as-is.
-*/
-function detachFromPool(buf) {
-	if (ownsBackingStore(buf)) return buf;
-	const owned = Buffer.allocUnsafeSlow(buf.length);
-	buf.copy(owned);
-	return owned;
-}
-/**
-* A copy of `src[start, end)` that OWNS its backing store — the shape a reader
-* hands out when the packet it cuts from its socket buffer is going to be
-* retained anyway.
-*
-* Every source video packet is: `captureRtpForPreBuffer` keeps it for the life
-* of a GOP, and the restreamer's keyframe cache keeps the keyframe ones. With
-* the readers cutting packets as `Buffer.from(subarray)` — a pooled view — the
-* ring then had to copy each one a second time to own it: two copies of every
-* packet on the ingest path (measured 15.1 + 14.8 MB/s on the hub's load).
-* Cutting the packet owned up front makes the ring's and the cache's
-* `detachFromPool` an identity, so the packet is copied once and shared by the
-* reader's fan-out, the ring, the cache and every session's write queue.
-*/
-function copyOwned(src, start, end) {
-	const owned = Buffer.allocUnsafeSlow(end - start);
-	src.copy(owned, 0, start, end);
-	recordCopyOwned(end - start);
-	return owned;
-}
-//#endregion
 //#region src/stream-broker/rtsp/sdp-builder.ts
 function buildSdp(options) {
 	const { codec, parameterSets, streamName } = options;
@@ -24943,6 +25061,23 @@ var AdaptiveSession = class AdaptiveSession {
 		} });
 	}
 	/** Whether the session has an active feed (vs idle/pooled). */
+	/**
+	* What this session pins outside the heap, by backing store: werift's NACK
+	* history (video + audio, one tally so a slab shared by both counts once) and
+	* the paced replay queue. Probe / lifecycle-line time only — it walks
+	* ≤ 2 × RTP_HISTORY_SIZE slots and the replay tail, never per packet.
+	*/
+	readMemTotals() {
+		const cache = new BackingTally();
+		tallyRtpCache(cache, this.videoSender?.rtpCache);
+		tallyRtpCache(cache, this.audioSender?.rtpCache);
+		const replay = new BackingTally();
+		this.activeRtpReplay?.forEachQueued((pkt) => replay.add(pkt));
+		return {
+			rtpCache: cache.totals(),
+			rtpReplay: replay.totals()
+		};
+	}
 	get isFeeding() {
 		return this.feedAbort !== null && !this.feedAbort.signal.aborted;
 	}
@@ -28206,6 +28341,13 @@ var ScrubFragmentCache = class {
 	get size() {
 		return this.map.size;
 	}
+	/**
+	* Feed every cached access unit to `tally` — the BACKING-store view of what
+	* `byteSize` counts by `.length`. Probe time only (`session-mem.ts`).
+	*/
+	tallyBacking(tally) {
+		for (const frag of this.map.values()) for (const au of frag.aus) tally.add(au.data);
+	}
 };
 //#endregion
 //#region src/stream-broker/recorded/segment-pts-repair.ts
@@ -35260,7 +35402,8 @@ var BrokerWebrtcServer = class {
 				iceServers: setup.iceServerCount,
 				codec: setup.sessionCodec,
 				useRtpRepacketizer: setup.useRtpRepacketizer,
-				relayOnly: opts.relayOnly === true
+				relayOnly: opts.relayOnly === true,
+				...this.sessionMemMetaFor(setup.sessionId)
 			} });
 			return {
 				sessionId: setup.sessionId,
@@ -35310,7 +35453,8 @@ var BrokerWebrtcServer = class {
 				setupMs,
 				negotiateMs,
 				totalMs: Date.now() - offerStart,
-				...setup.setupTimings
+				...setup.setupTimings,
+				...this.sessionMemMetaFor(setup.sessionId)
 			} });
 			return {
 				sessionId: setup.sessionId,
@@ -35419,7 +35563,7 @@ var BrokerWebrtcServer = class {
 		let framesDroppedSinceLog = 0;
 		let purgesSinceLog = 0;
 		let unalignedPurgesSinceLog = 0;
-		const { source, pushFrame, close: closeSource } = createPushFrameSource((count, keyframeAligned) => {
+		const { source, pushFrame, close: closeSource, forEachQueued: forEachQueuedFrame } = createPushFrameSource((count, keyframeAligned) => {
 			framesDroppedSinceLog += count;
 			purgesSinceLog += 1;
 			if (!keyframeAligned) unalignedPurgesSinceLog += 1;
@@ -35698,6 +35842,7 @@ var BrokerWebrtcServer = class {
 			unsubSdpParams,
 			unsubSourceRestart,
 			closeSource,
+			forEachQueuedFrame,
 			...talkback !== void 0 ? { closeTalkback: () => void talkback.close() } : {},
 			rebind: {
 				deviceId,
@@ -35990,6 +36135,7 @@ var BrokerWebrtcServer = class {
 				})
 			});
 			entry.timeline = timeline;
+			entry.scrubCache = scrubCache;
 			sessionLogger.info("recorded: TimelineSession wired (control accessors present)", { meta: {
 				sessionId,
 				deviceId
@@ -36121,6 +36267,55 @@ var BrokerWebrtcServer = class {
 		return this.sessions.size;
 	}
 	/**
+	* One session's pinned-memory reading, by backing store (`session-mem.ts`).
+	* Walks the session's NACK history, push queue, replay tail and scrub cache:
+	* call at probe / lifecycle-line time only.
+	*/
+	readSessionMem(sessionId, entry) {
+		const own = entry.session.readMemTotals();
+		const push = new BackingTally();
+		entry.forEachQueuedFrame((mf) => push.add(mf.frame.data));
+		const scrub = new BackingTally();
+		entry.scrubCache?.tallyBacking(scrub);
+		return {
+			sessionId,
+			deviceId: deviceIdFromBrokerId(entry.brokerId),
+			rtpCache: own.rtpCache,
+			rtpReplay: own.rtpReplay,
+			pushQueue: push.totals(),
+			scrub: scrub.totals()
+		};
+	}
+	/** Every live session's reading, for the `[broker-mem]` line. */
+	getSessionMemReadings() {
+		const readings = [];
+		for (const [sessionId, entry] of this.sessions) try {
+			readings.push(this.readSessionMem(sessionId, entry));
+		} catch (err) {
+			this.logger.warn("session memory reading failed", {
+				tags: { deviceId: deviceIdFromBrokerId(entry.brokerId) },
+				meta: {
+					sessionId,
+					error: require_dist.errMsg(err)
+				}
+			});
+		}
+		return readings;
+	}
+	/** The counters a lifecycle line carries; empty when the session is gone. */
+	sessionMemMetaFor(sessionId) {
+		const entry = this.sessions.get(sessionId);
+		if (entry === void 0) return {};
+		try {
+			return sessionMemMeta(this.readSessionMem(sessionId, entry));
+		} catch (err) {
+			return {
+				memReadFailed: true,
+				memReadError: require_dist.errMsg(err)
+			};
+		}
+	}
+	/**
 	* People watching LIVE through this server, per camera (`live-viewers.ts`):
 	* sessions of a viewer kind whose track is on the live edge. A session on a
 	* recorded playhead is not watching live; the system's own subscriptions
@@ -36929,6 +37124,15 @@ var BrokerWebrtcServer = class {
 	cleanupSession(sessionId) {
 		const entry = this.sessions.get(sessionId);
 		if (!entry) return;
+		const deviceId = deviceIdFromBrokerId(entry.brokerId);
+		this.logger.info("WebRTC session closing: pinned-memory counters", {
+			tags: { deviceId },
+			meta: {
+				sessionId,
+				deviceId,
+				...this.sessionMemMetaFor(sessionId)
+			}
+		});
 		this.sessions.delete(sessionId);
 		reportConsumerEvent(this.logger, "detach", `webrtc:${entry.consumerKind}`, entry.rebind?.deviceId, this.sessions.size);
 		entry.pendingSwitch?.cancel();
@@ -37123,28 +37327,33 @@ function createPushFrameSource(onFramesDropped) {
 			});
 		}
 	};
-	return {
-		source: (async function* () {
-			try {
-				while (true) {
-					const item = queue.shift();
-					if (item) {
-						yield item;
-						continue;
-					}
-					if (done) return;
-					const result = await new Promise((r) => {
-						resolve = r;
-					});
-					if (result.done) return;
-					yield result.value;
+	const source = (async function* () {
+		try {
+			while (true) {
+				const item = queue.shift();
+				if (item) {
+					yield item;
+					continue;
 				}
-			} finally {
-				done = true;
+				if (done) return;
+				const result = await new Promise((r) => {
+					resolve = r;
+				});
+				if (result.done) return;
+				yield result.value;
 			}
-		})(),
+		} finally {
+			done = true;
+		}
+	})();
+	const forEachQueued = (visit) => {
+		for (const frame of queue) visit(frame);
+	};
+	return {
+		source,
 		pushFrame,
-		close
+		close,
+		forEachQueued
 	};
 }
 //#endregion
@@ -37678,7 +37887,9 @@ async function emitOnce(deps) {
 		churn: consumerChurn.drain(),
 		copied: readCopyOwnedTotals(),
 		v8: readV8Heap(),
-		worker
+		worker,
+		sessions: deps.sessionMem(),
+		arrayBuffersBytes: process.memoryUsage().arrayBuffers
 	});
 	deps.logger.info(line);
 }
@@ -38229,7 +38440,8 @@ var StreamBrokerAddon = class extends require_dist.BaseAddon {
 		this.stopBrokerMemReport = startBrokerMemReport({
 			logger: this.ctx.logger,
 			manager: this.brokerManager,
-			webrtcSessions: () => webrtcServer.getSessionCount()
+			webrtcSessions: () => webrtcServer.getSessionCount(),
+			sessionMem: () => webrtcServer.getSessionMemReadings()
 		});
 		const widgetsProvider = { listWidgets: async () => [{
 			tab: "device-tab",