@camstack/addon-pipeline 1.2.366 → 1.2.368

package/dist/stream-broker/index.js

@@ -3,12 +3,12 @@ Object.defineProperties(exports, {
 	[Symbol.toStringTag]: { value: "Module" }
 });
 const require_chunk = require("../chunk-emK7D4bc.js");
-const require_dist = require("../dist-i4kS_3O7.js");
-const require_node = require("../node-TzgGsCvW.js");
+const require_dist = require("../dist-DDXsZaRH.js");
+const require_node = require("../node-CiciZWts.js");
 const require_restream_intent = require("../restream-intent-7uDNqMKs.js");
 const require_remote_restream = require("../remote-restream-CO36Sr30.js");
 const require_retire_root_keys = require("../retire-root-keys-Csb8z2f3.js");
-const require_segment_demux_js = require("../segment-demux-js-C9AYdKkD.js");
+const require_segment_demux_js = require("../segment-demux-js-0CRf7GAb.js");
 let node_os = require("node:os");
 node_os = require_chunk.__toESM(node_os);
 let node_crypto = require("node:crypto");
@@ -22,6 +22,7 @@ let node_fs_promises = require("node:fs/promises");
 let node_url = require("node:url");
 let node_module = require("node:module");
 let _camstack_system_addon_utils = require("@camstack/system/addon-utils");
+let node_v8 = require("node:v8");
 let node_net = require("node:net");
 node_net = require_chunk.__toESM(node_net);
 let node_dgram = require("node:dgram");
@@ -3086,6 +3087,25 @@ var RtpPacketizer = class {
 };
 //#endregion
 //#region src/stream-broker/stream-broker/broker-mem-probe.ts
+/**
+* broker-mem-probe — what owns the broker runner's NATIVE memory.
+*
+* The runner's `[mem]` heartbeat (`kernel/heap-watch.ts`) says how big the
+* process is; it cannot say who holds it. On the hub, `stream-broker` grew
+* 250 -> 1015 MB in steps while `heapUsed` stayed at ~90 MB and `arrayBuffers`
+* at ~150 MB, so ~600 MB sat outside everything the heartbeat reports. This
+* module is the attribution half: one `[broker-mem]` line every five minutes
+* carrying the kernel's view of the process (RssAnon / RssFile, the `[heap]`
+* brk mapping, the count of anonymous mappings) NEXT TO the broker's own
+* structures (rings, write queues, consumers by kind, egress children), so a
+* step in RSS can be laid against what changed.
+*
+* It is a probe, not a fix: no caps, no allocator tuning, no snapshot. Cost is
+* O(brokers + sessions) per line and O(1) per packet (two integer adds in
+* `copyOwned`). Everything here is a pure function over plain data so the
+* aggregation and the `/proc` parsers are testable on fixture text.
+*/
+var MB = 1024 * 1024;
 /** Cadence of the `[broker-mem]` line — aligned with the runner's `[mem]`. */
 var BROKER_MEM_INTERVAL_MS = 5 * 6e4;
 var copiedPackets = 0;
@@ -3124,6 +3144,7 @@ function parseProcMaps(text) {
 	let heapBytes = 0;
 	let anonMappings = 0;
 	let anonBytes = 0;
+	let v8PageMaps = 0;
 	let from = 0;
 	while (from < text.length) {
 		let to = text.indexOf("\n", from);
@@ -3139,14 +3160,50 @@ function parseProcMaps(text) {
 		if (path === void 0 || path === "") {
 			anonMappings += 1;
 			anonBytes += size;
+			if (size === 262144) v8PageMaps += 1;
 		} else if (path === "[heap]") heapBytes += size;
 	}
 	return {
 		heapBytes,
 		anonMappings,
-		anonBytes
+		anonBytes,
+		v8PageMaps
 	};
 }
+/** Read the main isolate. Probe tick only: `getHeapSpaceStatistics` walks the spaces. */
+function readV8Heap() {
+	const h = (0, node_v8.getHeapStatistics)();
+	return {
+		totalHeapBytes: h.total_heap_size,
+		physicalBytes: h.total_physical_size,
+		mallocedBytes: h.malloced_memory,
+		peakMallocedBytes: h.peak_malloced_memory,
+		externalBytes: h.external_memory,
+		spaces: (0, node_v8.getHeapSpaceStatistics)().map((x) => ({
+			name: x.space_name,
+			committedBytes: x.space_size
+		}))
+	};
+}
+var SPACE_ABBREVIATIONS = {
+	read_only_space: "ro",
+	old_space: "old",
+	new_space: "new",
+	large_object_space: "lo",
+	new_large_object_space: "nlo",
+	code_space: "code",
+	code_large_object_space: "clo",
+	map_space: "map",
+	shared_space: "shared",
+	shared_large_object_space: "slo",
+	trusted_space: "trusted",
+	trusted_large_object_space: "tlo"
+};
+/** `old:120,new:24,lo:8` — committed MB per space, empty spaces left out. */
+function formatHeapSpaces(spaces) {
+	const parts = spaces.filter((x) => x.committedBytes > 0).map((x) => `${SPACE_ABBREVIATIONS[x.name] ?? x.name}:${Math.round(x.committedBytes / MB)}`);
+	return parts.length === 0 ? "none" : parts.join(",");
+}
 /** Opened/closed consumers since the last reading; `drain()` resets. */
 var ConsumerChurn = class {
 	opened = 0;
@@ -3224,7 +3281,6 @@ function reportConsumerEvent(logger, event, kind, deviceId, total, now = Date.no
 	const suppressed = verdict.suppressedBefore > 0 ? ` suppressedBefore=${verdict.suppressedBefore}` : "";
 	logger.info(`consumer ${event} kind=${kind} total=${total}${suppressed}`, { ...deviceId !== void 0 ? { tags: { deviceId } } : {} });
 }
-var MB = 1024 * 1024;
 function ringBytesOf(s) {
 	return s.preBufferBytes + s.clipBytes + s.rtpRingBytes + s.keyframeCacheBytes;
 }
@@ -3267,7 +3323,11 @@ function formatBrokerMem(input) {
 	const top = [...samples].filter((s) => s.deviceId !== void 0).sort((a, b) => ringBytesOf(b) - ringBytesOf(a)).slice(0, 5).map((s) => `${s.deviceId}:${mb(ringBytesOf(s))}`).join(",");
 	const parts = ["[broker-mem]"];
 	if (proc.status !== null) parts.push(`rssAnonMb=${(proc.status.rssAnonKb / 1024).toFixed(0)}`, `rssFileMb=${(proc.status.rssFileKb / 1024).toFixed(0)}`);
-	if (proc.maps !== null) parts.push(`heapMapMb=${mb(proc.maps.heapBytes)}`, `anonMaps=${proc.maps.anonMappings}`, `anonMapsMb=${mb(proc.maps.anonBytes)}`);
+	if (proc.maps !== null) parts.push(`heapMapMb=${mb(proc.maps.heapBytes)}`, `anonMaps=${proc.maps.anonMappings}`, `anonMapsMb=${mb(proc.maps.anonBytes)}`, `v8PageMaps=${proc.maps.v8PageMaps}`);
+	const v8 = input.v8;
+	if (v8 !== void 0 && v8 !== null) parts.push(`v8TotalHeapMb=${mb(v8.totalHeapBytes)}`, `v8PhysicalMb=${mb(v8.physicalBytes)}`, `v8MallocedMb=${mb(v8.mallocedBytes)}`, `v8PeakMallocedMb=${mb(v8.peakMallocedBytes)}`, `v8ExternalMb=${mb(v8.externalBytes)}`, `spaces=${formatHeapSpaces(v8.spaces)}`);
+	const w = input.worker;
+	if (w !== void 0 && w !== null) parts.push(`workers=${w.workers}`, `workerHeapMb=${mb(w.heapBytes)}`, `workerPhysicalMb=${mb(w.physicalBytes)}`, `workerUsedMb=${mb(w.usedBytes)}`, `workerAgeS=${Math.round(w.maxAgeMs / 1e3)}`, `workerIdleExit=${w.idleTerminates ? "yes" : "never"}`);
 	parts.push(`brokers=${samples.length}`, `ingesting=${ingesting}`, `annexBPlanes=${planes}`, `rtsp=${rtsp}`, `webrtc=${input.webrtcSessions}`, `encoded=${kindList(consumers)}`, `audioSubs=${audio}`, `pipeClients=${pipe}`, `egress=${input.egressTranscodes}`, `children=${input.childProcesses}`, `opened=${churn.opened}`, `closed=${churn.closed}`, `openedKinds=${kindList(churn.openedByKind)}`, `closedKinds=${kindList(churn.closedByKind)}`, `preBufferMb=${mb(pre)}`, `clipMb=${mb(clip)}`, `rtpRingMb=${mb(rtp)}`, `keyframeMb=${mb(kf)}`, `writeQueueSumMb=${mb(qSum)}`, `writeQueueMaxMb=${mb(qMax)}`, `copiedPackets=${copied.packets}`, `copiedMb=${mb(copied.bytes)}`, `topRingMb=${top === "" ? "none" : top}`);
 	return parts.join(" ");
 }
@@ -5087,6 +5147,17 @@ var RtspSession = class {
 //#endregion
 //#region src/stream-broker/rtsp/rtsp-listen-server.ts
 /**
+* Attribution of the synthetic consumer a {@link BrokerWarmUp} subscribes:
+* the RTSP listen server holding a cold broker open for a DESCRIBE. Named
+* after that real consumer — it used to subscribe with no attribution and
+* surfaced as `unknown` (45 attach/detach pairs per 5 min on the hub).
+*/
+var RTSP_DESCRIBE_WARMUP_ATTRIBUTION = {
+	kind: "rtsp-listen",
+	label: "describe-warmup",
+	media: "both"
+};
+/**
 * Single TCP server for all RTSP restream connections.
 * Routes clients to the correct RtspRestreamer by a random token path
 * (not the predictable brokerId) for security.
@@ -19933,7 +20004,7 @@ var StreamBrokerManager = class StreamBrokerManager {
 			const sourceBroker = this.brokers.get(sourceBrokerId);
 			if (!sourceBroker) continue;
 			const existingToken = this.persistedTokens.get(profileBrokerId);
-			const warmUp = () => sourceBroker.onEncodedData(() => {});
+			const warmUp = () => sourceBroker.onEncodedData(() => {}, RTSP_DESCRIBE_WARMUP_ATTRIBUTION);
 			const openPrestreamWindow = () => sourceBroker.openPrestreamWindow();
 			this.rtspServer.registerRestreamer(profileBrokerId, sourceBroker.getRtspRestreamer(), warmUp, existingToken, openPrestreamWindow);
 			this.profileRestreamerBindings.set(profileBrokerId, sourceBrokerId);
@@ -21592,7 +21663,7 @@ var StreamBrokerManager = class StreamBrokerManager {
 		this.brokers.set(brokerId, broker);
 		this.applyClipRetention(deviceId);
 		const existingToken = this.persistedTokens.get(brokerId);
-		const warmUp = () => broker.onEncodedData(() => {});
+		const warmUp = () => broker.onEncodedData(() => {}, RTSP_DESCRIBE_WARMUP_ATTRIBUTION);
 		const openPrestreamWindow = () => broker.openPrestreamWindow();
 		this.rtspServer.registerRestreamer(brokerId, broker.getRtspRestreamer(), warmUp, existingToken, openPrestreamWindow);
 		if (!existingToken) this.rtspProvider.persistTokens();
@@ -33870,23 +33941,37 @@ function createHttpClipStreamOpener(options = {}) {
 		req.end();
 	});
 }
-//#endregion
-//#region src/stream-broker/recorded/demux-off-thread.ts
 /**
-* Off-thread JS demux client — mp4box parse on a `worker_threads` Worker
-* inside the stream-broker runner (D228). Same addon, same WebRTC track;
-* the live restream event loop is not the one walking boxes.
-*
-* The worker file is a separate vite entry (`dist/stream-broker/demux-worker-child.js`).
-* This module is inlined into `dist/stream-broker/index.js`, so
-* `import.meta.url` is THAT file and the child is a sibling — the same
-* resolution trick as `fork-decode-worker.ts` / `decode-worker-main.js`.
-*
-* Missing worker file (vitest from `src/`, a botched deploy) falls back to
-* {@link demuxSegmentFullJs} on this thread. A worker that errors a job
-* returns null so {@link demuxSegmentPreferJs} takes the ffmpeg subprocess
-* (already off-loop) instead of re-running mp4box here.
-*/
+* Heap shaping for the demux isolate. The runner's process-wide V8 flags
+* (`--initial-old-space-size=512`, `--max-semi-space-size=8`, from the
+* stream-broker manifest) are sized for the MAIN isolate and leak into every
+* Worker: with a 512 MB initial old space the worker grew to 144 MB of
+* garbage without a single major GC. Explicit `resourceLimits` replace the
+* inherited sizing (measured under those flags with the same allocation
+* churn: worker heap 474 MB without limits, 88 MB with them; see
+* docs/history/sessions.md).
+*
+* 128 MB old generation: parsing a recorded fragment retains about 1 MB
+* after GC for an 86 KB input (fixture bench), so even a 10 MB fragment (a
+* long 4K segment) holds well under 30 MB live; 128 MB is more than 4x that.
+* A parse that still exceeds it fails the worker cleanly
+* (`ERR_WORKER_OUT_OF_MEMORY`): in-flight parses resolve null (ffmpeg
+* fallback) and the next parse spawns a fresh worker.
+*/
+var DEMUX_WORKER_RESOURCE_LIMITS = {
+	maxOldGenerationSizeMb: 128,
+	maxYoungGenerationSizeMb: 16
+};
+var defaultTimers = {
+	set: (fn, ms) => {
+		const t = setTimeout(fn, ms);
+		t.unref();
+		return t;
+	},
+	clear: (handle) => {
+		if (handle !== void 0) clearTimeout(handle);
+	}
+};
 function isCodec(value) {
 	return value === "H264" || value === "H265";
 }
@@ -33921,55 +34006,93 @@ function parseWorkerMessage(message) {
 function defaultWorkerFilename() {
 	return (0, node_url.fileURLToPath)(new URL("./demux-worker-child.js", require("url").pathToFileURL(__filename).href));
 }
-/**
-* JS parse that posts each fragment to one process-wide Worker. Concurrent
-* callers are matched by request id; the worker's own event loop serializes
-* mp4box. Live RTP on the parent thread is not waiting on box parsing.
-*/
-function createOffThreadJsParse(deps) {
+/** {@link createOffThreadJsParse} plus the heap accessor the memory probe reads. */
+function createOffThreadDemuxClient(deps) {
 	const Ctor = deps.WorkerCtor;
-	let worker = null;
+	const idleMs = deps.idleMs ?? 6e4;
+	const timers = deps.timers ?? defaultTimers;
+	const now = deps.now ?? Date.now;
+	let current = null;
 	let nextId = 1;
-	const pending = /* @__PURE__ */ new Map();
-	const failAll = () => {
-		const waiting = [...pending.values()];
-		pending.clear();
-		for (const resolve of waiting) resolve(null);
+	const failAll = (slot, why) => {
+		const waiting = [...slot.pending.values()];
+		slot.pending.clear();
+		if (waiting.length > 0 && deps.logger) for (const deviceId of new Set(waiting.map((p) => p.deviceId))) deps.logger.warn(`demux worker lost with parses in flight (${why}) — ffmpeg fallback`, {
+			...deviceId !== void 0 ? { tags: { deviceId } } : {},
+			meta: { inFlight: waiting.length }
+		});
+		for (const p of waiting) p.resolve(null);
+	};
+	const retire = (slot) => {
+		timers.clear(slot.idleTimer);
+		slot.idleTimer = void 0;
+		if (current === slot) current = null;
+	};
+	const onIdle = async (slot) => {
+		slot.idleTimer = void 0;
+		if (current !== slot || slot.pending.size > 0) return;
+		current = null;
+		const heapBytes = (await slot.worker.getHeapStatistics?.().catch(() => null))?.total_heap_size ?? slot.lastHeapBytes;
+		deps.logger?.info("demux worker idle — terminating", { meta: {
+			ageMs: Math.max(0, now() - slot.spawnedAt),
+			idleMs,
+			lastHeapMb: Math.round(heapBytes / 1048576)
+		} });
+		slot.worker.terminate();
+	};
+	const armIdle = (slot) => {
+		timers.clear(slot.idleTimer);
+		slot.idleTimer = timers.set(() => void onIdle(slot), idleMs);
 	};
-	const attach = (w) => {
+	const attach = (slot) => {
+		const w = slot.worker;
 		w.on("message", (message) => {
 			const parsed = parseWorkerMessage(message);
 			if (!parsed) return;
-			const resolve = pending.get(parsed.id);
-			if (!resolve) return;
-			pending.delete(parsed.id);
-			resolve(parsed.aus);
+			const pending = slot.pending.get(parsed.id);
+			if (!pending) return;
+			slot.pending.delete(parsed.id);
+			pending.resolve(parsed.aus);
+			if (slot.pending.size === 0 && current === slot) armIdle(slot);
 		});
-		w.on("error", () => {
-			worker = null;
-			failAll();
+		w.on("error", (err) => {
+			retire(slot);
+			failAll(slot, String(err));
 		});
 		w.on("exit", () => {
-			worker = null;
-			failAll();
+			retire(slot);
+			failAll(slot, "exit");
 		});
 	};
 	const ensureWorker = () => {
-		if (worker) return worker;
+		if (current) return current;
 		if (!Ctor) return null;
 		try {
-			const spawned = new Ctor(deps.workerFilename);
-			attach(spawned);
-			worker = spawned;
-			return spawned;
-		} catch {
+			const slot = {
+				worker: new Ctor(deps.workerFilename, { resourceLimits: DEMUX_WORKER_RESOURCE_LIMITS }),
+				pending: /* @__PURE__ */ new Map(),
+				spawnedAt: now(),
+				idleTimer: void 0,
+				lastHeapBytes: 0
+			};
+			attach(slot);
+			current = slot;
+			deps.logger?.info("demux worker spawned", { meta: {
+				resourceLimits: DEMUX_WORKER_RESOURCE_LIMITS,
+				idleMs
+			} });
+			return slot;
+		} catch (err) {
+			deps.logger?.warn(`demux worker spawn failed: ${String(err)} — parsing in-process`);
 			return null;
 		}
 	};
-	return async (bytes) => {
+	const parse = async (bytes, ctx) => {
 		if (!deps.existsSync(deps.workerFilename)) return deps.parseInProcess(bytes);
-		const w = ensureWorker();
-		if (!w) return deps.parseInProcess(bytes);
+		const slot = ensureWorker();
+		if (!slot) return deps.parseInProcess(bytes);
+		timers.clear(slot.idleTimer);
+		slot.idleTimer = void 0;
 		const id = nextId;
 		nextId += 1;
 		const copy = new Uint8Array(bytes);
@@ -33978,25 +34101,73 @@ function createOffThreadJsParse(deps) {
 			bytes: copy
 		};
 		return new Promise((resolve) => {
-			pending.set(id, resolve);
-			w.postMessage(request, [copy.buffer]);
+			slot.pending.set(id, {
+				resolve,
+				deviceId: ctx?.deviceId
+			});
+			slot.worker.postMessage(request, [copy.buffer]);
 		});
 	};
+	const readHeaps = async (at = Date.now()) => {
+		const live = current;
+		const empty = {
+			workers: 0,
+			heapBytes: 0,
+			physicalBytes: 0,
+			usedBytes: 0,
+			maxAgeMs: 0,
+			idleTerminates: true
+		};
+		if (live === null) return empty;
+		const stats = await live.worker.getHeapStatistics?.().catch(() => null);
+		if (stats) live.lastHeapBytes = stats.total_heap_size;
+		return {
+			...empty,
+			workers: 1,
+			maxAgeMs: Math.max(0, at - live.spawnedAt),
+			...stats ? {
+				heapBytes: stats.total_heap_size,
+				physicalBytes: stats.total_physical_size,
+				usedBytes: stats.used_heap_size
+			} : {}
+		};
+	};
+	return {
+		parse,
+		readHeaps
+	};
 }
-function defaultOffThreadJsParse() {
-	return createOffThreadJsParse({
+function defaultOffThreadClient() {
+	return createOffThreadDemuxClient({
 		workerFilename: defaultWorkerFilename(),
 		existsSync: node_fs.existsSync,
 		parseInProcess: require_segment_demux_js.demuxSegmentFullJs,
-		WorkerCtor: node_worker_threads.Worker
+		WorkerCtor: node_worker_threads.Worker,
+		...sharedLogger ? { logger: sharedLogger } : {}
 	});
 }
 var shared;
 /** Process-wide JS parse: one Worker, shared by every WebRTC session. */
-var offThreadJsParse = (bytes) => {
-	shared ??= defaultOffThreadJsParse();
-	return shared(bytes);
+var offThreadJsParse = (bytes, ctx) => {
+	shared ??= defaultOffThreadClient();
+	return shared.parse(bytes, ctx);
 };
+var sharedLogger;
+/** Give the process-wide client a logger; call before the first parse. */
+function setDemuxWorkerLogger(logger) {
+	sharedLogger = logger;
+}
+/** Heap of the shared demux worker, for the `[broker-mem]` probe (tick only). */
+async function readDemuxWorkerHeaps(now) {
+	return shared ? shared.readHeaps(now) : {
+		workers: 0,
+		heapBytes: 0,
+		physicalBytes: 0,
+		usedBytes: 0,
+		maxAgeMs: 0,
+		idleTerminates: true
+	};
+}
 /**
 * Drop-in for the session `demuxVideo` hook: JS parse off the live loop,
 * ffmpeg subprocess fallback unchanged.
@@ -34897,6 +35068,7 @@ var BrokerWebrtcServer = class {
 	stopped = false;
 	constructor(options) {
 		this.logger = options.logger;
+		setDemuxWorkerLogger(options.logger.child("demux-worker"));
 		this.getIceServersFn = options.getIceServers;
 		this.staticIceServers = options.iceServers;
 		this.icePortRange = options.icePortRange;
@@ -35562,6 +35734,7 @@ var BrokerWebrtcServer = class {
 		if (locateRecorded && readRecorded) {
 			const demuxVideo = (b) => demuxSegmentPreferJsOffThread(b, {
 				ffmpegPath: resolveFfmpeg(),
+				deviceId,
 				onDemuxPath: (outcome) => {
 					const report = demuxTally.record(outcome, Date.now());
 					if (!report) return;
@@ -37494,7 +37667,7 @@ async function readProc() {
 }
 async function emitOnce(deps) {
 	const now = Date.now();
-	const proc = await readProc();
+	const [proc, worker] = await Promise.all([readProc(), readDemuxWorkerHeaps(now)]);
 	const resources = process.getActiveResourcesInfo();
 	const line = formatBrokerMem({
 		samples: deps.manager.getBrokerInstances().map((b) => b.getMemSample(now)),
@@ -37503,7 +37676,9 @@ async function emitOnce(deps) {
 		childProcesses: countActiveResources(resources, "ProcessWrap"),
 		proc,
 		churn: consumerChurn.drain(),
-		copied: readCopyOwnedTotals()
+		copied: readCopyOwnedTotals(),
+		v8: readV8Heap(),
+		worker
 	});
 	deps.logger.info(line);
 }