@camstack/addon-pipeline 1.2.45 → 1.2.46

package/dist/stream-broker/index.js

@@ -3,7 +3,7 @@ Object.defineProperties(exports, {
 	[Symbol.toStringTag]: { value: "Module" }
 });
 const require_chunk = require("../chunk-emK7D4bc.js");
-const require_dist = require("../dist-QyZ7QSRP.js");
+const require_dist = require("../dist-CPfFUIl9.js");
 const require_remote_restream = require("../remote-restream-CO36Sr30.js");
 const require_model_download_service_D8B_4ktF = require("../model-download-service-D8B-4ktF-7-YxE9Wx.js");
 let node_crypto = require("node:crypto");
@@ -3012,6 +3012,147 @@ function buildAudioCodecCapApi(api) {
 	};
 }
 //#endregion
+//#region src/stream-broker/stream-broker/dead-decode-backends.ts
+/**
+* "This decode backend cannot create a device in THIS process" — remembered
+* across children, not per child.
+*
+* ## The thing this stops
+*
+* `-hwaccel vaapi` works for the decoder addon on this hub (ten concurrent 4K
+* decodes) and returns ENOMEM inside the stream-broker runner: `code=244`,
+* "Device creation failed: -12 … Cannot allocate memory". Every egress child
+* therefore spends ~3 s dying before its own software retry rescues it.
+*
+* That was already reported — `EgressTranscodeManager.onDecodeHwAccelFailed`
+* poisoned the broker's 60 s `egressHwAccelCache`. The memo is the wrong shape
+* for the verdict: it exists to avoid a per-session cross-process
+* `decoder.getInfo`, so it expires on a **read** cadence, and one minute later
+* the decoder answers `vaapi` again and the next cold child re-pays the death.
+* Measured on camera 615 on 2026-08-07: the hardware failure recurs at 22:17:07,
+* 22:18:42, 22:28:48, 22:30:52, 22:32:39, 22:34:07, 22:52:07 — once per memo
+* window, forever, on a hub where it has never once succeeded.
+*
+* ## Why this is a quarantine and not a tombstone
+*
+* A permanent verdict is wrong for the failure that is actually observed:
+* ENOMEM on device creation is a RESOURCE condition, and a resource condition
+* can clear (the operator gives the container more of the device, another
+* process releases it). A process that decided "never again" at 03:00 would
+* still be decoding in software at noon with a working accelerator, and nothing
+* would ever say so.
+*
+* So the verdict LASTS and escalates: each consecutive failure of the same
+* backend widens its quarantine — 30 min, 2 h, then 6 h — so a hub where the
+* backend is genuinely dead pays one probe per six hours instead of one per
+* minute, while a hub where it recovers still finds out.
+*
+* The FIRST rung is deliberately not a minute. One report is already a strong
+* signal: the manager only reports a backend after its child produced ZERO
+* packets *and* the software retry proved the same argv otherwise works, which
+* is a statement about the backend and not about the source. And the cost of
+* being wrong is bounded and already being paid — the egress encodes with
+* `libx264` either way, so an over-long quarantine costs a software DECODE on a
+* path that is decoding in software today, 100 % of the time.
+*
+* The invalidations are motivated, and there are exactly three:
+*
+*  1. **The window elapses.** The next acquire probes hardware once. If it
+*     works, {@link DeadDecodeBackends.markAlive} clears the record and the
+*     escalation starts from zero again.
+*  2. **The backend changes.** The record is keyed by backend name, so an
+*     operator switching the decoder from vaapi to qsv is obeyed immediately —
+*     qsv has no record.
+*  3. **The runner respawns.** This is instance state on the broker manager,
+*     never a module global, for the same reason `HwAccelCache` is: a module
+*     global outlives an addon respawn, and an addon respawn is exactly the
+*     event after which a stale verdict must not survive.
+*
+* ## Why it lives here and not in `@camstack/types`
+*
+* `addon-export-hap` has the identical gap (its `failedBackend` memo forgets on
+* the same 60 s cadence) and this is deliberately NOT shared with it yet: a
+* framework package is only live once a `@camstack/server` containing it is
+* installed, whereas this ships with `camstack deploy packages/addon-pipeline`.
+* The cross-PROCESS gap D67 names — addons may not import each other and no cap
+* carries "this backend is dead on this node right now" — is unchanged either
+* way; what is closed here is the broker's own repetition of the cost.
+*/
+/**
+* How long a backend stays quarantined after its 1st, 2nd, 3rd+ consecutive
+* failure. Escalating rather than fixed, and it starts at HALF AN HOUR: a
+* client watchdog cycles in seconds, so any first rung measured in minutes is
+* re-paid by the very churn this exists to stop. The last rung makes a
+* permanently-broken backend cost ~nothing while still being re-probed.
+*/
+var DEAD_BACKEND_QUARANTINE_MS = [
+	18e5,
+	72e5,
+	216e5
+];
+/**
+* The decoder's answer, overruled while the backend it names is quarantined.
+*
+* Composed rather than folded into `resolveEgressDecodeHwAccel`: that function
+* answers "what SHOULD decode this", which is a per-node property of the
+* hardware; this answers "what has been observed to work in THIS process",
+* which is a property of the runner. Keeping them apart is what stops a
+* runner-local ENOMEM from being mistaken for a node that has no accelerator —
+* the first is a flow bug to fix, the second a capability limit to accept.
+*/
+async function resolveWithQuarantine(input) {
+	const backend = await input.resolve();
+	if (backend === null || !input.dead.isQuarantined(backend)) return {
+		backend,
+		veto: null
+	};
+	return {
+		backend: null,
+		veto: {
+			backend,
+			failures: input.dead.failuresOf(backend),
+			remainingMs: input.dead.remainingMs(backend)
+		}
+	};
+}
+function createDeadDecodeBackends(options = {}) {
+	const now = options.now ?? (() => Date.now());
+	const records = /* @__PURE__ */ new Map();
+	const quarantineFor = (failures) => {
+		return DEAD_BACKEND_QUARANTINE_MS[Math.min(failures, DEAD_BACKEND_QUARANTINE_MS.length) - 1] ?? DEAD_BACKEND_QUARANTINE_MS[0] ?? 0;
+	};
+	return {
+		markDead(backend) {
+			const failures = (records.get(backend)?.failures ?? 0) + 1;
+			const quarantineMs = quarantineFor(failures);
+			records.set(backend, {
+				failures,
+				until: now() + quarantineMs
+			});
+			return {
+				backend,
+				failures,
+				quarantineMs
+			};
+		},
+		markAlive(backend) {
+			records.delete(backend);
+		},
+		isQuarantined(backend) {
+			const record = records.get(backend);
+			return record !== void 0 && now() < record.until;
+		},
+		remainingMs(backend) {
+			const record = records.get(backend);
+			if (record === void 0) return 0;
+			return Math.max(0, record.until - now());
+		},
+		failuresOf(backend) {
+			return records.get(backend)?.failures ?? 0;
+		}
+	};
+}
+//#endregion
 //#region src/stream-broker/stream-broker/ffmpeg-invocation.ts
 /** The full set of hwaccel settings (for validation + option surfaces). */
 var FFMPEG_HW_ACCEL_SETTINGS = [
@@ -4360,6 +4501,7 @@ var EgressTranscodeManager = class {
 		}
 		const actualHwAccel = child.decodeHwAccel();
 		if (decodeHwAccel !== null && !require_dist.isSoftwareDecode(decodeHwAccel) && actualHwAccel === null) this.opts.onDecodeHwAccelFailed?.(decodeHwAccel);
+		else if (actualHwAccel !== null && !require_dist.isSoftwareDecode(actualHwAccel)) this.opts.onDecodeHwAccelSucceeded?.(actualHwAccel);
 		const entry = {
 			key,
 			deviceId,
@@ -13316,6 +13458,15 @@ var StreamBrokerManager = class StreamBrokerManager {
 	*/
 	egressHwAccelCache = require_dist.createHwAccelCache({ ttlMs: 6e4 });
 	/**
+	* Backends that could not create a device in THIS process. Separate from the
+	* memo above on purpose: the memo answers "what should decode this" on a read
+	* cadence and expires in a minute, so folding the failure into it meant the
+	* next cold child one minute later re-paid the same ~3 s death — measured
+	* seven times on 615 between 22:17 and 22:52 on 2026-08-07. See
+	* `dead-decode-backends.ts`.
+	*/
+	deadDecodeBackends = createDeadDecodeBackends();
+	/**
 	* Snapshot of the system `ffmpeg` config section (binaryPath / hwAccel /
 	* threadCount). Refreshed by the host addon via `setFfmpegConfig`; the
 	* transcode manager reads it live per spawn. Defaults are safe (PATH ffmpeg,
@@ -13515,8 +13666,17 @@ var StreamBrokerManager = class StreamBrokerManager {
 					};
 				},
 				onDecodeHwAccelFailed: (backend) => {
-					this.egressHwAccelCache.write(null);
-					this.logger.warn("egress transcode: decode backend FAILED in this process — software until the memo expires", { meta: { backend } });
+					const verdict = this.deadDecodeBackends.markDead(backend);
+					this.logger.warn("egress transcode: decode backend FAILED in this process — QUARANTINED, software until it is re-probed", { meta: {
+						backend,
+						failures: verdict.failures,
+						quarantineMs: verdict.quarantineMs
+					} });
+				},
+				onDecodeHwAccelSucceeded: (backend) => {
+					if (this.deadDecodeBackends.failuresOf(backend) === 0) return;
+					this.deadDecodeBackends.markAlive(backend);
+					this.logger.info("egress transcode: decode backend RECOVERED in this process — the quarantine is lifted", { meta: { backend } });
 				}
 			});
 		}
@@ -13539,6 +13699,22 @@ var StreamBrokerManager = class StreamBrokerManager {
 	* hub).
 	*/
 	async resolveEgressHwAccel(deviceId, override) {
+		const resolved = await resolveWithQuarantine({
+			resolve: () => this.readEgressHwAccel(deviceId, override),
+			dead: this.deadDecodeBackends
+		});
+		if (resolved.veto !== null) this.logger.warn("egress transcode: decoding in SOFTWARE — the backend is QUARANTINED after failing in this process", {
+			tags: { deviceId },
+			meta: {
+				backend: resolved.veto.backend,
+				failures: resolved.veto.failures,
+				remainingMs: resolved.veto.remainingMs
+			}
+		});
+		return resolved.backend;
+	}
+	/** The decoder addon's answer for this node — before the process quarantine. */
+	async readEgressHwAccel(deviceId, override) {
 		const decoderApi = this.buildDecoderCapApi(deviceId);
 		return require_dist.resolveEgressDecodeHwAccel({
 			readDecoderBackend: async () => {
@@ -18247,6 +18423,17 @@ var AdaptiveSession = class AdaptiveSession {
 	acquireTranscodeEgress;
 	/** The acquired egress, while this session holds a reference to it. */
 	transcodeEgressLeg = null;
+	/**
+	* An acquire started at the ANSWER and not yet consumed by the feed.
+	*
+	* Read-and-null-out is the whole ownership protocol: whichever of
+	* `startTranscodeFeed` and `releaseTranscodeEgress` reads a non-null value
+	* owns the release, and since both read it synchronously exactly one can.
+	* Without that, a session closing while its prewarm is in flight either
+	* double-releases (the manager logs it as a consumer bookkeeping bug) or
+	* leaks a child the primitive has no TTL to reclaim.
+	*/
+	transcodeEgressPending = null;
 	/** Unsubscribe from the acquired egress's video plane. */
 	detachTranscodeEgress = null;
 	/** Codec actually negotiated with the browser after SDP answer. */
@@ -18886,6 +19073,7 @@ var AdaptiveSession = class AdaptiveSession {
 			negotiated: this.negotiatedCodec,
 			needsTranscode: this.needsTranscode
 		} });
+		this.prewarmTranscodeEgress();
 		if (this.debug) this.logger.info("Codec negotiation detail", { meta: {
 			phase: "session",
 			sessionId: this.sessionId,
@@ -19084,6 +19272,7 @@ var AdaptiveSession = class AdaptiveSession {
 			setRemoteMs,
 			createAnswerMs
 		} });
+		this.prewarmTranscodeEgress();
 		if (this.debug) this.logger.info("Codec negotiation detail (client-offer)", { meta: {
 			phase: "session",
 			sessionId: this.sessionId,
@@ -19804,6 +19993,51 @@ var AdaptiveSession = class AdaptiveSession {
 	* unchanged — the request asks for `audio: 'passthrough'` precisely so the
 	* child does not encode a plane this session never reads.
 	*/
+	/**
+	* Start the egress acquire NOW, in parallel with ICE and DTLS.
+	*
+	* `needsTranscode` is final the moment the answer's codec is known — the
+	* "Codec negotiated" line already prints it — but the acquire used to happen
+	* inside `startFeedingFrames`, which runs only after the DTLS wait. On camera
+	* 615 that put the leg ~2 s after session creation and the first packet ~5 s
+	* after that, against a browser watchdog that re-offers every ~5 s: the viewer
+	* was always gone before the media existed, and the next session paid for a
+	* cold child again. Three minutes of that on 2026-08-07 22:17–22:20.
+	*
+	* Only the ffmpeg moves. The ATTACH stays behind DTLS, because `sendRtp`
+	* silently drops before the transport is connected.
+	*
+	* Idempotent and safe to call from any negotiation path; a session that needs
+	* no transcode does nothing. Public for the same reason
+	* {@link armSourceSilenceWatchdog} is — the acquire/release contract is
+	* testable without standing up werift.
+	*/
+	prewarmTranscodeEgress() {
+		if (this.closed) return;
+		if (!this.needsTranscode) return;
+		if (this.transcodeEgressPending !== null || this.transcodeEgressLeg !== null) return;
+		const acquire = this.acquireTranscodeEgress;
+		if (acquire === null) return;
+		this.logger.info("Transcode leg: prewarming the egress while ICE/DTLS negotiate", { meta: {
+			phase: "session",
+			sessionId: this.sessionId,
+			deviceId: this.deviceId,
+			sourceCodec: this.sourceCodec
+		} });
+		this.transcodeEgressPending = acquire({
+			profile: this.transcodeProfile,
+			sessionId: this.sessionId
+		});
+	}
+	/**
+	* Take ownership of the in-flight prewarm, if there is one. Nulls the field
+	* synchronously so the release path can never also claim it — see the field.
+	*/
+	claimPrewarmedEgress() {
+		const pending = this.transcodeEgressPending;
+		this.transcodeEgressPending = null;
+		return pending;
+	}
 	startTranscodeFeed(signal) {
 		const acquire = this.acquireTranscodeEgress;
 		if (acquire === null) {
@@ -19815,20 +20049,32 @@ var AdaptiveSession = class AdaptiveSession {
 			if (!this.closed) this.close();
 			return;
 		}
+		const prewarmed = this.claimPrewarmedEgress();
 		this.logger.info(`Starting ${this.sourceCodec}→H.264 Baseline transcode leg`, { meta: {
 			phase: "session",
 			sessionId: this.sessionId,
 			deviceId: this.deviceId,
 			sourceCodec: this.sourceCodec,
-			mode: "egress-primitive"
+			mode: "egress-primitive",
+			prewarmed: prewarmed !== null
 		} });
+		const acquireNow = () => acquire({
+			profile: this.transcodeProfile,
+			sessionId: this.sessionId
+		});
 		(async () => {
 			let leg;
 			try {
-				leg = await acquire({
-					profile: this.transcodeProfile,
-					sessionId: this.sessionId
-				});
+				leg = await (prewarmed === null ? acquireNow() : prewarmed.catch((err) => {
+					if (signal.aborted || this.closed) throw err;
+					this.logger.warn("Transcode leg: the prewarmed acquire failed — retrying now that the feed has started", { meta: {
+						phase: "session",
+						sessionId: this.sessionId,
+						deviceId: this.deviceId,
+						error: require_dist.errMsg(err)
+					} });
+					return acquireNow();
+				}));
 			} catch (err) {
 				this.logger.error("Transcode feed: acquiring the egress transcode FAILED", { meta: {
 					phase: "session",
@@ -19896,6 +20142,14 @@ var AdaptiveSession = class AdaptiveSession {
 		try {
 			detach?.();
 		} catch {}
+		const orphan = this.claimPrewarmedEgress();
+		if (orphan !== null) orphan.then((prewarmedLeg) => prewarmedLeg.release()).catch((err) => {
+			this.logger.debug("Transcode leg: the unclaimed prewarm ended without a handle", { meta: {
+				phase: "session",
+				sessionId: this.sessionId,
+				error: require_dist.errMsg(err)
+			} });
+		});
 		const leg = this.transcodeEgressLeg;
 		this.transcodeEgressLeg = null;
 		if (!leg) return;