@camstack/addon-pipeline 1.2.19 → 1.2.28

package/dist/pipeline-runner/index.mjs

@@ -1,8 +1,8 @@
-import { B as errMsg, G as createEvent, M as pipelineRunnerCapability, S as defineCustomActions, V as BaseAddon, X as nodePin, at as lazy, b as customAction, ct as object, ft as EventCategory, nt as array, rt as boolean, s as DEVICE_BACKEND_TO_FORMAT, st as number, tt as _enum, ut as string } from "../dist-DZBePRjP.mjs";
+import { B as errMsg, G as createEvent, M as pipelineRunnerCapability, S as defineCustomActions, V as BaseAddon, X as nodePin, at as lazy, b as customAction, ct as object, ft as EventCategory, nt as array, rt as boolean, s as DEVICE_BACKEND_TO_FORMAT, st as number, tt as _enum, ut as string } from "../dist-CsKaXmP8.mjs";
 import { n as profileForStreamId, t as isRemoteRestream } from "../remote-restream-BeHi78PZ.mjs";
 import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
 import { t as isWorkerReply } from "../worker-protocol-D7RzZIla.mjs";
-import { a as getStepDefinition } from "../step-definitions-BwB6TBMB.mjs";
+import { a as getStepDefinition } from "../step-definitions-C8A78dBi.mjs";
 import { fork } from "node:child_process";
 import { fileURLToPath } from "node:url";
 import sharp from "sharp";
@@ -569,6 +569,22 @@ var PipelineTimingSampler = class {
 //#endregion
 //#region src/pipeline-runner/runner.ts
 /**
+* Bound on one `processFrame` call. Chosen well above the slowest legitimate
+* inference seen in production (8 617 ms, GPU contention on the hub) so the
+* timeout only ever fires on a call that is genuinely never coming back.
+*/
+var DEFAULT_INFERENCE_TIMEOUT_MS = 2e4;
+/** Grace period before a permit-starved scheduler with queued work warns. */
+var DEFAULT_INFERENCE_STALL_WARN_MS = 6e4;
+/** Re-warn cadence while a stall persists, so a wedge stays visible. */
+var INFERENCE_STALL_REWARN_MS = 6e4;
+/**
+* Sentinel distinguishing "the bound expired" from a legitimate `null`
+* (`processFrame` returning no result). A unique symbol, so it can never
+* collide with a value the executor produces.
+*/
+var TIMED_OUT = Symbol("inference-timed-out");
+/**
 * Clone a {@link DetectionQueueEntry} for the queue's copy-on-retain
 * boundary: the inner frame's pixel buffer is detached via
 * {@link ownFrame}, the `FrameHandle` is carried through unchanged (it's
@@ -651,6 +667,10 @@ var PipelineRunner = class {
 	arrivalCounts = /* @__PURE__ */ new Map();
 	lastGovernorRecomputeAt = Date.now();
 	governorIntervalHandle = null;
+	/** When the scheduler first found itself with zero permits; null when healthy. */
+	inferenceStallSince = null;
+	/** Last stall warning, so a persistent wedge re-warns instead of going quiet. */
+	inferenceStallWarnedAt = null;
 	constructor(config) {
 		this.config = config;
 		this.logger = config.logger;
@@ -1041,7 +1061,12 @@ var PipelineRunner = class {
 	}
 	tick() {
 		this.drainMotionQueues();
-		if (this.semaphore.available <= 0) return;
+		if (this.semaphore.available <= 0) {
+			this.noteInferenceStall();
+			return;
+		}
+		this.inferenceStallSince = null;
+		this.inferenceStallWarnedAt = null;
 		const picked = this.pickNextDetectionFrame();
 		if (!picked) return;
 		const { deviceId, entry, state } = picked;
@@ -1064,11 +1089,18 @@ var PipelineRunner = class {
 		const pickedAt = Date.now();
 		const { frame, handle } = entry;
 		const enqueuedAt = frame._enqueuedAt ?? pickedAt;
-		const release = await this.semaphore.acquire();
+		const rawRelease = await this.semaphore.acquire();
+		let released = false;
+		const release = () => {
+			if (released) return;
+			released = true;
+			rawRelease();
+		};
 		const semAcquiredAt = Date.now();
 		const executorHandle = state.detectionQueue.size <= HANDLE_PASSTHROUGH_MAX_QUEUE_DEPTH && this.semaphore.available > 0 ? handle : void 0;
 		try {
-			const result = await this.config.processFrame(deviceId, frameInput, executorHandle);
+			const result = await this.awaitInferenceBounded(deviceId, frameInput, executorHandle, release);
+			if (result === TIMED_OUT) return;
 			const inferDoneAt = Date.now();
 			const inferenceMs = inferDoneAt - semAcquiredAt;
 			state.inferenceTimes.push(inferenceMs);
@@ -1092,6 +1124,73 @@ var PipelineRunner = class {
 			release();
 		}
 	}
+	/**
+	* Await `processFrame` with a hard bound. On expiry the permit is released
+	* immediately (via the caller's release-once closure) and {@link TIMED_OUT}
+	* is returned; the underlying call is left to settle into the void, since
+	* the executor contract offers no cancellation.
+	*/
+	async awaitInferenceBounded(deviceId, frameInput, executorHandle, release) {
+		const timeoutMs = this.config.inferenceTimeoutMs ?? DEFAULT_INFERENCE_TIMEOUT_MS;
+		let timer = null;
+		const expiry = new Promise((resolve) => {
+			timer = setTimeout(() => resolve(TIMED_OUT), timeoutMs);
+			timer.unref?.();
+		});
+		try {
+			const outcome = await Promise.race([this.config.processFrame(deviceId, frameInput, executorHandle), expiry]);
+			if (outcome === TIMED_OUT) {
+				release();
+				this.logger?.warn("inference timed out — permit released, result discarded", {
+					tags: { deviceId },
+					meta: {
+						timeoutMs,
+						availablePermits: this.semaphore.available
+					}
+				});
+			}
+			return outcome;
+		} finally {
+			if (timer !== null) clearTimeout(timer);
+		}
+	}
+	/**
+	* Called on every tick that finds zero available permits. Warns once the
+	* starvation outlives `inferenceStallWarnMs` WITH queued work — queued work
+	* is the discriminator: a runner with no permits and no backlog is simply
+	* busy, while one with a backlog it cannot touch is wedged.
+	*/
+	noteInferenceStall() {
+		let queuedFrames = 0;
+		for (const state of this.cameras.values()) queuedFrames += state.detectionQueue.size;
+		if (queuedFrames === 0) {
+			this.inferenceStallSince = null;
+			this.inferenceStallWarnedAt = null;
+			return;
+		}
+		const now = Date.now();
+		if (this.inferenceStallSince === null) {
+			this.inferenceStallSince = now;
+			return;
+		}
+		const stalledMs = now - this.inferenceStallSince;
+		if (stalledMs < (this.config.inferenceStallWarnMs ?? DEFAULT_INFERENCE_STALL_WARN_MS)) return;
+		if (this.inferenceStallWarnedAt !== null && now - this.inferenceStallWarnedAt < INFERENCE_STALL_REWARN_MS) return;
+		this.inferenceStallWarnedAt = now;
+		for (const [deviceId, state] of this.cameras) {
+			if (state.detectionQueue.size === 0) continue;
+			this.logger?.warn("no inference permit available while frames are queued", {
+				tags: { deviceId },
+				meta: {
+					stalledMs,
+					queuedFrames: state.detectionQueue.size,
+					droppedFrames: state.detectionQueue.droppedFrames,
+					phase: state.phase,
+					concurrency: this.semaphore.concurrency
+				}
+			});
+		}
+	}
 	async notifyCallbacks(deviceId, frame, result, streamType, handle) {
 		for (const callback of this.resultCallbacks) try {
 			await callback(deviceId, frame, result, streamType, handle);
@@ -1231,7 +1330,24 @@ function forkDecodeWorker(_source) {
 //#endregion
 //#region src/session-decode/guarded-session-decode.ts
 /**
-* Guards the in-flight restream-acquire against a detach/unsubscribe race.
+* 2s → 5s → 15s → 30s, then 30s forever.
+*
+* Indefinite is deliberate. The failure this recovers from is "the broker has
+* no profile bound yet" or "the broker call failed right now" — both of which
+* clear on their own, often minutes later (a source re-dialing, a runner
+* restarting). A bounded retry would turn a slow recovery back into the
+* permanent outage this exists to prevent, and teardown already stops it: a
+* detach aborts the loop, so nothing retries for a camera nobody wants.
+*/
+var DEFAULT_ACQUIRE_RETRY_DELAYS_MS = [
+	2e3,
+	5e3,
+	15e3,
+	3e4
+];
+/**
+* Guards the in-flight restream-acquire against a detach/unsubscribe race,
+* and retries an acquire that fails.
 *
 * `acquireSessionDecodeRestream` + forking the decode worker both take real
 * time (a broker round-trip, then a process fork). If the caller's teardown
@@ -1243,29 +1359,71 @@ function forkDecodeWorker(_source) {
 *
 * This helper returns the teardown SYNCHRONOUSLY. The acquire + pump-start
 * run in the background; an `aborted` flag recorded by the (possibly
-* already-called) teardown is checked right after the acquire resolves —
-* if set, the restream is released immediately and the pump is NEVER
+* already-called) teardown is checked right after each acquire resolves —
+* if set, any restream is released immediately and the pump is NEVER
 * started (no orphan worker). Otherwise the pump starts and the teardown,
 * once called, stops the pump then releases the restream.
+*
+* **A failed acquire is retried, not swallowed.** The teardown handed back
+* here is stored by the caller (as `attachment.detectionUnsubscribe`) the
+* instant this returns, so a `return` on failure left that handle permanently
+* non-null with no pump behind it: `onEnded` could never fire because nothing
+* had ever started, the camera's phase stayed `active`, and not one frame
+* arrived again. The motion path was hardened against this shape by a frame
+* freshness window (`MOTION_FRESHNESS_WINDOW_MS`); this is the detection
+* path's half.
 */
 function startGuardedSessionDecode(deps) {
 	let aborted = false;
 	let stopPump = null;
 	let release = null;
+	let retryTimer = null;
+	const delays = deps.acquireRetryDelaysMs && deps.acquireRetryDelaysMs.length > 0 ? deps.acquireRetryDelaysMs : DEFAULT_ACQUIRE_RETRY_DELAYS_MS;
+	/** Resolves after the backoff for `attempt`, or immediately once aborted. */
+	const waitBackoff = (attempt) => new Promise((resolve) => {
+		const ms = delays[Math.min(attempt, delays.length - 1)] ?? delays[delays.length - 1] ?? 2e3;
+		retryTimer = setTimeout(() => {
+			retryTimer = null;
+			resolve();
+		}, ms);
+		retryTimer.unref?.();
+	});
 	(async () => {
-		const acquired = await deps.acquire();
-		if (!acquired) return;
-		if (aborted) {
-			acquired.release();
+		for (let attempt = 0; !aborted; attempt++) {
+			let acquired = null;
+			try {
+				acquired = await deps.acquire();
+			} catch (err) {
+				(attempt === 0 ? deps.logger.warn : deps.logger.debug).call(deps.logger, "session-decode: acquire failed — retrying", { meta: {
+					attempt: attempt + 1,
+					error: errMsg(err)
+				} });
+				await waitBackoff(attempt);
+				continue;
+			}
+			if (aborted) {
+				if (acquired) acquired.release();
+				return;
+			}
+			if (!acquired) {
+				(attempt === 0 ? deps.logger.warn : deps.logger.debug).call(deps.logger, "session-decode: no broker profile bound yet — retrying", { meta: { attempt: attempt + 1 } });
+				await waitBackoff(attempt);
+				continue;
+			}
+			if (attempt > 0) deps.logger.info("session-decode: acquire succeeded after retry", { meta: { attempts: attempt + 1 } });
+			release = acquired.release;
+			stopPump = deps.startPump(acquired.source);
 			return;
 		}
-		release = acquired.release;
-		stopPump = deps.startPump(acquired.source);
 	})().catch((err) => {
 		deps.logger.warn("session-decode: guarded start failed", { meta: { error: errMsg(err) } });
 	});
 	return () => {
 		aborted = true;
+		if (retryTimer !== null) {
+			clearTimeout(retryTimer);
+			retryTimer = null;
+		}
 		stopPump?.();
 		if (release) release();
 	};
@@ -2057,10 +2215,14 @@ var NativeCropRegistry = class {
 	* Bind `request` to `handle`; evicts the oldest entry past the cap, releasing
 	* its worker-side RAM lease so a lease never outlives its registry entry.
 	*/
-	register(handle, request) {
+	register(handle, request, deviceId) {
 		const k = this.key(handle);
 		this.entries.delete(k);
-		this.entries.set(k, request);
+		this.entries.set(k, {
+			request,
+			deviceId,
+			registeredAt: Date.now()
+		});
 		while (this.entries.size > this.capacity) {
 			const oldest = this.entries.keys().next().value;
 			if (oldest === void 0) break;
@@ -2069,7 +2231,22 @@ var NativeCropRegistry = class {
 	}
 	/** The requester for `handle`, or `null` if never registered / already evicted. */
 	get(handle) {
-		return this.entries.get(this.key(handle)) ?? null;
+		const entry = this.entries.get(this.key(handle));
+		if (entry === void 0) return null;
+		return {
+			request: entry.request,
+			deviceId: entry.deviceId,
+			ageMs: Date.now() - entry.registeredAt
+		};
+	}
+	/**
+	* The deviceId a handle was registered under, even AFTER its entry was
+	* evicted — kept in a small parallel ring so a `no-registry-entry` miss can
+	* still be attributed to a camera (a miss line without `deviceId` cannot
+	* answer "why is 617 worse than 615?").
+	*/
+	deviceIdFor(handle) {
+		return this.evictedDeviceIds.get(this.key(handle)) ?? this.entries.get(this.key(handle))?.deviceId ?? null;
 	}
 	/**
 	* Release the worker-side RAM native lease for `handle` and drop its entry —
@@ -2081,15 +2258,27 @@ var NativeCropRegistry = class {
 	}
 	/** Drop every entry, releasing each frame's worker-side lease (node/addon shutdown). */
 	clear() {
-		for (const request of this.entries.values()) safeRelease(request);
+		for (const entry of this.entries.values()) safeRelease(entry.request);
 		this.entries.clear();
+		this.evictedDeviceIds.clear();
 	}
-	/** Release + drop one entry by its map key. */
+	/** Release + drop one entry by its map key, remembering its camera. */
 	releaseEntry(key) {
-		const request = this.entries.get(key);
-		if (!request) return;
+		const entry = this.entries.get(key);
+		if (!entry) return;
 		this.entries.delete(key);
-		safeRelease(request);
+		this.rememberEvicted(key, entry.deviceId);
+		safeRelease(entry.request);
+	}
+	/** Bounded memory of evicted handles → deviceId, for miss attribution. */
+	evictedDeviceIds = /* @__PURE__ */ new Map();
+	rememberEvicted(key, deviceId) {
+		this.evictedDeviceIds.set(key, deviceId);
+		while (this.evictedDeviceIds.size > this.capacity) {
+			const oldest = this.evictedDeviceIds.keys().next().value;
+			if (oldest === void 0) break;
+			this.evictedDeviceIds.delete(oldest);
+		}
 	}
 };
 //#endregion
@@ -3295,16 +3484,29 @@ var PipelineRunnerAddon = class extends BaseAddon {
 		const isFullFrame = b.x <= .001 && b.y <= .001 && b.w >= .999 && b.h >= .999;
 		const isUnboundedFullFrame = isFullFrame && input.maxWidth === void 0;
 		let nativeResult = null;
+		let missReason = null;
 		if (!isUnboundedFullFrame) {
-			const requester = this.nativeCropRegistry.get(input.handle);
-			if (requester) {
-				const result = await requester(input.bbox, input.maxWidth);
+			const found = this.nativeCropRegistry.get(input.handle);
+			if (found) {
+				const result = await found.request(input.bbox, input.maxWidth);
 				if (result) nativeResult = {
 					bytes: result.bytes,
 					width: result.width,
 					height: result.height
 				};
-			}
+				else missReason = "worker-lease-gone";
+			} else missReason = "no-registry-entry";
+			if (missReason !== null) this.ctx.logger.debug("native crop miss", {
+				tags: { deviceId: found?.deviceId ?? this.nativeCropRegistry.deviceIdFor(input.handle) ?? -1 },
+				meta: {
+					reason: missReason,
+					handle: `${input.handle.shmId}#${input.handle.slot}#${input.handle.seq}`,
+					handleNodeId: input.handle.nodeId,
+					handleAgeMs: found?.ageMs ?? null,
+					registrySize: this.nativeCropRegistry.size,
+					registryCap: DEFAULT_NATIVE_CROP_REGISTRY_CAP
+				}
+			});
 		}
 		const tier = resolveNativeCropSource(isFullFrame, nativeResult !== null);
 		if (tier === "native" && nativeResult) return this.encodeNativeCropResult(nativeResult, tier, input.encodeJpeg);
@@ -3785,7 +3987,7 @@ var PipelineRunnerAddon = class extends BaseAddon {
 					onEnded: () => this.handleDetectionDecodeEnded(config.deviceId),
 					onDecodedFrame: (frame, nativeCrop) => {
 						const handle = retainedFrames?.write(config.deviceId, frame) ?? void 0;
-						if (handle && nativeCrop) this.nativeCropRegistry.register(handle, nativeCrop);
+						if (handle && nativeCrop) this.nativeCropRegistry.register(handle, nativeCrop, config.deviceId);
 						this.enqueueSharedDetectionFrame(config.deviceId, frame, handle);
 					}
 				});
@@ -4298,7 +4500,7 @@ var PipelineRunnerAddon = class extends BaseAddon {
 				onDecodedFrame: (frame, nativeCrop) => {
 					if (settled) return;
 					const handle = retainedFrames?.write(config.deviceId, frame) ?? void 0;
-					if (handle && nativeCrop) this.nativeCropRegistry.register(handle, nativeCrop);
+					if (handle && nativeCrop) this.nativeCropRegistry.register(handle, nativeCrop, config.deviceId);
 					runner.enqueueOccupancyFrame(deviceId, frame, handle);
 					count++;
 					if (count >= frames) {