@camstack/addon-pipeline 1.2.90 → 1.2.94

package/dist/pipeline-runner/index.mjs

@@ -1,6 +1,6 @@
-import { At as number, Et as boolean, F as deriveDetailCropRect, Ft as EventCategory, K as pickDetailCropConvention, N as defineCustomActions, Nt as string, Ot as lazy, Tt as array, Y as pipelineRunnerCapability, c as DEFAULT_DETAIL_CROP_CONVENTION, d as DEVICE_BACKEND_TO_FORMAT, it as errMsg, j as customAction, jt as object, kt as literal, mt as createEvent, q as pickNativeLeaseOverride, ut as BaseAddon, wt as _enum, yt as nodePin } from "../dist-BJYwWEZ9.mjs";
-import { c as getStepDefinition, t as attributeStall } from "../event-loop-stall-monitor-BiPffk0D.mjs";
-import { a as nativeLeaseSettingEnv, t as isWorkerReply } from "../worker-protocol-XZ1e36rM.mjs";
+import { At as literal, Dt as boolean, Et as array, F as deriveDetailCropRect, It as EventCategory, K as pickDetailCropConvention, Mt as object, N as defineCustomActions, Pt as string, Tt as _enum, Y as pipelineRunnerCapability, at as errMsg, bt as nodePin, c as DEFAULT_DETAIL_CROP_CONVENTION, d as DEVICE_BACKEND_TO_FORMAT, dt as BaseAddon, ht as createEvent, j as customAction, jt as number, kt as lazy, q as pickNativeLeaseOverride } from "../dist-XbrYiMV3.mjs";
+import { c as getStepDefinition, t as attributeStall } from "../event-loop-stall-monitor-CFWrOZ6G.mjs";
+import { a as nativeLeaseSettingEnv, t as isWorkerReply } from "../worker-protocol-D0ewr67U.mjs";
 import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
 import { t as getSharp } from "../lazy-sharp-6oymT_yf.mjs";
 import { i as withDetectionIntent, n as profileForStreamId, t as isRemoteRestream } from "../remote-restream-Ci7RXNGb.mjs";
@@ -128,6 +128,8 @@ function startGuardedSessionDecode(deps) {
 	let stopPump = null;
 	let release = null;
 	let retryTimer = null;
+	/** The live-worker slot held by the RUNNING session, released on teardown. */
+	let workerLease = null;
 	const delays = deps.acquireRetryDelaysMs && deps.acquireRetryDelaysMs.length > 0 ? deps.acquireRetryDelaysMs : DEFAULT_ACQUIRE_RETRY_DELAYS_MS;
 	/** Resolves after the backoff for `attempt`, or immediately once aborted. */
 	const waitBackoff = (attempt) => new Promise((resolve) => {
@@ -140,10 +142,19 @@ function startGuardedSessionDecode(deps) {
 	});
 	(async () => {
 		for (let attempt = 0; !aborted; attempt++) {
+			let lease = null;
+			if (deps.admitWorker) {
+				lease = deps.admitWorker();
+				if (!lease) {
+					await waitBackoff(attempt);
+					continue;
+				}
+			}
 			let acquired = null;
 			try {
 				acquired = await deps.acquire();
 			} catch (err) {
+				lease?.release();
 				(attempt === 0 ? deps.logger.warn : deps.logger.debug).call(deps.logger, "session-decode: acquire failed — retrying", { meta: {
 					attempt: attempt + 1,
 					error: errMsg(err)
@@ -152,16 +163,19 @@ function startGuardedSessionDecode(deps) {
 				continue;
 			}
 			if (aborted) {
+				lease?.release();
 				if (acquired) acquired.release();
 				return;
 			}
 			if (!acquired) {
+				lease?.release();
 				(attempt === 0 ? deps.logger.warn : deps.logger.debug).call(deps.logger, "session-decode: no broker profile bound yet — retrying", { meta: { attempt: attempt + 1 } });
 				await waitBackoff(attempt);
 				continue;
 			}
 			if (attempt > 0) deps.logger.info("session-decode: acquire succeeded after retry", { meta: { attempts: attempt + 1 } });
 			release = acquired.release;
+			workerLease = lease;
 			stopPump = deps.startPump(acquired.source);
 			return;
 		}
@@ -175,6 +189,8 @@ function startGuardedSessionDecode(deps) {
 			retryTimer = null;
 		}
 		stopPump?.();
+		workerLease?.release();
+		workerLease = null;
 		if (release) release();
 	};
 }
@@ -1387,7 +1403,9 @@ var RunSyntheticBenchInputSchema = object({
 	sessionId: string().optional(),
 	simulatePipeline: boolean().optional(),
 	nodeId: string().optional(),
-	engine: BenchEngineChoiceSchema.optional()
+	engine: BenchEngineChoiceSchema.optional(),
+	deviceKey: string().optional(),
+	plane: _enum(["full", "frame"]).optional()
 });
 var TimingSplitSchema = object({
 	mean: number(),
@@ -1442,6 +1460,48 @@ var pipelineRunnerBenchActions = defineCustomActions({
 	abortCapacityRamp: customAction(AbortCapacityRampInputSchema, AbortCapacityRampResultSchema, { kind: "mutation" })
 });
 //#endregion
+//#region src/pipeline-runner/bench-dispatch.ts
+/**
+* Why the cached fast path is unavailable, or `null` if nothing blocks it.
+*
+* The fast path is `cacheFrameInPool` once + `inferCached` per call, and
+* NEITHER cap method takes a `deviceKey` or a `plane` — only `runPipeline`
+* does. Forwarding a device selector into it would scope the WARMUP
+* `runPipeline` to the chosen accelerator and leave every TIMED call on the
+* node's default pool: a measurement of the wrong device, reported as the
+* right one. Refusing the fast path costs a slower bench and buys a true
+* number.
+*/
+function fastPathRefusalReason(input) {
+	if (input.deviceKey) return `deviceKey '${input.deviceKey}' requested — inferCached has no device selector, so the timed calls would land on the node's default pool; using the runPipeline path instead`;
+	if (input.plane) return `plane '${input.plane}' requested — inferCached has no plane selector; using the runPipeline path instead`;
+	return null;
+}
+/**
+* The cached fast path is only for a plain, single-step, default-device,
+* default-plane inference bench.
+*/
+function canUseCachedFastPath(isSingleStep, input) {
+	return isSingleStep && !input.simulatePipeline && fastPathRefusalReason(input) === null;
+}
+/**
+* Build the `pipelineExecutor.runPipeline` argument for one bench call.
+*
+* Every optional is ABSENT when unset rather than present-and-undefined: an
+* omitted `deviceKey` means "the runner's default device" and an omitted
+* `plane` means the cap's `'full'` default, and the two are not the same as
+* an explicit `undefined` on a `.loose()` router input.
+*/
+function buildBenchRunPipelineInput(req) {
+	return {
+		steps: [...req.steps],
+		frame: req.frame,
+		...req.engine ? { engine: req.engine } : {},
+		...req.deviceKey ? { deviceKey: req.deviceKey } : {},
+		...req.plane ? { plane: req.plane } : {}
+	};
+}
+//#endregion
 //#region src/pipeline-runner/capacity/capacity-runner.ts
 /**
 * Run ONE ramp step: `cameras` independent open-loop pacers for
@@ -3863,6 +3923,76 @@ var PipelineRunner = class {
 	}
 };
 //#endregion
+//#region src/session-decode/live-worker-admission.ts
+/**
+* The node's live-decode-worker budget.
+*
+* One instance per runner addon. Not a semaphore: a semaphore's contract is
+* "wait", and waiting is the thing this deliberately does not do — see
+* {@link DEFAULT_MAX_LIVE_DECODE_WORKERS}.
+*/
+var LiveDecodeWorkerAdmission = class {
+	limit;
+	logger;
+	live = 0;
+	constructor(options) {
+		this.limit = Math.max(1, options.limit);
+		this.logger = options.logger;
+	}
+	/** Workers currently holding a slot. */
+	get inFlight() {
+		return this.live;
+	}
+	get ceiling() {
+		return this.limit;
+	}
+	/**
+	* Apply an operator-changed ceiling.
+	*
+	* Lowering it never kills a running session — a decode worker torn down
+	* mid-motion loses the event it was started for. It only stops the next
+	* admission, so the count drains to the new ceiling as sessions end.
+	*/
+	resize(limit) {
+		this.limit = Math.max(1, limit);
+	}
+	/**
+	* Take a slot, or refuse.
+	*
+	* A refusal is a WARN and carries `tags.deviceId`, because the only useful
+	* form of this question is per-camera ("why did 617 not run when 615 did?").
+	* A branch that drops work silently reads as "it never happened".
+	*/
+	admit(deviceId, label) {
+		if (this.live >= this.limit) {
+			this.logger.warn("session-decode: REFUSED — node live decode-worker ceiling reached, will retry", {
+				tags: { deviceId },
+				meta: {
+					label,
+					live: this.live,
+					ceiling: this.limit
+				}
+			});
+			return null;
+		}
+		this.live += 1;
+		this.logger.debug("session-decode: live decode-worker slot taken", {
+			tags: { deviceId },
+			meta: {
+				label,
+				live: this.live,
+				ceiling: this.limit
+			}
+		});
+		let released = false;
+		return { release: () => {
+			if (released) return;
+			released = true;
+			this.live -= 1;
+		} };
+	}
+};
+//#endregion
 //#region src/pipeline-runner/stateless-step.ts
 function modelServability(step, modelId, format) {
 	const entry = step.models.find((m) => m.id === modelId);
@@ -4001,7 +4131,8 @@ var DEFAULT_CONFIG = {
 	maxConcurrentInferences: 16,
 	targetLoadPercent: 80,
 	minThrottledFps: 1,
-	maxConcurrentDecodeSpawns: 3
+	maxConcurrentDecodeSpawns: 3,
+	maxLiveDecodeWorkers: 8
 };
 /**
 * Camera attachment record kept by the runner addon. Holds the original
@@ -4304,6 +4435,14 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
 	*/
 	spawnGate = null;
 	/**
+	* Per-node ceiling on decode workers ALIVE at once, sized to
+	* `config.maxLiveDecodeWorkers`. The spawn gate above bounds how fast workers
+	* START; this bounds how many EXIST — measured live at 1 -> 8 workers and a
+	* ~4 GiB move in 8 seconds, with nothing stopping the 9th. `resize()`d live in
+	* `onConfigChanged`. Null until `onInitialize`.
+	*/
+	liveWorkers = null;
+	/**
 	* Node-level map from a retention-ring `FrameHandle` to the decode worker
 	* still holding that frame's NATIVE surface — backs the `getNativeCrop` cap
 	* so post-analysis can cut native-res crops (best-effort; a miss falls back
@@ -4527,6 +4666,10 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
 			}
 		});
 		this.spawnGate = new Semaphore(Math.max(1, this.config.maxConcurrentDecodeSpawns));
+		this.liveWorkers = new LiveDecodeWorkerAdmission({
+			limit: this.config.maxLiveDecodeWorkers,
+			logger: this.ctx.logger
+		});
 		this.runner.timingSampler.setLogger(this.ctx.logger.child("timing"));
 		this.runner.onDetectionStreamChange((deviceId, action) => {
 			this.handleDetectionStreamChange(deviceId, action);
@@ -4687,8 +4830,14 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
 			children: s.children ?? []
 		}));
 		const enabledSteps = stepsToRun.filter((s) => s.enabled);
-		const useFastPath = enabledSteps.length === 1 && (!enabledSteps[0].children || enabledSteps[0].children.filter((c) => c.enabled).length === 0) && !input.simulatePipeline;
+		const isSingleStep = enabledSteps.length === 1 && (!enabledSteps[0].children || enabledSteps[0].children.filter((c) => c.enabled).length === 0);
+		const useFastPath = canUseCachedFastPath(isSingleStep, input);
 		const rootStep = enabledSteps[0];
+		const fastPathRefusal = isSingleStep ? fastPathRefusalReason(input) : null;
+		if (fastPathRefusal) ctx.logger.info("synthetic bench: cached fast path declined", { meta: {
+			reason: fastPathRefusal,
+			node: benchNode
+		} });
 		const sharedFrame = {
 			data: cached.data,
 			format: cached.format,
@@ -4708,11 +4857,11 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
 				height: cached.height,
 				format: cached.format
 			}, pin)).frameId;
-			await api.pipelineExecutor.runPipeline.mutate({
+			await api.pipelineExecutor.runPipeline.mutate({ ...buildBenchRunPipelineInput({
 				steps: stepsToRun,
 				frame: sharedFrame,
 				...input.engine ? { engine: input.engine } : {}
-			}, pin);
+			}) }, pin);
 			const warmupCount = input.warmup ?? 1;
 			for (let w = 0; w < warmupCount; w++) await api.pipelineExecutor.inferCached.mutate({
 				stepId: rootStep.addonId,
@@ -4805,16 +4954,20 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
 		}
 		ctx.logger.info("synthetic bench: using full runPipeline path", { meta: {
 			steps: enabledSteps.length,
-			simulatePipeline: !!input.simulatePipeline
+			simulatePipeline: !!input.simulatePipeline,
+			deviceKey: input.deviceKey ?? null,
+			plane: input.plane ?? null
 		} });
 		let _callCount = 0;
 		const callOnce = async () => {
 			const t0 = performance.now();
-			const result = await api.pipelineExecutor.runPipeline.mutate({
+			const result = await api.pipelineExecutor.runPipeline.mutate({ ...buildBenchRunPipelineInput({
 				steps: stepsToRun,
 				frame: sharedFrame,
-				...input.engine ? { engine: input.engine } : {}
-			}, pin);
+				...input.engine ? { engine: input.engine } : {},
+				...input.deviceKey ? { deviceKey: input.deviceKey } : {},
+				...input.plane ? { plane: input.plane } : {}
+			}) }, pin);
 			const wallMs = performance.now() - t0;
 			const n = ++_callCount;
 			if (n <= 20) ctx.logger.info("bench call trace", { meta: {
@@ -6191,6 +6344,18 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
 	* blocked by a missing gate. `label` distinguishes detect vs motion forks in
 	* the logs; the device id is already tagged on `log`.
 	*/
+	/**
+	* The node's live-decode-worker admission, bound to one camera.
+	*
+	* `undefined` when the ceiling is not configured (pre-`onInitialize`), which
+	* `startGuardedSessionDecode` reads as "no ceiling" — the historical
+	* behaviour. Never a queue: see {@link LiveDecodeWorkerAdmission}.
+	*/
+	buildLiveWorkerAdmit(deviceId, label) {
+		const admission = this.liveWorkers;
+		if (!admission) return void 0;
+		return () => admission.admit(deviceId, label);
+	}
 	buildDecodeSpawnAcquire(log, label) {
 		return async () => {
 			const gate = this.spawnGate;
@@ -6218,6 +6383,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
 		const requestedAt = Date.now();
 		const teardown = startGuardedSessionDecode({
 			logger: log,
+			admitWorker: this.buildLiveWorkerAdmit(config.deviceId, "detect"),
 			acquire: () => this.acquireSessionDecodeRestream(api, config.deviceId, config.detectionStreamId, `session-decode:detect:${config.deviceId}`, resolveSessionDecodeHostname(config, process.env["CAMSTACK_HUB_URL"]), resolveSessionDecodeOwnerNodeId(config), true),
 			startPump: (source) => {
 				const dialMs = Date.now() - requestedAt;
@@ -6290,6 +6456,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
 		if (!runner) return () => {};
 		return startGuardedSessionDecode({
 			logger: log,
+			admitWorker: this.buildLiveWorkerAdmit(config.deviceId, "motion"),
 			acquire: () => this.acquireSessionDecodeRestream(api, config.deviceId, config.motionStreamId, `session-decode:motion:${config.deviceId}`, resolveSessionDecodeHostname(config, process.env["CAMSTACK_HUB_URL"]), resolveSessionDecodeOwnerNodeId(config)),
 			startPump: (source) => {
 				log.info("session-decode: motion routed through per-session decode worker (gray)", { meta: {
@@ -6846,6 +7013,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
 		}, OCCUPANCY_BURST_SAFETY_TIMEOUT_MS);
 		teardown = startGuardedSessionDecode({
 			logger: log,
+			admitWorker: this.buildLiveWorkerAdmit(deviceId, "occupancy"),
 			acquire: () => this.acquireSessionDecodeRestream(api, config.deviceId, config.detectionStreamId, `session-decode:occupancy:${config.deviceId}`, resolveSessionDecodeHostname(config, process.env["CAMSTACK_HUB_URL"]), resolveSessionDecodeOwnerNodeId(config)),
 			startPump: (source) => startSessionDecodePump({
 				fork: this.forkDecodeWorkerWithLease,
@@ -7019,6 +7187,17 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
 					step: 1,
 					default: DEFAULT_CONFIG.maxConcurrentDecodeSpawns,
 					showValue: true
+				},
+				{
+					type: "slider",
+					key: "maxLiveDecodeWorkers",
+					label: "Max live decode workers",
+					description: "Hard ceiling on decode workers ALIVE at once on this node (~320 MB of RAM each). Unlike the spawn gate above, which only limits how fast workers START, this limits how many EXIST. A camera that does not fit is REFUSED and logged, then retried automatically as soon as a slot frees — it is never queued, because a queued live session would decode a moment that has already passed. Raise it if the node has RAM to spare; lower it if the host is memory-constrained.",
+					min: 1,
+					max: 32,
+					step: 1,
+					default: DEFAULT_CONFIG.maxLiveDecodeWorkers,
+					showValue: true
 				}
 			]
 		}] });
@@ -7026,12 +7205,14 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
 	async onConfigChanged() {
 		this.runner?.updateLimits(this.config);
 		this.spawnGate?.resize(Math.max(1, this.config.maxConcurrentDecodeSpawns));
+		this.liveWorkers?.resize(this.config.maxLiveDecodeWorkers);
 		this.ctx.logger.info("pipeline-runner tuning updated", { meta: {
 			maxQueueDepth: this.config.maxQueueDepth,
 			maxConcurrentInferences: this.config.maxConcurrentInferences,
 			targetLoadPercent: this.config.targetLoadPercent,
 			minThrottledFps: this.config.minThrottledFps,
-			maxConcurrentDecodeSpawns: this.config.maxConcurrentDecodeSpawns
+			maxConcurrentDecodeSpawns: this.config.maxConcurrentDecodeSpawns,
+			maxLiveDecodeWorkers: this.config.maxLiveDecodeWorkers
 		} });
 	}
 };