@camstack/addon-pipeline 1.2.302 → 1.2.304

package/dist/pipeline-runner/index.js

@@ -41,14 +41,18 @@ var KILL_SIGNALS = {
 * parameter rather than a second `child.on('exit')` in the caller because
 * every listener added out here is one somebody can forget to remove.
 */
-function wrapChild(child, onExit) {
+function wrapChild(child, onExit, killedByRunner = () => false) {
 	let spawned = false;
 	let exitDelivered = false;
 	const exitListeners = [];
-	const deliverExit = (info) => {
+	const deliverExit = (exit) => {
 		if (exitDelivered) return;
 		exitDelivered = true;
 		onExit();
+		const info = killedByRunner() ? {
+			...exit,
+			killedByRunner: true
+		} : exit;
 		for (const listener of exitListeners) listener(info);
 	};
 	child.on("spawn", () => {
@@ -102,19 +106,179 @@ function wrapChild(child, onExit) {
 * node-av through `CAMSTACK_FRAMEWORK_DIR` / `CAMSTACK_BUNDLED_ADDONS_DIR` on
 * agents, so a replaced env would silently break decoding off the hub.
 */
-function forkDecodeWorker(source, costRegistry) {
-	const child = (0, node_child_process.fork)(CHILD_ENTRY_PATH, [], {
+function forkDecodeWorker(source, deps) {
+	return forkDecodeWorkerAt(CHILD_ENTRY_PATH, source, deps ?? {});
+}
+/**
+* {@link forkDecodeWorker} with the entry named — the one seam a process-level
+* test needs to fork a stub through the REAL fork, claim and live-worker path.
+* Production forks only {@link CHILD_ENTRY_PATH}.
+*/
+function forkDecodeWorkerAt(entryPath, source, deps) {
+	const child = (0, node_child_process.fork)(entryPath, [], {
 		serialization: "advanced",
 		env: { ...process.env }
 	});
-	const costClaim = costRegistry?.claim({
+	const costClaim = deps.costRegistry?.claim({
 		role: "decode",
 		deviceId: source.deviceId,
 		unit: `${source.deviceId}/decode`,
 		pid: child.pid
 	}) ?? require_node.NO_COST_CLAIM;
-	return wrapChild(child, () => costClaim.release());
+	let killedByRunner = false;
+	const untrack = deps.liveWorkers?.add({
+		deviceId: source.deviceId,
+		pid: child.pid,
+		killNow: () => {
+			killedByRunner = true;
+			child.kill("SIGKILL");
+		}
+	}) ?? (() => void 0);
+	return wrapChild(child, () => {
+		costClaim.release();
+		untrack();
+	}, () => killedByRunner);
+}
+//#endregion
+//#region src/session-decode/live-decode-workers.ts
+var KILL_MESSAGE = {
+	shutdown: "session-decode: worker killed at shutdown",
+	"forked-after-shutdown": "session-decode: worker forked after shutdown began — killed at once",
+	"process-exit": "session-decode: worker killed at process exit"
+};
+/** Send the signal. Never throws: a failure is the outcome. */
+function signal(worker) {
+	try {
+		worker.killNow();
+		return {
+			worker,
+			error: null
+		};
+	} catch (err) {
+		return {
+			worker,
+			error: require_dist.errMsg(err)
+		};
+	}
+}
+/**
+* One line per outcome, each in its own swallowing try. The logger may throw:
+* the `'exit'` hook runs while the transport goes down, and the runner's
+* write path has no guard. A line that fails must not cost another line.
+*/
+function logOutcomes(outcomes, reason, logger) {
+	for (const { worker, error } of outcomes) try {
+		if (error === null) logger.info(KILL_MESSAGE[reason], {
+			tags: { deviceId: worker.deviceId },
+			meta: {
+				pid: worker.pid ?? null,
+				signal: "SIGKILL",
+				reason
+			}
+		});
+		else logger.warn("session-decode: worker kill failed — it may outlive the runner", {
+			tags: { deviceId: worker.deviceId },
+			meta: {
+				pid: worker.pid ?? null,
+				reason,
+				error
+			}
+		});
+	} catch {}
 }
+var LiveDecodeWorkers = class {
+	live = /* @__PURE__ */ new Set();
+	/** Set by {@link close}: from then on a worker is killed as it is added. */
+	closedLogger = null;
+	/** Track a forked worker; call the returned function from its real exit. */
+	add(worker) {
+		if (this.closedLogger !== null) {
+			const outcome = signal(worker);
+			logOutcomes([outcome], "forked-after-shutdown", this.closedLogger);
+			if (outcome.error === null) return () => void 0;
+		}
+		this.live.add(worker);
+		return () => {
+			this.live.delete(worker);
+		};
+	}
+	get size() {
+		return this.live.size;
+	}
+	/**
+	* SIGKILL every tracked worker now, then one log line per worker. SIGKILL
+	* rather than SIGTERM: the runner is about to exit and cannot wait for a
+	* fallback, and a worker blocked in a native call never runs a SIGTERM
+	* handler (D652).
+	*
+	* EVERY worker is signalled before ANY line is written: a logger that throws
+	* must not stop the kills. A signalled worker is forgotten (a second pass
+	* logs nothing twice); one whose kill threw stays tracked, so the exit hook
+	* or a later pass tries it again. Returns how many it signalled.
+	*/
+	killAll(reason, logger) {
+		const outcomes = [...this.live].map(signal);
+		for (const outcome of outcomes) if (outcome.error === null) this.live.delete(outcome.worker);
+		logOutcomes(outcomes, reason, logger);
+		return outcomes.filter((outcome) => outcome.error === null).length;
+	}
+	/** Kill everything tracked, and kill every later `add` at once. */
+	close(reason, logger) {
+		this.closedLogger = logger;
+		return this.killAll(reason, logger);
+	}
+};
+/**
+* The runner's decode workers: every fork is registered in the set, and
+* {@link shutdown} and any process exit kill whatever is still alive.
+*/
+var RunnerDecodeWorkers = class {
+	deps;
+	live = new LiveDecodeWorkers();
+	exitHookArmed = false;
+	shutDown = false;
+	constructor(deps) {
+		this.deps = deps;
+	}
+	/** Fork one worker, registered and cost-claimed. */
+	fork = (source) => {
+		this.armExitHook();
+		return this.deps.fork(source, {
+			costRegistry: this.deps.costRegistry,
+			liveWorkers: this.live
+		});
+	};
+	/**
+	* The addon is shutting down: kill every worker, and every later fork at
+	* once. Synchronous, so it is done before the runner's `process.exit`.
+	*/
+	shutdown() {
+		this.shutDown = true;
+		const killed = this.live.close("shutdown", this.deps.logger);
+		if (this.live.size === 0) this.disarmExitHook();
+		return killed;
+	}
+	/**
+	* The process is exiting without having shut this addon down (an uncaught
+	* error, or an `onShutdown` that never reached {@link shutdown}). The log
+	* line is attempted but will probably not land: the transport goes down
+	* with the process.
+	*/
+	onProcessExit = () => {
+		this.exitHookArmed = false;
+		this.live.close("process-exit", this.deps.logger);
+	};
+	armExitHook() {
+		if (this.exitHookArmed || this.shutDown) return;
+		this.exitHookArmed = true;
+		this.deps.process.once("exit", this.onProcessExit);
+	}
+	disarmExitHook() {
+		if (!this.exitHookArmed) return;
+		this.exitHookArmed = false;
+		this.deps.process.removeListener("exit", this.onProcessExit);
+	}
+};
 //#endregion
 //#region src/session-decode/native-lease-config.ts
 /**
@@ -1722,6 +1886,11 @@ var SessionDecodeSession = class {
 	/** Set when `kill()` is sent, so handleExit can tell an intentional stop from a death. */
 	killSent = false;
 	/**
+	* A SIGKILL already went out — an `'immediate'` teardown from the pump
+	* (D666) and the generator's later `finally` both escalate; one kill is enough.
+	*/
+	sigkillSent = false;
+	/**
 	* Images yielded and not yet closed that a CONSUMER still holds — queued,
 	* in flight, or being read. Each dies with the worker and somebody was going
 	* to read it first. This is what a drain waits on.
@@ -1933,7 +2102,7 @@ var SessionDecodeSession = class {
 	}
 	/** The drain (or a SIGTERM still waiting on its fallback) gives way to SIGKILL now. */
 	escalateToImmediate() {
-		if (this.exited) return;
+		if (this.exited || this.sigkillSent) return;
 		if (this.drainTimer !== null) {
 			clearTimeout(this.drainTimer);
 			this.drainTimer = null;
@@ -2120,7 +2289,14 @@ var SessionDecodeSession = class {
 	}
 	handleExit(info) {
 		if (!this.exited) this.hooks.onWorkerExit?.();
-		if (!this.killSent && !this.exited) this.logger.warn(this.stopRequested ? "session-decode worker exited unexpectedly while draining" : "session-decode worker exited unexpectedly", {
+		if (!this.killSent && !this.exited && info?.killedByRunner === true) this.logger.debug("session-decode worker ended by the runner shutdown kill", {
+			tags: { deviceId: this.deviceId },
+			meta: {
+				code: info.code,
+				signal: info.signal
+			}
+		});
+		else if (!this.killSent && !this.exited) this.logger.warn(this.stopRequested ? "session-decode worker exited unexpectedly while draining" : "session-decode worker exited unexpectedly", {
 			tags: { deviceId: this.deviceId },
 			meta: {
 				code: info?.code ?? null,
@@ -2312,11 +2488,15 @@ var SessionDecodeSession = class {
 	tryKill(signal) {
 		try {
 			this.child.kill(signal);
+			if (signal === "SIGKILL") this.sigkillSent = true;
 		} catch (err) {
-			this.logger.warn("session-decode-coordinator: kill() on child failed", { meta: {
-				signal,
-				err: toError(err).message
-			} });
+			this.logger.warn("session-decode-coordinator: kill() on child failed", {
+				tags: { deviceId: this.deviceId },
+				meta: {
+					signal,
+					err: toError(err).message
+				}
+			});
 		}
 	}
 };
@@ -2586,6 +2766,7 @@ function startSessionDecodePump(deps) {
 		stopped = true;
 		teardownMode = mode;
 		releaseSpawnSlotOnce();
+		if (mode === "immediate") escalateSession?.("immediate");
 		iterator.return().catch(() => {});
 	};
 }
@@ -8009,6 +8190,22 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
 	*/
 	costRegistry = new require_node.ChildCostRegistry();
 	/**
+	* Every decode worker this process forked, from ANY spawn site (live
+	* sessions, occupancy bursts), until its exit. The first thing `onShutdown`
+	* does is kill them, a fork after that is killed at once, and any process
+	* exit kills what is left (D666). The logger is only unready before
+	* `initialize()`, when no worker can have been forked.
+	*/
+	decodeWorkers = new RunnerDecodeWorkers({
+		fork: forkDecodeWorker,
+		costRegistry: this.costRegistry,
+		logger: {
+			info: (message, extras) => this.ctxIfReady?.logger.info(message, extras),
+			warn: (message, extras) => this.ctxIfReady?.logger.warn(message, extras)
+		},
+		process
+	});
+	/**
 	* Per-camera inference-deadline counters reported through
 	* `failure-contribution`. Bounded, cumulative, and read (never drained) by
 	* the provider. Constructed eagerly: created on the first timeout it would
@@ -8458,6 +8655,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
 		};
 	}
 	async onShutdown() {
+		this.decodeWorkers.shutdown();
 		if (this.runtimeInfoTimer !== null) {
 			clearInterval(this.runtimeInfoTimer);
 			this.runtimeInfoTimer = null;
@@ -10008,7 +10206,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends require_dist.BaseAdd
 	* environment. A worker is therefore forkable before anyone knows which
 	* camera it will serve.
 	*/
-	forkDecodeWorkerWithLease = (source) => forkDecodeWorker(source, this.costRegistry);
+	forkDecodeWorkerWithLease = (source) => this.decodeWorkers.fork(source);
 	/**
 	* The lease knobs for the session that is about to start — `setting > env >
 	* default`, resolved HERE because this process is the one that carries the

package/dist/pipeline-runner/index.mjs

@@ -34,14 +34,18 @@ var KILL_SIGNALS = {
 * parameter rather than a second `child.on('exit')` in the caller because
 * every listener added out here is one somebody can forget to remove.
 */
-function wrapChild(child, onExit) {
+function wrapChild(child, onExit, killedByRunner = () => false) {
 	let spawned = false;
 	let exitDelivered = false;
 	const exitListeners = [];
-	const deliverExit = (info) => {
+	const deliverExit = (exit) => {
 		if (exitDelivered) return;
 		exitDelivered = true;
 		onExit();
+		const info = killedByRunner() ? {
+			...exit,
+			killedByRunner: true
+		} : exit;
 		for (const listener of exitListeners) listener(info);
 	};
 	child.on("spawn", () => {
@@ -95,19 +99,179 @@ function wrapChild(child, onExit) {
 * node-av through `CAMSTACK_FRAMEWORK_DIR` / `CAMSTACK_BUNDLED_ADDONS_DIR` on
 * agents, so a replaced env would silently break decoding off the hub.
 */
-function forkDecodeWorker(source, costRegistry) {
-	const child = fork(CHILD_ENTRY_PATH, [], {
+function forkDecodeWorker(source, deps) {
+	return forkDecodeWorkerAt(CHILD_ENTRY_PATH, source, deps ?? {});
+}
+/**
+* {@link forkDecodeWorker} with the entry named — the one seam a process-level
+* test needs to fork a stub through the REAL fork, claim and live-worker path.
+* Production forks only {@link CHILD_ENTRY_PATH}.
+*/
+function forkDecodeWorkerAt(entryPath, source, deps) {
+	const child = fork(entryPath, [], {
 		serialization: "advanced",
 		env: { ...process.env }
 	});
-	const costClaim = costRegistry?.claim({
+	const costClaim = deps.costRegistry?.claim({
 		role: "decode",
 		deviceId: source.deviceId,
 		unit: `${source.deviceId}/decode`,
 		pid: child.pid
 	}) ?? NO_COST_CLAIM;
-	return wrapChild(child, () => costClaim.release());
+	let killedByRunner = false;
+	const untrack = deps.liveWorkers?.add({
+		deviceId: source.deviceId,
+		pid: child.pid,
+		killNow: () => {
+			killedByRunner = true;
+			child.kill("SIGKILL");
+		}
+	}) ?? (() => void 0);
+	return wrapChild(child, () => {
+		costClaim.release();
+		untrack();
+	}, () => killedByRunner);
+}
+//#endregion
+//#region src/session-decode/live-decode-workers.ts
+var KILL_MESSAGE = {
+	shutdown: "session-decode: worker killed at shutdown",
+	"forked-after-shutdown": "session-decode: worker forked after shutdown began — killed at once",
+	"process-exit": "session-decode: worker killed at process exit"
+};
+/** Send the signal. Never throws: a failure is the outcome. */
+function signal(worker) {
+	try {
+		worker.killNow();
+		return {
+			worker,
+			error: null
+		};
+	} catch (err) {
+		return {
+			worker,
+			error: errMsg(err)
+		};
+	}
+}
+/**
+* One line per outcome, each in its own swallowing try. The logger may throw:
+* the `'exit'` hook runs while the transport goes down, and the runner's
+* write path has no guard. A line that fails must not cost another line.
+*/
+function logOutcomes(outcomes, reason, logger) {
+	for (const { worker, error } of outcomes) try {
+		if (error === null) logger.info(KILL_MESSAGE[reason], {
+			tags: { deviceId: worker.deviceId },
+			meta: {
+				pid: worker.pid ?? null,
+				signal: "SIGKILL",
+				reason
+			}
+		});
+		else logger.warn("session-decode: worker kill failed — it may outlive the runner", {
+			tags: { deviceId: worker.deviceId },
+			meta: {
+				pid: worker.pid ?? null,
+				reason,
+				error
+			}
+		});
+	} catch {}
 }
+var LiveDecodeWorkers = class {
+	live = /* @__PURE__ */ new Set();
+	/** Set by {@link close}: from then on a worker is killed as it is added. */
+	closedLogger = null;
+	/** Track a forked worker; call the returned function from its real exit. */
+	add(worker) {
+		if (this.closedLogger !== null) {
+			const outcome = signal(worker);
+			logOutcomes([outcome], "forked-after-shutdown", this.closedLogger);
+			if (outcome.error === null) return () => void 0;
+		}
+		this.live.add(worker);
+		return () => {
+			this.live.delete(worker);
+		};
+	}
+	get size() {
+		return this.live.size;
+	}
+	/**
+	* SIGKILL every tracked worker now, then one log line per worker. SIGKILL
+	* rather than SIGTERM: the runner is about to exit and cannot wait for a
+	* fallback, and a worker blocked in a native call never runs a SIGTERM
+	* handler (D652).
+	*
+	* EVERY worker is signalled before ANY line is written: a logger that throws
+	* must not stop the kills. A signalled worker is forgotten (a second pass
+	* logs nothing twice); one whose kill threw stays tracked, so the exit hook
+	* or a later pass tries it again. Returns how many it signalled.
+	*/
+	killAll(reason, logger) {
+		const outcomes = [...this.live].map(signal);
+		for (const outcome of outcomes) if (outcome.error === null) this.live.delete(outcome.worker);
+		logOutcomes(outcomes, reason, logger);
+		return outcomes.filter((outcome) => outcome.error === null).length;
+	}
+	/** Kill everything tracked, and kill every later `add` at once. */
+	close(reason, logger) {
+		this.closedLogger = logger;
+		return this.killAll(reason, logger);
+	}
+};
+/**
+* The runner's decode workers: every fork is registered in the set, and
+* {@link shutdown} and any process exit kill whatever is still alive.
+*/
+var RunnerDecodeWorkers = class {
+	deps;
+	live = new LiveDecodeWorkers();
+	exitHookArmed = false;
+	shutDown = false;
+	constructor(deps) {
+		this.deps = deps;
+	}
+	/** Fork one worker, registered and cost-claimed. */
+	fork = (source) => {
+		this.armExitHook();
+		return this.deps.fork(source, {
+			costRegistry: this.deps.costRegistry,
+			liveWorkers: this.live
+		});
+	};
+	/**
+	* The addon is shutting down: kill every worker, and every later fork at
+	* once. Synchronous, so it is done before the runner's `process.exit`.
+	*/
+	shutdown() {
+		this.shutDown = true;
+		const killed = this.live.close("shutdown", this.deps.logger);
+		if (this.live.size === 0) this.disarmExitHook();
+		return killed;
+	}
+	/**
+	* The process is exiting without having shut this addon down (an uncaught
+	* error, or an `onShutdown` that never reached {@link shutdown}). The log
+	* line is attempted but will probably not land: the transport goes down
+	* with the process.
+	*/
+	onProcessExit = () => {
+		this.exitHookArmed = false;
+		this.live.close("process-exit", this.deps.logger);
+	};
+	armExitHook() {
+		if (this.exitHookArmed || this.shutDown) return;
+		this.exitHookArmed = true;
+		this.deps.process.once("exit", this.onProcessExit);
+	}
+	disarmExitHook() {
+		if (!this.exitHookArmed) return;
+		this.exitHookArmed = false;
+		this.deps.process.removeListener("exit", this.onProcessExit);
+	}
+};
 //#endregion
 //#region src/session-decode/native-lease-config.ts
 /**
@@ -1715,6 +1879,11 @@ var SessionDecodeSession = class {
 	/** Set when `kill()` is sent, so handleExit can tell an intentional stop from a death. */
 	killSent = false;
 	/**
+	* A SIGKILL already went out — an `'immediate'` teardown from the pump
+	* (D666) and the generator's later `finally` both escalate; one kill is enough.
+	*/
+	sigkillSent = false;
+	/**
 	* Images yielded and not yet closed that a CONSUMER still holds — queued,
 	* in flight, or being read. Each dies with the worker and somebody was going
 	* to read it first. This is what a drain waits on.
@@ -1926,7 +2095,7 @@ var SessionDecodeSession = class {
 	}
 	/** The drain (or a SIGTERM still waiting on its fallback) gives way to SIGKILL now. */
 	escalateToImmediate() {
-		if (this.exited) return;
+		if (this.exited || this.sigkillSent) return;
 		if (this.drainTimer !== null) {
 			clearTimeout(this.drainTimer);
 			this.drainTimer = null;
@@ -2113,7 +2282,14 @@ var SessionDecodeSession = class {
 	}
 	handleExit(info) {
 		if (!this.exited) this.hooks.onWorkerExit?.();
-		if (!this.killSent && !this.exited) this.logger.warn(this.stopRequested ? "session-decode worker exited unexpectedly while draining" : "session-decode worker exited unexpectedly", {
+		if (!this.killSent && !this.exited && info?.killedByRunner === true) this.logger.debug("session-decode worker ended by the runner shutdown kill", {
+			tags: { deviceId: this.deviceId },
+			meta: {
+				code: info.code,
+				signal: info.signal
+			}
+		});
+		else if (!this.killSent && !this.exited) this.logger.warn(this.stopRequested ? "session-decode worker exited unexpectedly while draining" : "session-decode worker exited unexpectedly", {
 			tags: { deviceId: this.deviceId },
 			meta: {
 				code: info?.code ?? null,
@@ -2305,11 +2481,15 @@ var SessionDecodeSession = class {
 	tryKill(signal) {
 		try {
 			this.child.kill(signal);
+			if (signal === "SIGKILL") this.sigkillSent = true;
 		} catch (err) {
-			this.logger.warn("session-decode-coordinator: kill() on child failed", { meta: {
-				signal,
-				err: toError(err).message
-			} });
+			this.logger.warn("session-decode-coordinator: kill() on child failed", {
+				tags: { deviceId: this.deviceId },
+				meta: {
+					signal,
+					err: toError(err).message
+				}
+			});
 		}
 	}
 };
@@ -2579,6 +2759,7 @@ function startSessionDecodePump(deps) {
 		stopped = true;
 		teardownMode = mode;
 		releaseSpawnSlotOnce();
+		if (mode === "immediate") escalateSession?.("immediate");
 		iterator.return().catch(() => {});
 	};
 }
@@ -8001,6 +8182,22 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
 	*/
 	costRegistry = new ChildCostRegistry();
 	/**
+	* Every decode worker this process forked, from ANY spawn site (live
+	* sessions, occupancy bursts), until its exit. The first thing `onShutdown`
+	* does is kill them, a fork after that is killed at once, and any process
+	* exit kills what is left (D666). The logger is only unready before
+	* `initialize()`, when no worker can have been forked.
+	*/
+	decodeWorkers = new RunnerDecodeWorkers({
+		fork: forkDecodeWorker,
+		costRegistry: this.costRegistry,
+		logger: {
+			info: (message, extras) => this.ctxIfReady?.logger.info(message, extras),
+			warn: (message, extras) => this.ctxIfReady?.logger.warn(message, extras)
+		},
+		process
+	});
+	/**
 	* Per-camera inference-deadline counters reported through
 	* `failure-contribution`. Bounded, cumulative, and read (never drained) by
 	* the provider. Constructed eagerly: created on the first timeout it would
@@ -8450,6 +8647,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
 		};
 	}
 	async onShutdown() {
+		this.decodeWorkers.shutdown();
 		if (this.runtimeInfoTimer !== null) {
 			clearInterval(this.runtimeInfoTimer);
 			this.runtimeInfoTimer = null;
@@ -10000,7 +10198,7 @@ var PipelineRunnerAddon = class PipelineRunnerAddon extends BaseAddon {
 	* environment. A worker is therefore forkable before anyone knows which
 	* camera it will serve.
 	*/
-	forkDecodeWorkerWithLease = (source) => forkDecodeWorker(source, this.costRegistry);
+	forkDecodeWorkerWithLease = (source) => this.decodeWorkers.fork(source);
 	/**
 	* The lease knobs for the session that is about to start — `setting > env >
 	* default`, resolved HERE because this process is the one that carries the