@camstack/addon-pipeline 1.2.295 → 1.2.296

package/dist/detection-pipeline/index.js

@@ -3,11 +3,11 @@ Object.defineProperties(exports, {
 	[Symbol.toStringTag]: { value: "Module" }
 });
 const require_chunk = require("../chunk-emK7D4bc.js");
-const require_dist = require("../dist-BsVcf5wO.js");
-const require_node = require("../node-UFk6I2f6.js");
-const require_default_detection_model = require("../default-detection-model-wNEISVA9.js");
+const require_dist = require("../dist-8up-f2TX.js");
+const require_node = require("../node-lpQgHes9.js");
+const require_default_detection_model = require("../default-detection-model-D1daTtqT.js");
 const require_lazy_sharp = require("../lazy-sharp-BuwKlQLR.js");
-const require_process_memory = require("../process-memory-CJV29sPf.js");
+const require_process_memory = require("../process-memory-CX_92V_r.js");
 let node_os = require("node:os");
 node_os = require_chunk.__toESM(node_os);
 let node_fs = require("node:fs");
@@ -422,6 +422,12 @@ var ClusterModelSource = class {
 			const next = require_dist.pickClusterStepModels(view);
 			const nextSettings = require_dist.pickClusterStepSettings(view);
 			this.lastReadAtMs = this.now();
+			const invalid = require_dist.pickInvalidClusterStepModels(view);
+			if (invalid.length > 0) this.logger.warn("cluster step model row holds a value that is not a model id — running the catalog default for that step; no re-embed pass will run on it", { meta: {
+				owner: CLUSTER_MODEL_OWNER_ADDON_ID,
+				invalid,
+				inUse: Object.fromEntries(invalid.map((e) => [e.stepId, next[e.stepId]]))
+			} });
 			const changed = Object.keys(next).filter((stepId) => next[stepId] !== this.models[stepId]);
 			const settingsChanged = Object.keys(nextSettings).some((stepId) => {
 				const from = this.settings[stepId] ?? {};
@@ -569,6 +575,98 @@ var DeviceOverrideMirror = class {
 	}
 };
 //#endregion
+//#region src/detection-pipeline/engine/pool-worker-health.ts
+/**
+* The one table of load bounds (D653, fix round 1).
+*
+* The soft bound decides how soon the operator HEARS about a slow compile; the
+* hard bound decides when a compile is given up for hung. A single 120 s bound
+* that also killed the compile was wrong twice: a slow but finite compile was
+* killed before it could write its cache, so every respawn faced the same cold
+* compile and three of them walked the device to `failed`; and on CoreML the
+* bound was shorter than the cross-process compile-cache lock wait alone.
+*/
+var POOL_MODEL_LOAD_BOUNDS = {
+	openvino: {
+		softMs: 12e4,
+		hardMs: 6e5,
+		why: "A cold iGPU compile of the WHOLE model set measured ~25 s on the N100; one model is 1-3 s (YuNet 1.4 s on a fresh worker, 2026-09-26). 120 s is ~5x the worst measured set. No evidence of a legitimate compile past it; the hard bound gives one 5x more before the worker is recycled.",
+		gilMayBeHeldDuringCompile: false,
+		gilWhy: "The OpenVINO Python binding is believed to release the GIL around compile_model (gil_scoped_release). Not measured on the hub; the passive recipe in D653 checks it."
+	},
+	coreml: {
+		softMs: 3e5,
+		hardMs: 9e5,
+		why: "A load may first WAIT up to 180 s for another worker holding the compile-cache lock (`COMPILE_CACHE_LOCK_TIMEOUT_SEC` in inference_pool.py), then compile itself: 180 s of lock plus 120 s of compile. The old 120 s bound was shorter than the lock wait alone.",
+		gilMayBeHeldDuringCompile: true,
+		gilWhy: "Unverified whether coremltools releases the GIL while it compiles an mlpackage. If it does not, the loop freezes and nothing — not the soft timer, not mem_stats — can answer."
+	},
+	onnxruntime: {
+		softMs: 12e4,
+		hardMs: 6e5,
+		why: "Session creation, no persistent compile cache; CUDA/CoreML EP init is the slow case. Same numbers as OpenVINO for lack of any measurement saying otherwise.",
+		gilMayBeHeldDuringCompile: false,
+		gilWhy: "Not known to hold the GIL through session creation, and no frozen loop has been observed. Claiming the grace would blind stall detection during every load."
+	},
+	edgetpu: {
+		softMs: 12e4,
+		hardMs: 6e5,
+		why: "An Edge TPU model is precompiled; a load is an interpreter + delegate bind of seconds. Kept at the shared default rather than tightened without a measurement.",
+		gilMayBeHeldDuringCompile: false,
+		gilWhy: "Nothing is compiled: the load is an interpreter + delegate bind of seconds."
+	}
+};
+/**
+* The host-side deadline of a `load` / `replace`: the soft bound plus a margin,
+* so the worker's NAMED answer always lands first.
+*/
+var POOL_MODEL_LOAD_REPLY_MARGIN_MS = 3e4;
+function chargesDeviceBudget(reason) {
+	return reason !== "compile-hung";
+}
+/**
+* Tracks live inference outcomes on one worker and says when its executor has
+* produced nothing for too long. Consulted on demand — no timer.
+*
+* A SHED (`dropped`) does not count as a result: the Python loop answers sheds
+* itself, so a worker whose executor threads are hung keeps shedding happily.
+*/
+var InferStallDetector = class {
+	unanswered = 0;
+	lastResultAt;
+	/**
+	* When the current streak of unanswered requests began — the first one after
+	* a result. IDLE time is not silence: quiet cameras send nothing, and a
+	* worker that was asked nothing has failed nothing. Measuring from the last
+	* result let a 60 s lull plus the first 20 sheds of a burst (~8 s with six
+	* cameras) poison a healthy worker and charge the device.
+	*/
+	streakStartedAt = null;
+	constructor(now) {
+		this.lastResultAt = now;
+	}
+	/** A reply that carried a real result (or a real error) from the executor. */
+	noteResult(now) {
+		this.unanswered = 0;
+		this.lastResultAt = now;
+		this.streakStartedAt = null;
+	}
+	/**
+	* A live request that ended without a result (deadline expiry or a shed).
+	* Returns the stall when the worker has crossed both thresholds.
+	*/
+	noteUnanswered(now) {
+		this.unanswered += 1;
+		if (this.streakStartedAt === null) this.streakStartedAt = now;
+		const silentMs = now - Math.max(this.lastResultAt, this.streakStartedAt);
+		if (this.unanswered >= 20 && silentMs >= 3e4) return {
+			unanswered: this.unanswered,
+			silentMs
+		};
+		return null;
+	}
+};
+//#endregion
 //#region src/detection-pipeline/engine/shared-inference-pool.ts
 /**
 * SharedInferencePool — TypeScript wrapper for inference_pool.py.
@@ -610,6 +708,27 @@ var RAW_FMT_CODE = {
 	gray: 2
 };
 /**
+* A load the worker answered with a machine-readable failure class. The
+* provider tells a `compile-timeout` (counted against THAT MODEL on the
+* device) from an ordinary failure by `reason`, never by parsing text.
+*/
+var PoolModelLoadError = class extends Error {
+	reason;
+	/** The model's file stem, as the worker named it. */
+	model;
+	constructor(message, reason, model) {
+		super(message);
+		this.name = "PoolModelLoadError";
+		this.reason = reason;
+		this.model = model;
+	}
+};
+/**
+* Request id the worker uses for UNSOLICITED events — a reply to no request
+* (`compile-finished-late`). Never allocated to a request.
+*/
+var WORKER_EVENT_REQ_ID = 4294967295;
+/**
 * Per-inference-request reply timeout (ms). Turns a wedged request (worker
 * alive but no reply) into a rejection so every caller settles — runtime frame
 * dispatch drops the frame; the benchmark full-tree run rejects the wedged
@@ -618,6 +737,22 @@ var RAW_FMT_CODE = {
 * large frame) never trips; override via env for constrained hardware.
 */
 var POOL_INFER_TIMEOUT_MS = Math.max(1e3, Number(process.env["CAMSTACK_POOL_INFER_TIMEOUT_MS"]) || 6e4);
+/** Commands that compile a model — the only ones that get the load deadline. */
+var MODEL_LOAD_COMMANDS = new Set(["load", "replace"]);
+/**
+* Deadline of the liveness probe (`mem_stats`) sent after a command times out.
+* The worker answers `mem_stats` on its event loop, never behind a compile
+* (D653), so a healthy worker replies in milliseconds even mid-compile; ten
+* seconds only has to outlast a GC pause or a burst of inference replies.
+*/
+var POOL_LIVENESS_PROBE_TIMEOUT_MS = 1e4;
+/**
+* How long a poisoned worker may keep its in-flight LIVE requests before it is
+* killed anyway. A poisoned worker takes no new work, and every live request
+* carries a {@link POOL_LIVE_INFER_TIMEOUT_MS} deadline, so the drain is over
+* by then; the second of slack keeps the backstop from racing the last reply.
+*/
+var POOL_POISON_DRAIN_SLACK_MS = 1e3;
 /**
 * Max inference requests outstanding to ONE worker before new ones are SHED.
 *
@@ -805,6 +940,27 @@ var PoolWorker = class {
 	wireVersion = 1;
 	nextRequestId = 1;
 	ready = false;
+	/** Set once this worker is declared unusable while ALIVE (D653). Never cleared:
+	*  a poisoned worker is recycled, not revived. */
+	poisonVerdict = null;
+	/** The kill of a poisoned worker has been started. */
+	recycling = false;
+	/** Kills a poisoned worker whose drain did not finish in time. */
+	recycleBackstop = null;
+	/** A liveness probe is in flight — one at a time, never a probe storm. */
+	probeInFlight = false;
+	/** The child has exited (any cause). */
+	exited = false;
+	/** A compile past its soft bound, still running in the worker (D653). */
+	abandonedCompile = null;
+	/** Says when live inference has produced nothing for too long (D653). */
+	inferStall = new InferStallDetector(Date.now());
+	/**
+	* A load's HOST deadline expired with no reply since the last reply of any
+	* kind (D653 round 3): the loop missed its own soft bound, so the GIL grace
+	* is over for this worker until something — anything — comes back.
+	*/
+	loadDeadlineMissed = false;
 	log;
 	opts;
 	constructor(opts) {
@@ -829,6 +985,24 @@ var PoolWorker = class {
 	isReady() {
 		return this.ready;
 	}
+	/** Why this worker was recycled, typed for the provider's budget, or `null`. */
+	getDeathCause() {
+		const v = this.poisonVerdict;
+		const message = this.getPoisonDescription();
+		if (v === null || message === null) return null;
+		return {
+			reason: v.reason,
+			chargesDeviceBudget: chargesDeviceBudget(v.reason),
+			message
+		};
+	}
+	/** Why this live worker was declared unusable, or `null` (D653). */
+	getPoisonDescription() {
+		const v = this.poisonVerdict;
+		if (v === null) return null;
+		const target = v.command.model !== null ? ` of ${v.command.model}` : "";
+		return `${v.reason} on ${v.command.cmd}${target} (pid ${v.pid ?? "unknown"})`;
+	}
 	async initialize(initialModels) {
 		this.process = (0, node_child_process.spawn)(this.opts.pythonPath, [this.opts.scriptPath], { stdio: [
 			"pipe",
@@ -852,7 +1026,11 @@ var PoolWorker = class {
 		const spawnedProcess = this.process;
 		spawnedProcess.on("exit", (code, signal) => {
 			this.ready = false;
+			this.exited = true;
+			if (this.recycleBackstop) clearTimeout(this.recycleBackstop);
+			if (this.abandonedCompile) clearTimeout(this.abandonedCompile.hardTimer);
 			if (this.process !== spawnedProcess) return;
+			const poisonedBy = this.getPoisonDescription();
 			this.log.error("Worker process exited", { meta: {
 				worker: this.opts.workerLabel,
 				pid: spawnedProcess.pid ?? null,
@@ -860,9 +1038,10 @@ var PoolWorker = class {
 				device: this.opts.device ?? "default",
 				code,
 				signal,
-				inFlight: this.pending.size
+				inFlight: this.pending.size,
+				...poisonedBy !== null ? { poisonedBy } : {}
 			} });
-			this.rejectAll(/* @__PURE__ */ new Error(`PoolWorker[${this.opts.workerLabel}]: worker process exited (code=${code ?? "null"}, signal=${signal ?? "none"})`));
+			this.rejectAll(/* @__PURE__ */ new Error(`PoolWorker[${this.opts.workerLabel}]: worker process exited (code=${code ?? "null"}, signal=${signal ?? "none"})` + (poisonedBy !== null ? ` — recycled, poisoned by ${poisonedBy}` : "")));
 		});
 		this.process.stdout.on("data", (chunk) => {
 			const t0 = Date.now();
@@ -876,6 +1055,7 @@ var PoolWorker = class {
 			runtime: this.opts.poolRuntime,
 			concurrency: this.opts.concurrency,
 			protocolVersion: 2,
+			modelLoadTimeoutMs: POOL_MODEL_LOAD_BOUNDS[this.opts.poolRuntime].softMs,
 			models: initialModels.map((m) => serializeModelConfig(m))
 		};
 		if (this.opts.device) config["device"] = this.opts.device;
@@ -898,6 +1078,7 @@ var PoolWorker = class {
 					clearTimeout(timeout);
 					if (result["status"] === "ready") {
 						this.ready = true;
+						this.inferStall.noteResult(Date.now());
 						const loadedCount = result["models"];
 						const startupMs = result["startupMs"];
 						const workers = result["workers"] ?? 1;
@@ -920,7 +1101,7 @@ var PoolWorker = class {
 	async infer(modelByte, jpeg, deviceId) {
 		this.ensureReady();
 		const payload = Buffer.concat([Buffer.from([modelByte]), jpeg]);
-		return this.dispatch(MSG_INFER_JPEG, payload, deviceId);
+		return this.dispatch(MSG_INFER_JPEG, payload, { deviceId });
 	}
 	async inferRaw(modelByte, raw, width, height, format, deviceId) {
 		this.ensureReady();
@@ -973,12 +1154,18 @@ var PoolWorker = class {
 		const payload = Buffer.allocUnsafe(5);
 		payload[0] = modelByte;
 		payload.writeUInt32LE(frameId, 1);
-		return this.dispatch(MSG_INFER_CACHED, payload, deviceId);
+		return this.dispatch(MSG_INFER_CACHED, payload, { deviceId });
 	}
 	async sendCommand(cmd) {
 		this.ensureReady();
+		const command = describeCommand(cmd);
 		const payload = Buffer.from(JSON.stringify(cmd), "utf8");
-		return await this.dispatch(MSG_COMMAND, payload);
+		const sentAt = Date.now();
+		if (MODEL_LOAD_COMMANDS.has(command.cmd)) this.warnIfLoadQueued(command);
+		const raw = await this.dispatch(MSG_COMMAND, payload, { command });
+		if (MODEL_LOAD_COMMANDS.has(command.cmd)) this.inferStall.noteResult(Date.now());
+		this.inspectCommandReply(raw, command, sentAt);
+		return raw;
 	}
 	/** Command whose reply shape is NOT the load/unload/replace/status envelope
 	*  (e.g. `mem_stats`) — returns the raw JSON record, so no cast is needed
@@ -986,13 +1173,15 @@ var PoolWorker = class {
 	async sendRawCommand(cmd) {
 		this.ensureReady();
 		const payload = Buffer.from(JSON.stringify(cmd), "utf8");
-		return this.dispatch(MSG_COMMAND, payload);
+		return this.dispatch(MSG_COMMAND, payload, { command: describeCommand(cmd) });
 	}
 	async dispose() {
 		const proc = this.process;
 		if (!proc) return;
 		this.process = null;
 		this.ready = false;
+		if (this.recycleBackstop) clearTimeout(this.recycleBackstop);
+		if (this.abandonedCompile) clearTimeout(this.abandonedCompile.hardTimer);
 		await terminateChild(proc, POOL_WORKER_TERM_GRACE_MS);
 		this.rejectAll(/* @__PURE__ */ new Error(`PoolWorker[${this.opts.workerLabel}]: pool disposed while the request was in flight`));
 	}
@@ -1035,8 +1224,11 @@ var PoolWorker = class {
 	}
 	/** Live inference gets seconds; commands and model loads keep the long
 	*  timeout — see {@link POOL_LIVE_INFER_TIMEOUT_MS}. */
-	deadlineFor(msgType) {
-		return SHEDDABLE_MSG_TYPES.has(msgType) ? POOL_LIVE_INFER_TIMEOUT_MS : POOL_INFER_TIMEOUT_MS;
+	deadlineFor(msgType, opts = {}) {
+		if (opts.timeoutMs !== void 0) return opts.timeoutMs;
+		if (SHEDDABLE_MSG_TYPES.has(msgType)) return POOL_LIVE_INFER_TIMEOUT_MS;
+		if (opts.command !== void 0 && MODEL_LOAD_COMMANDS.has(opts.command.cmd)) return POOL_MODEL_LOAD_BOUNDS[this.opts.poolRuntime].softMs + POOL_MODEL_LOAD_REPLY_MARGIN_MS;
+		return POOL_INFER_TIMEOUT_MS;
 	}
 	/**
 	* The request's ABSOLUTE deadline for the wire (D350), or `null` when this
@@ -1053,16 +1245,18 @@ var PoolWorker = class {
 		stamp.writeBigUInt64LE(BigInt(Date.now() + this.deadlineFor(msgType)), 0);
 		return stamp;
 	}
-	dispatch(msgType, payload, deviceId) {
-		const shed = this.shedIfSaturated(msgType, deviceId);
+	dispatch(msgType, payload, opts = {}) {
+		const shed = this.shedIfSaturated(msgType, opts.deviceId);
 		if (shed) return Promise.resolve(shed);
 		const reqId = this.allocRequestId();
 		return new Promise((resolve, reject) => {
-			const timer = this.armRequestTimeout(reqId, msgType, resolve, reject, deviceId);
+			const timer = this.armRequestTimeout(reqId, msgType, resolve, reject, opts);
 			this.pending.set(reqId, {
 				resolve,
 				reject,
-				timer
+				timer,
+				...opts.command !== void 0 ? { command: opts.command } : {},
+				...SHEDDABLE_MSG_TYPES.has(msgType) ? { live: true } : {}
 			});
 			try {
 				const stamp = this.deadlineStamp(msgType);
@@ -1093,8 +1287,9 @@ var PoolWorker = class {
 	* - Anything else (commands, model loads) still REJECTS with the error the
 	*   dashboards grep for — a lost command is a fault, not flow control.
 	*/
-	armRequestTimeout(reqId, msgType, resolve, reject, deviceId) {
-		const timeoutMs = this.deadlineFor(msgType);
+	armRequestTimeout(reqId, msgType, resolve, reject, opts = {}) {
+		const { deviceId, command } = opts;
+		const timeoutMs = this.deadlineFor(msgType, opts);
 		const timer = setTimeout(() => {
 			if (this.pending.delete(reqId)) {
 				if (SHEDDABLE_MSG_TYPES.has(msgType)) {
@@ -1118,6 +1313,7 @@ var PoolWorker = class {
 						dropped: true,
 						shedReason: "deadline-expired"
 					});
+					this.noteInferUnanswered();
 					return;
 				}
 				this.timedOutCount++;
@@ -1130,10 +1326,20 @@ var PoolWorker = class {
 						device: this.opts.device ?? "default",
 						inFlight: this.pending.size,
 						reqId,
-						timeoutMs
+						timeoutMs,
+						...command !== void 0 ? {
+							command: command.cmd,
+							modelIndex: command.modelIndex,
+							model: command.model
+						} : {}
 					}
 				});
-				reject(/* @__PURE__ */ new Error(`PoolWorker[${this.opts.workerLabel}]: inference request ${reqId} timed out after ${timeoutMs}ms on ${this.opts.poolRuntime}:${this.opts.device ?? "default"} (worker alive, no reply)`));
+				const message = `PoolWorker[${this.opts.workerLabel}]: inference request ${reqId} timed out after ${timeoutMs}ms on ${this.opts.poolRuntime}:${this.opts.device ?? "default"} (worker alive, no reply)`;
+				if (command !== void 0 && MODEL_LOAD_COMMANDS.has(command.cmd)) {
+					this.loadDeadlineMissed = true;
+					reject(new PoolModelLoadError(`compile-timeout: ${message}`, "compile-timeout", command.model));
+				} else reject(new Error(message));
+				if (opts.probe !== true && command !== void 0) this.probeLiveness(command);
 			}
 		}, timeoutMs);
 		timer.unref?.();
@@ -1144,11 +1350,12 @@ var PoolWorker = class {
 		if (shed) return Promise.resolve(shed);
 		const reqId = this.allocRequestId();
 		return new Promise((resolve, reject) => {
-			const timer = this.armRequestTimeout(reqId, msgType, resolve, reject, deviceId);
+			const timer = this.armRequestTimeout(reqId, msgType, resolve, reject, { deviceId });
 			this.pending.set(reqId, {
 				resolve,
 				reject,
-				timer
+				timer,
+				...SHEDDABLE_MSG_TYPES.has(msgType) ? { live: true } : {}
 			});
 			try {
 				if (!this.process?.stdin) throw new Error("PoolWorker: not initialized");
@@ -1170,10 +1377,10 @@ var PoolWorker = class {
 	}
 	allocRequestId() {
 		let id = this.nextRequestId;
-		this.nextRequestId = id >= 4294967295 ? 1 : id + 1;
+		this.nextRequestId = id >= WORKER_EVENT_REQ_ID - 1 ? 1 : id + 1;
 		while (this.pending.has(id)) {
 			id = this.nextRequestId;
-			this.nextRequestId = id >= 4294967295 ? 1 : id + 1;
+			this.nextRequestId = id >= WORKER_EVENT_REQ_ID - 1 ? 1 : id + 1;
 		}
 		return id;
 	}
@@ -1188,6 +1395,8 @@ var PoolWorker = class {
 		this.process.stdin.write(payload);
 	}
 	ensureReady() {
+		const poisonedBy = this.getPoisonDescription();
+		if (poisonedBy !== null) throw new Error(`PoolWorker[${this.opts.workerLabel}]: poisoned (${poisonedBy}) — recycling, not taking work`);
 		if (!this.ready || !this.process?.stdin) throw new Error(`PoolWorker[${this.opts.workerLabel}]: not initialized`);
 	}
 	/** Time spent in `Buffer.concat` since the last report. */
@@ -1227,23 +1436,273 @@ var PoolWorker = class {
 			const reqId = this.receiveBuffer.readUInt32LE(4);
 			const jsonBytes = this.receiveBuffer.subarray(8, 4 + totalLen);
 			this.receiveBuffer = this.receiveBuffer.subarray(4 + totalLen);
+			if (reqId === WORKER_EVENT_REQ_ID) {
+				this.handleWorkerEvent(jsonBytes);
+				continue;
+			}
 			const entry = this.pending.get(reqId);
 			if (!entry) {
 				this.log.warn("Response for unknown request id", { meta: {
 					worker: this.opts.workerLabel,
 					reqId
 				} });
+				this.noteLateReply(jsonBytes);
 				continue;
 			}
 			this.pending.delete(reqId);
 			if (entry.timer) clearTimeout(entry.timer);
 			try {
 				const parsed = JSON.parse(jsonBytes.toString("utf8"));
+				if (entry.live === true) if (parsed["dropped"] === true) this.noteInferUnanswered();
+				else this.inferStall.noteResult(Date.now());
 				entry.resolve(parsed);
 			} catch (err) {
 				entry.reject(err instanceof Error ? err : new Error(String(err)));
 			}
 		}
+		this.noteAnyReply();
+		if (this.poisonVerdict !== null && this.pending.size === 0) this.recycle();
+	}
+	/**
+	* Something came back from the worker: its loop was alive just now, so the
+	* missed-deadline verdict is lifted.
+	*/
+	noteAnyReply() {
+		this.loadDeadlineMissed = false;
+	}
+	/**
+	* The invariant the load deadline depends on (D653 § 2): loads into one
+	* pool are serialised, so at most ONE load is outstanding per worker. The
+	* worker runs model commands in order and starts a load's soft bound only
+	* when it begins, while the host's deadline for it runs from the SEND. A
+	* load queued behind another could therefore see its host deadline fire
+	* before the worker's named answer. Nothing in the provider issues that; if
+	* anything ever does, this says so.
+	*/
+	warnIfLoadQueued(next) {
+		const ahead = [...this.pending.values()].map((p) => p.command).filter((c) => c !== void 0 && MODEL_LOAD_COMMANDS.has(c.cmd));
+		if (ahead.length === 0) return;
+		this.log.warn("model load queued behind another on one worker — its deadline may fire before the worker answers", { meta: {
+			worker: this.opts.workerLabel,
+			pid: this.getPid(),
+			runtime: this.opts.poolRuntime,
+			device: this.opts.device ?? "default",
+			model: next.model,
+			queuedBehind: ahead.map((c) => c.model)
+		} });
+	}
+	hasPendingLoad() {
+		for (const p of this.pending.values()) if (p.command !== void 0 && MODEL_LOAD_COMMANDS.has(p.command.cmd)) return true;
+		return false;
+	}
+	/**
+	* A load reply that says a compile outlived its SOFT bound (`compile-timeout`)
+	* or that an earlier one still has (`worker-poisoned`). The worker is not
+	* recycled for it (fix round 1): the compile keeps running so a slow but
+	* finite one writes its cache, the loaded models keep serving, and the worker
+	* starts no new load. Only a compile still running at the HARD bound costs
+	* the process.
+	*/
+	inspectCommandReply(raw, command, sentAt) {
+		const reason = raw["reason"];
+		if (reason !== "compile-timeout" && reason !== "worker-poisoned") return;
+		if (this.abandonedCompile !== null || this.poisonVerdict !== null || this.exited) return;
+		const model = raw["modelId"];
+		const abandoned = {
+			...command,
+			model: typeof model === "string" ? model : command.model
+		};
+		const bounds = POOL_MODEL_LOAD_BOUNDS[this.opts.poolRuntime];
+		const reported = raw["compileElapsedMs"];
+		const elapsedMs = typeof reported === "number" && Number.isFinite(reported) && reported >= 0 ? reported : Date.now() - sentAt;
+		const hardTimer = setTimeout(() => this.poison("compile-hung", abandoned, `the compile of ${abandoned.model ?? "unknown"} was still running at its hard bound (${bounds.hardMs}ms)`), Math.max(0, bounds.hardMs - elapsedMs));
+		hardTimer.unref?.();
+		this.abandonedCompile = {
+			command: abandoned,
+			since: Date.now(),
+			hardTimer
+		};
+		this.log.warn("model compile past its soft bound — left running so its cache can be written", { meta: {
+			worker: this.opts.workerLabel,
+			pid: this.getPid(),
+			runtime: this.opts.poolRuntime,
+			device: this.opts.device ?? "default",
+			model: abandoned.model,
+			modelIndex: abandoned.modelIndex,
+			softMs: bounds.softMs,
+			hardMs: bounds.hardMs,
+			loads: "refused until it returns"
+		} });
+	}
+	/** An unsolicited worker event (`compile-finished-late`). */
+	handleWorkerEvent(jsonBytes) {
+		let event;
+		try {
+			event = JSON.parse(jsonBytes.toString("utf8"));
+		} catch {
+			return;
+		}
+		if (event["event"] !== "compile-finished-late") return;
+		const abandoned = this.abandonedCompile;
+		if (abandoned !== null) clearTimeout(abandoned.hardTimer);
+		this.abandonedCompile = null;
+		this.inferStall.noteResult(Date.now());
+		const ok = event["ok"] === true;
+		const meta = {
+			worker: this.opts.workerLabel,
+			pid: this.getPid(),
+			runtime: this.opts.poolRuntime,
+			device: this.opts.device ?? "default",
+			model: typeof event["modelId"] === "string" ? event["modelId"] : null,
+			elapsedMs: typeof event["elapsedMs"] === "number" ? event["elapsedMs"] : null,
+			...typeof event["error"] === "string" ? { error: event["error"] } : {}
+		};
+		if (ok) this.log.info("slow model compile finished after its soft bound — cache written, loads re-enabled", { meta });
+		else this.log.warn("slow model compile failed after its soft bound — loads re-enabled", { meta });
+		this.opts.onCompileFinishedLate?.({
+			model: meta.model,
+			ok,
+			...typeof event["error"] === "string" ? { error: event["error"] } : {}
+		});
+	}
+	/**
+	* The GIL grace (D653 round 2): may a silent loop be a compile holding the
+	* GIL rather than a wedge? Only while a load is SENT AND UNANSWERED, and only
+	* on a runtime whose compile may hold the GIL (`POOL_MODEL_LOAD_BOUNDS`).
+	*
+	* NOT while a compile runs past its soft bound: the worker REPLIED
+	* `compile-timeout`, so its loop is proven alive, and an inference hang
+	* behind that compile — the 2026-09-26 incident exactly — must be seen on
+	* the normal rule, not ~600-900 s later at the hard bound.
+	*/
+	gilGraceActive() {
+		if (!POOL_MODEL_LOAD_BOUNDS[this.opts.poolRuntime].gilMayBeHeldDuringCompile) return false;
+		if (this.loadDeadlineMissed) return false;
+		return this.hasPendingLoad();
+	}
+	/**
+	* The reason a silent worker is recycled with: a freeze that began with a
+	* load missing its deadline is that compile's doing (`compile-hung`, charged
+	* to the model, not the device); anything else is the device's.
+	*/
+	silenceReason(fallback) {
+		return this.loadDeadlineMissed ? "compile-hung" : fallback;
+	}
+	/** A live request ended with no result: judge the executor (D653, item 4). */
+	noteInferUnanswered() {
+		const stall = this.inferStall.noteUnanswered(Date.now());
+		if (stall === null || this.gilGraceActive()) return;
+		this.poison(this.silenceReason("infer-unresponsive"), {
+			cmd: "infer",
+			modelIndex: null,
+			model: null
+		}, `${stall.unanswered} live requests in a row ended without a result, none for ${stall.silentMs}ms`);
+	}
+	/** A reply whose request was already abandoned: a real result still proves the executor runs. */
+	noteLateReply(jsonBytes) {
+		try {
+			const parsed = JSON.parse(jsonBytes.toString("utf8"));
+			if (parsed["dropped"] !== true && parsed["cmd"] === void 0) this.inferStall.noteResult(Date.now());
+		} catch {}
+	}
+	/**
+	* Ask a worker whose command just timed out whether its loop still answers.
+	* `mem_stats` is served on the loop, never behind a compile, so a worker
+	* that cannot answer it inside {@link POOL_LIVENESS_PROBE_TIMEOUT_MS} is not
+	* slow — it is wedged. That is the 2026-09-26 shape exactly: 123 of 123
+	* `mem_stats` lost while the process looked alive.
+	*/
+	probeLiveness(timedOut) {
+		if (this.probeInFlight || this.poisonVerdict !== null || this.exited) return;
+		this.probeInFlight = true;
+		this.dispatch(MSG_COMMAND, Buffer.from(JSON.stringify({ cmd: "mem_stats" }), "utf8"), {
+			command: {
+				cmd: "mem_stats",
+				modelIndex: null,
+				model: null
+			},
+			timeoutMs: POOL_LIVENESS_PROBE_TIMEOUT_MS,
+			probe: true
+		}).then(() => {
+			this.log.warn("pool command timed out but the worker answers its liveness probe — slow, not wedged", { meta: {
+				worker: this.opts.workerLabel,
+				pid: this.getPid(),
+				runtime: this.opts.poolRuntime,
+				device: this.opts.device ?? "default",
+				command: timedOut.cmd,
+				modelIndex: timedOut.modelIndex,
+				model: timedOut.model
+			} });
+		}, (err) => {
+			if (this.gilGraceActive()) {
+				this.log.warn("liveness probe unanswered while a model load is outstanding — not poisoning yet", { meta: {
+					worker: this.opts.workerLabel,
+					pid: this.getPid(),
+					runtime: this.opts.poolRuntime,
+					device: this.opts.device ?? "default",
+					command: timedOut.cmd,
+					model: timedOut.model
+				} });
+				return;
+			}
+			this.poison(this.silenceReason("unresponsive"), timedOut, `${timedOut.cmd} timed out, then the liveness probe failed: ${err instanceof Error ? err.message : String(err)}`);
+		}).finally(() => {
+			this.probeInFlight = false;
+		});
+	}
+	/**
+	* Declare this LIVE worker unusable, say so once at ERROR, stop taking work,
+	* and recycle it once it has drained.
+	*
+	* `ready = false` is what hands the pool back to the provider: its next
+	* dispatch finds the factory not ready and condemns it under the per-device
+	* restart budget (3 deaths in 10 min, then a terminal `failed` the balancer
+	* excludes) — the same path a crashed worker takes, except that a
+	* `compile-hung` death is not charged to the device (see PoolPoisonReason).
+	* An unresponsive worker is killed at once (it will answer nothing it
+	* holds); a compile-hung one keeps serving its loaded models until its
+	* in-flight requests are answered, bounded by the live deadline.
+	*/
+	poison(reason, command, detail) {
+		if (this.poisonVerdict !== null || this.exited || this.process === null) return;
+		this.poisonVerdict = {
+			reason,
+			command,
+			detail,
+			pid: this.getPid()
+		};
+		this.ready = false;
+		const inFlightCommands = [...this.pending.values()].map((p) => p.command).filter((c) => c !== void 0).map((c) => c.model !== null ? `${c.cmd}:${c.model}` : c.cmd);
+		this.log.error("pool worker POISONED — recycling it", { meta: {
+			worker: this.opts.workerLabel,
+			pid: this.getPid(),
+			runtime: this.opts.poolRuntime,
+			device: this.opts.device ?? "default",
+			reason,
+			command: command.cmd,
+			modelIndex: command.modelIndex,
+			model: command.model,
+			detail,
+			inFlight: this.pending.size,
+			inFlightCommands
+		} });
+		if (reason !== "compile-hung" || this.pending.size === 0) {
+			this.recycle();
+			return;
+		}
+		this.recycleBackstop = setTimeout(() => this.recycle(), POOL_LIVE_INFER_TIMEOUT_MS + POOL_POISON_DRAIN_SLACK_MS);
+		this.recycleBackstop.unref?.();
+	}
+	/**
+	* Kill a poisoned worker WITHOUT nulling `this.process`, so its `exit` is
+	* reported and rejects whatever it still held — a deliberate dispose would
+	* silence both, and this is not one.
+	*/
+	recycle() {
+		if (this.recycling || this.exited || this.process === null) return;
+		this.recycling = true;
+		if (this.recycleBackstop) clearTimeout(this.recycleBackstop);
+		terminateChild(this.process, POOL_WORKER_TERM_GRACE_MS);
 	}
 	rejectAll(err) {
 		const entries = [...this.pending.values()];
@@ -1290,8 +1749,10 @@ var SharedInferencePool = class {
 		this.tuning = options.tuning ?? null;
 		this.numWorkers = Math.max(1, options.numWorkers ?? 1);
 		this.device = options.device;
+		this.onCompileFinishedLate = options.onCompileFinishedLate;
 	}
 	device;
+	onCompileFinishedLate;
 	/** Pid of the first worker (for legacy callers). Use `getPids()` for all. */
 	/** Summed backlog and shed count across the pool's workers. A rising
 	*  `inFlight` with a rising `shed` is a worker falling behind; a rising
@@ -1330,7 +1791,8 @@ var SharedInferencePool = class {
 			tuning: this.tuning,
 			logger: this.log,
 			workerLabel: `w${i}`,
-			...this.device ? { device: this.device } : {}
+			...this.device ? { device: this.device } : {},
+			...this.onCompileFinishedLate ? { onCompileFinishedLate: this.onCompileFinishedLate } : {}
 		}));
 		const t0 = performance.now();
 		const results = await Promise.all(this.workers.map((w) => w.initialize(initialModels)));
@@ -1425,7 +1887,7 @@ var SharedInferencePool = class {
 			index,
 			config: serializeModelConfig(config)
 		})));
-		for (const resp of responses) if (resp.status !== "ok") throw new Error(`Failed to load model at index ${index}: ${resp.error ?? "unknown"}`);
+		for (const resp of responses) if (resp.status !== "ok") throw new PoolModelLoadError(`Failed to load model at index ${index}: ${describeCommandFailure(resp)}`, resp.reason ?? null, resp.modelId ?? null);
 		if (index >= this.nextFreeIndex) this.nextFreeIndex = index + 1;
 		return { loadMs: Math.max(...responses.map((r) => r.loadMs ?? 0)) };
 	}
@@ -1461,7 +1923,7 @@ var SharedInferencePool = class {
 			index,
 			config: serializeModelConfig(config)
 		})));
-		for (const resp of responses) if (resp.status !== "ok") throw new Error(`Failed to replace model at index ${index}: ${resp.error ?? "unknown"}`);
+		for (const resp of responses) if (resp.status !== "ok") throw new PoolModelLoadError(`Failed to replace model at index ${index}: ${describeCommandFailure(resp)}`, resp.reason ?? null, resp.modelId ?? null);
 		return { loadMs: Math.max(...responses.map((r) => r.loadMs ?? 0)) };
 	}
 	/**
@@ -1511,9 +1973,35 @@ var SharedInferencePool = class {
 	allocateIndex() {
 		return this.nextFreeIndex++;
 	}
+	/**
+	* Give back an index whose load FAILED, so the retry reuses it. Only the most
+	* recent allocation can be returned. Loads into one pool are serialised by
+	* the provider, so a failed load is normally the latest one; anything else
+	* is left allocated rather than risk handing out a live slot twice.
+	*/
+	releaseIndex(index) {
+		if (index === this.nextFreeIndex - 1) this.nextFreeIndex = index;
+	}
 	isReady() {
 		return this.workers.length > 0 && this.workers.every((w) => w.isReady());
 	}
+	/**
+	* Why a worker of this pool was declared unusable while alive (D653), or
+	* `null`. The provider charges the restart budget with THIS — or, for a
+	* `compile-hung` death, does not charge the device at all — so the
+	* `inference device FAILED` line names the cause instead of "pool worker is
+	* not ready".
+	*/
+	getDeathCause() {
+		for (const w of this.workers) {
+			const cause = w.getDeathCause();
+			if (cause !== null) return {
+				...cause,
+				message: `worker ${cause.message}`
+			};
+		}
+		return null;
+	}
 	async dispose() {
 		await Promise.all(this.workers.map((w) => w.dispose()));
 		this.workers.length = 0;
@@ -1578,6 +2066,23 @@ var SharedInferencePool = class {
 		return found;
 	}
 };
+/** A failed command's reply as one message: its reason class first, when it has one. */
+function describeCommandFailure(resp) {
+	const error = resp.error ?? "unknown";
+	return resp.reason !== void 0 ? `${resp.reason}: ${error}` : error;
+}
+/** Name a command for the lines that must say which one hung (D653). */
+function describeCommand(cmd) {
+	const name = typeof cmd["cmd"] === "string" ? cmd["cmd"] : "unknown";
+	const index = cmd["index"];
+	const config = cmd["config"];
+	const modelPath = typeof config === "object" && config !== null && "path" in config ? config.path : void 0;
+	return {
+		cmd: name,
+		modelIndex: typeof index === "number" ? index : null,
+		model: typeof modelPath === "string" && modelPath.length > 0 ? node_path.basename(modelPath, node_path.extname(modelPath)) : null
+	};
+}
 function serializeModelConfig(config) {
 	const result = {
 		path: config.path,
@@ -1865,6 +2370,28 @@ function poolDecodeSettingsEqual(a, b) {
 }
 //#endregion
 //#region src/detection-pipeline/engine/pipeline-model-manager.ts
+/**
+* A (step, model) load the pool rejected — carrying the pool's failure class
+* (`compile-timeout`, …) so the provider can count a timeout against THAT
+* model without parsing a message (D653).
+*/
+var StepVariantLoadError = class extends Error {
+	stepId;
+	modelId;
+	poolIndex;
+	reason;
+	/** The model's file stem as the pool names it (`camstack-yunet-2023mar`). */
+	poolModel;
+	constructor(stepId, modelId, poolIndex, cause) {
+		super(cause instanceof Error ? cause.message : String(cause));
+		this.name = "StepVariantLoadError";
+		this.stepId = stepId;
+		this.modelId = modelId;
+		this.poolIndex = poolIndex;
+		this.reason = cause instanceof PoolModelLoadError ? cause.reason : null;
+		this.poolModel = cause instanceof PoolModelLoadError ? cause.model : null;
+	}
+};
 var PipelineModelManager = class {
 	pool;
 	source;
@@ -1876,11 +2403,20 @@ var PipelineModelManager = class {
 	lruClock = 0;
 	log;
 	maxModelsPerStep;
+	deviceKey;
+	/**
+	* Loads issued and not yet answered, keyed `stepId::modelId`. A second
+	* caller for the same pair JOINS the pending load instead of issuing its own
+	* (D653): on 2026-09-26 43 identical YuNet loads queued behind one hung GPU
+	* compile, each allocating a pool index of its own.
+	*/
+	pendingLoads = /* @__PURE__ */ new Map();
 	constructor(pool, source, logger, options) {
 		this.pool = pool;
 		this.source = source;
 		this.log = logger;
 		this.maxModelsPerStep = options?.maxModelsPerStep ?? 4;
+		this.deviceKey = options?.deviceKey ?? null;
 	}
 	/**
 	* Apply a new pipeline configuration — driven by the runtime config
@@ -2030,6 +2566,23 @@ var PipelineModelManager = class {
 			await this.reconcileDecode(existing, settings);
 			return existing;
 		}
+		const key = `${stepId}::${modelId}`;
+		const pending = this.pendingLoads.get(key);
+		if (pending) {
+			const joined = await pending;
+			await this.reconcileDecode(joined, settings);
+			return joined;
+		}
+		const load = this.issueLoad(perStep, stepId, modelId, settings);
+		this.pendingLoads.set(key, load);
+		try {
+			return await load;
+		} finally {
+			this.pendingLoads.delete(key);
+		}
+	}
+	/** Evict if at capacity, then send ONE load command and record the result. */
+	async issueLoad(perStep, stepId, modelId, settings) {
 		while (perStep.size >= this.maxModelsPerStep) {
 			const evicted = this.pickEvictionTarget(stepId);
 			if (!evicted) break;
@@ -2041,16 +2594,31 @@ var PipelineModelManager = class {
 				cap: this.maxModelsPerStep
 			} });
 		}
-		const index = this.pool.allocateIndex();
 		const config = this.source.buildConfig(stepId, modelId, settings);
 		const decode = poolDecodeSettingsOf(config);
+		const index = this.pool.allocateIndex();
 		this.log.info("Loading step variant", { meta: {
 			step: stepId,
 			modelId,
 			poolIndex: index,
+			deviceKey: this.deviceKey,
 			...decode
 		} });
-		const { loadMs } = await this.pool.loadModel(index, config);
+		let loadMs;
+		try {
+			({loadMs} = await this.pool.loadModel(index, config));
+		} catch (err) {
+			this.pool.releaseIndex(index);
+			this.log.error("Step variant load failed", { meta: {
+				step: stepId,
+				modelId,
+				poolIndex: index,
+				deviceKey: this.deviceKey,
+				reason: err instanceof PoolModelLoadError ? err.reason : null,
+				error: err instanceof Error ? err.message : String(err)
+			} });
+			throw new StepVariantLoadError(stepId, modelId, index, err);
+		}
 		this.log.info("Step variant loaded", { meta: {
 			step: stepId,
 			modelId,
@@ -2420,6 +2988,15 @@ var EngineFactory = class {
 	isReady() {
 		return this.pool?.isReady() ?? false;
 	}
+	/**
+	* Why this factory's pool stopped being usable while its process was alive —
+	* a compile that never returned, or a worker that answered nothing (D653) —
+	* or `null`. A pool that merely crashed says nothing here; its `Worker
+	* process exited` line already names the signal.
+	*/
+	getDeathCause() {
+		return this.pool?.getDeathCause() ?? null;
+	}
 	/** Native pid of the underlying Python pool, if any. */
 	getPoolPid() {
 		return this.pool?.getPid() ?? null;
@@ -2546,12 +3123,13 @@ var EngineFactory = class {
 			concurrency,
 			tuning: resolvedTuning,
 			numWorkers,
-			...this.opts.engine.device ? { device: this.opts.engine.device } : {}
+			...this.opts.engine.device ? { device: this.opts.engine.device } : {},
+			...this.opts.onCompileFinishedLate ? { onCompileFinishedLate: this.opts.onCompileFinishedLate } : {}
 		});
 		this.poolManager = new PipelineModelManager(this.pool, {
 			buildConfig: (stepId, modelId, settings) => this.buildPoolModelConfig(stepId, modelId, poolRuntime, settings),
 			resolveDecode: (stepId, modelId, settings) => this.resolveDecode(stepId, modelId, settings)
-		}, this.log.child("model-mgr"));
+		}, this.log.child("model-mgr"), { deviceKey: this.deviceKey });
 		await this.pool.initialize([]);
 		await this.poolManager.applyConfig(steps);
 	}
@@ -2667,6 +3245,175 @@ function buildPoolModelConfigForStep(inputs) {
 	};
 }
 //#endregion
+//#region src/detection-pipeline/engine/model-load-governor.ts
+/**
+* What the dispatch path may ask of a pool's model loads, and what it must say
+* when it may not (D653, fix round 1).
+*
+* Three pieces of state, consulted ON DEMAND by the dispatch that needs a
+* model — there is no timer anywhere in here:
+*
+*  1. A NEGATIVE CACHE per (pool, model). `needsPoolUpdate` stays true after a
+*     failed load, so without it every frame of every camera re-issued the
+*     failing load and wrote two ERRORs. After a failure the model is not
+*     re-issued on that pool for an interval that doubles per consecutive
+*     failure up to a cap; a success clears it.
+*  2. COMPILE TIMEOUTS per (device, model). A load that outlived its soft
+*     bound is not a crash of the device. The SAME model timing out
+*     {@link COMPILE_TIMEOUTS_BEFORE_REFUSAL} times on one device — which,
+*     since a slow compile now runs on and writes its cache, only a compile
+*     that never finishes can do — refuses THAT MODEL there, by name. The
+*     device keeps serving every other model.
+*  3. Which camera has already been TOLD about the current state of a model,
+*     so each camera gets one line per state change, never one per frame.
+*/
+/** The first back-off after a failed load. */
+var MODEL_LOAD_BACKOFF_INITIAL_MS = 5e3;
+/** The back-off doubles per consecutive failure up to this. */
+var MODEL_LOAD_BACKOFF_MAX_MS = 5 * 6e4;
+/** Timeouts older than this no longer count toward a refusal. */
+var COMPILE_TIMEOUT_WINDOW_MS = 60 * 6e4;
+var ModelLoadGovernor = class {
+	/** Keyed by the pool OBJECT, so a respawned pool starts clean. */
+	backoff = /* @__PURE__ */ new WeakMap();
+	timeouts = /* @__PURE__ */ new Map();
+	reported = /* @__PURE__ */ new Map();
+	generation = 0;
+	/** May this dispatch load `models` on `pool` (a device `deviceKey`) now? */
+	check(pool, deviceKey, models, now = Date.now()) {
+		for (const model of models) {
+			const t = this.timeouts.get(timeoutKey(deviceKey, model));
+			if (t?.refused === true) return {
+				kind: "refused",
+				model,
+				timeouts: t.at.length,
+				error: t.lastError
+			};
+		}
+		const perPool = this.backoff.get(pool);
+		if (perPool === void 0) return { kind: "go" };
+		for (const model of models) {
+			const b = perPool.get(model);
+			if (b !== void 0 && now < b.retryAtMs) return {
+				kind: "backoff",
+				model,
+				retryInMs: b.retryAtMs - now,
+				failures: b.failures,
+				error: b.error,
+				generation: b.generation
+			};
+		}
+		return { kind: "go" };
+	}
+	/** One load of `model` on `pool` failed. */
+	recordFailure(pool, deviceKey, model, error, compileTimeout, now = Date.now(), poolModel = null, reason = null) {
+		let perPool = this.backoff.get(pool);
+		if (perPool === void 0) {
+			perPool = /* @__PURE__ */ new Map();
+			this.backoff.set(pool, perPool);
+		}
+		const previousEntry = perPool.get(model);
+		const failures = (previousEntry?.failures ?? 0) + 1;
+		const retryInMs = Math.min(MODEL_LOAD_BACKOFF_MAX_MS, MODEL_LOAD_BACKOFF_INITIAL_MS * 2 ** (failures - 1));
+		this.generation += 1;
+		const generation = this.generation;
+		perPool.set(model, {
+			failures,
+			retryAtMs: now + retryInMs,
+			error,
+			generation,
+			poolModel: poolModel ?? previousEntry?.poolModel ?? null,
+			reason
+		});
+		if (!compileTimeout) return {
+			generation,
+			retryInMs,
+			refusedNow: false,
+			compileTimeouts: 0
+		};
+		const key = timeoutKey(deviceKey, model);
+		const previous = this.timeouts.get(key);
+		const at = [...(previous?.at ?? []).filter((t) => t >= now - COMPILE_TIMEOUT_WINDOW_MS), now];
+		const refused = at.length >= 2;
+		this.timeouts.set(key, {
+			at,
+			refused,
+			lastError: error
+		});
+		return {
+			generation,
+			retryInMs,
+			refusedNow: refused && previous?.refused !== true,
+			compileTimeouts: at.length
+		};
+	}
+	/**
+	* A compile that outlived its bound came back on `pool` (D653 round 3).
+	*
+	* `ok`: THAT model's cache is written and the worker loads again, so its
+	* back-off is lifted — and only its: another model's failure on the same
+	* pool is not the compile that just finished. Not ok: it is one more
+	* failure of that model, and its back-off grows. Refusals by name are never
+	* lifted here; a model refused for timing out twice stays refused until the
+	* operator re-arms. Returns the governor keys it touched.
+	*/
+	settleLateCompile(pool, deviceKey, poolModel, ok, error, now = Date.now()) {
+		const perPool = this.backoff.get(pool);
+		if (perPool === void 0) return [];
+		const refusedMeanwhile = [...perPool].filter(([, b]) => b.reason === "worker-poisoned" && b.poolModel !== poolModel).map(([key]) => key);
+		for (const key of refusedMeanwhile) perPool.delete(key);
+		const own = poolModel === null ? [] : [...perPool].filter(([, b]) => b.poolModel === poolModel).map(([key]) => key);
+		for (const key of own) if (ok) perPool.delete(key);
+		else this.recordFailure(pool, deviceKey, key, error, false, now, poolModel);
+		return [...refusedMeanwhile, ...own];
+	}
+	/** `models` loaded on `pool`: forget their failures there. */
+	recordSuccess(pool, deviceKey, models) {
+		const perPool = this.backoff.get(pool);
+		for (const model of models) {
+			perPool?.delete(model);
+			this.timeouts.delete(timeoutKey(deviceKey, model));
+		}
+	}
+	/**
+	* Has `camera` been told about this `state` of `model` on `deviceKey` yet?
+	* Returns `true` exactly once per state change.
+	*/
+	shouldReport(camera, deviceKey, model, state) {
+		const key = `${camera ?? "-"}|${deviceKey}|${model}`;
+		if (this.reported.get(key) === state) return false;
+		this.reported.set(key, state);
+		return true;
+	}
+	/** Every model refused on some device. */
+	refused() {
+		const out = [];
+		for (const [key, t] of this.timeouts) {
+			if (!t.refused) continue;
+			const [deviceKey = "", model = ""] = key.split("\0");
+			out.push({
+				deviceKey,
+				model,
+				timeouts: t.at.length,
+				lastError: t.lastError
+			});
+		}
+		return out;
+	}
+	/** Operator re-arm: forget the refusals on `deviceKey`. Returns how many. */
+	rearm(deviceKey) {
+		let cleared = 0;
+		for (const key of [...this.timeouts.keys()]) if (key.startsWith(`${deviceKey}\u0000`)) {
+			this.timeouts.delete(key);
+			cleared += 1;
+		}
+		return cleared;
+	}
+};
+function timeoutKey(deviceKey, model) {
+	return `${deviceKey}\u0000${model}`;
+}
+//#endregion
 //#region src/detection-pipeline/engine/idle-pool-reaper.ts
 var IdlePoolReaper = class {
 	lastUsed = /* @__PURE__ */ new Map();
@@ -6563,6 +7310,34 @@ function parseWavToAudioChunk(filePath) {
 function enginesEqual(a, b) {
 	return a.runtime === b.runtime && a.backend === b.backend && a.format === b.format && (a.device ?? null) === (b.device ?? null);
 }
+/** A dispatch's load refused before it was issued: a back-off, or a refusal by name. */
+var ModelLoadRefusedError = class extends Error {
+	model;
+	state;
+	constructor(message, model, state) {
+		super(message);
+		this.name = "ModelLoadRefusedError";
+		this.model = model;
+		this.state = state;
+	}
+};
+/** `step/model` — how the governor and the log lines name a model. */
+function stepModelKey(step) {
+	return `${step.addonId}/${step.modelId}`;
+}
+/** The reporting state a refusal verdict stands for. */
+function stateOf(verdict) {
+	return verdict.kind === "refused" ? "refused" : `failed:${verdict.generation}`;
+}
+/**
+* The reason a condemned pool is charged against its device's restart budget.
+* A pool recycled because it HUNG (D653) names the hang — `inference device
+* FAILED … reason: pool worker is not ready` could not tell a GPU compile that
+* never returned from a crash.
+*/
+function deathReasonOf(factory) {
+	return factory.getDeathCause()?.message ?? "pool worker is not ready";
+}
 /** Build a `RuntimeEnv` from the running process + probed hardware. */
 function runtimeEnvFromProcess(hardware) {
 	return {
@@ -8040,7 +8815,10 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				addonId: s.addonId,
 				modelId: s.modelId
 			});
-			await this.ensureModelsForSteps(benchmarkSteps, dispatchFactory, dispatchEngine.format);
+			await this.ensureModelsForSteps(benchmarkSteps, dispatchFactory, dispatchEngine.format, {
+				...input.deviceId !== void 0 ? { deviceId: input.deviceId } : {},
+				deviceKey: deviceKeyOf(dispatchEngine)
+			});
 			emit(`All models loaded`);
 		}
 		emit("Running inference...");
@@ -8337,53 +9115,193 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		throw new Error(message);
 	}
 	/**
-	* Single-flight gate around `ensureModelsForSteps`. Concurrent
-	* callers (every camera that fires motion in the same window calls
-	* `runPipeline` → `ensureModelsForSteps`) share the same in-flight
-	* promise instead of each racing into `loadAdditional`. Without this
-	* the pool gets the SAME model loaded N times under N distinct
-	* pool indices because every caller saw `isLoadedWithModel === false`
-	* before any of them committed via `stepToLoaded.set`.
-	*/
-	modelLoadInFlight = null;
+	* In-flight model loads, per POOL and per MODEL SET (D653). Two dispatches
+	* needing the same models on the same pool share ONE load and its outcome;
+	* a dispatch needing a different set is never handed another set's failure
+	* — it queues behind the pool's current load ({@link modelLoadTail}) and
+	* then decides for itself. Keyed by pool, not node-wide: one GPU compile
+	* that never returned held EVERY load on the node — the NPU's and the CPU's
+	* too — behind it.
+	*/
+	modelLoadInFlight = /* @__PURE__ */ new Map();
+	/** The last load issued on each pool — loads into one pool run one at a time. */
+	modelLoadTail = /* @__PURE__ */ new Map();
+	/** Stands in for "the node-default pool" before it exists. */
+	nullPoolKey = {};
+	/** Negative cache, per-model compile-timeout refusals, per-camera reporting (D653). */
+	loadGovernor = new ModelLoadGovernor();
 	/** Ensure all models needed by steps are downloaded and loaded in the engine pool.
 	* `factory`/`format` default to the node's engine; a per-device dispatch passes
-	* that device's factory + format (Phase 2 multi-device). */
-	async ensureModelsForSteps(steps, factory = this.engineFactory, format = this.currentEngine?.format ?? "onnx") {
-		while (this.modelLoadInFlight) await this.modelLoadInFlight;
-		const allEnabled = flattenSteps(steps);
+	* that device's factory + format (Phase 2 multi-device). `context` names the
+	* camera and the accelerator on the line a rejected load writes. */
+	async ensureModelsForSteps(steps, factory = this.engineFactory, format = this.currentEngine?.format ?? "onnx", context = {}) {
+		const needed = this.stepsNeedingLoad(steps, factory);
+		if (needed.length === 0) return;
+		const pool = factory ?? this.nullPoolKey;
+		const deviceKey = context.deviceKey ?? deviceKeyOf(this.currentEngine);
+		const models = needed.map(stepModelKey);
+		const refusal = this.loadRefusal(pool, deviceKey, models);
+		if (refusal !== null) {
+			this.reportDispatchLoadLoss(context, deviceKey, models, refusal.model, refusal.state, refusal.message);
+			throw refusal;
+		}
+		const setKey = [...models].sort().join(",");
+		let perPool = this.modelLoadInFlight.get(pool);
+		if (perPool === void 0) {
+			perPool = /* @__PURE__ */ new Map();
+			this.modelLoadInFlight.set(pool, perPool);
+		}
+		const flight = perPool.get(setKey) ?? this.issueModelLoad(pool, perPool, setKey, steps, factory, format, deviceKey);
+		try {
+			await flight.work;
+		} catch (err) {
+			if (err instanceof ModelLoadRefusedError) {
+				this.reportDispatchLoadLoss(context, deviceKey, models, err.model, err.state, err.message);
+				throw err;
+			}
+			const failed = err instanceof StepVariantLoadError ? `${err.stepId}/${err.modelId}` : models[0] ?? "";
+			this.reportDispatchLoadLoss(context, deviceKey, models, failed, `failed:${flight.generation ?? "unknown"}`, require_dist.errMsg(err));
+			throw err;
+		}
+	}
+	/**
+	* Why `models` may not be loaded on `pool` right now — a back-off after a
+	* failure, or a refusal by name after repeated compile timeouts — or `null`.
+	*/
+	loadRefusal(pool, deviceKey, models) {
+		const verdict = this.loadGovernor.check(pool, deviceKey, models);
+		if (verdict.kind === "go") return null;
+		return new ModelLoadRefusedError(verdict.kind === "refused" ? `model ${verdict.model} is refused on ${deviceKey}: its compile timed out ${verdict.timeouts} times (${verdict.error}) — re-arm with pipelineExecutor.rearmInferenceDevice` : `model ${verdict.model} failed to load on ${deviceKey} ${verdict.failures} time(s); not retried for ${verdict.retryInMs}ms (${verdict.error})`, verdict.model, stateOf(verdict));
+	}
+	/**
+	* A compile that outlived its soft bound came back on `factory`'s pool
+	* (D653 rounds 2-3). A SUCCESS wrote that model's cache and the worker loads
+	* again, so that model's back-off is lifted at once rather than waited out;
+	* no other model's is. A FAILURE is one more failure of that model, and its
+	* back-off grows.
+	*/
+	noteLateCompile(factory, deviceKey, event) {
+		const models = this.loadGovernor.settleLateCompile(factory, deviceKey, event.model, event.ok, event.error ?? "late compile failed");
+		if (event.ok) {
+			this.log.info("model loadable again after late compile", { meta: {
+				deviceKey,
+				model: event.model,
+				backoffLifted: models
+			} });
+			return;
+		}
+		this.log.warn("late compile failed — the model stays backed off", { meta: {
+			deviceKey,
+			model: event.model,
+			backoffExtended: models,
+			error: event.error ?? null
+		} });
+	}
+	/** The enabled steps the factory's pool does not reflect yet. */
+	stepsNeedingLoad(steps, factory) {
 		const needed = [];
-		for (const step of allEnabled) {
+		for (const step of flattenSteps(steps)) {
 			if (!step.enabled) continue;
 			if (factory !== null && !factory.needsPoolUpdate(step)) continue;
 			needed.push(step);
 		}
-		if (needed.length === 0) return;
-		const work = (async () => {
-			for (const step of needed) {
-				const modelEntry = await this.resolveModelEntry(step.addonId, step.modelId);
-				if (modelEntry && !(0, _camstack_system_addon_utils.isModelDownloaded)(this.modelsDir, modelEntry, format)) {
-					if (modelEntry.formats[format] === void 0) throw new Error(`Model "${modelEntry.id}" has no ${format} format build (available: ${Object.keys(modelEntry.formats).join(", ") || "none"}) — not retrying a permanent format mismatch`);
-					if (modelEntry.formats[format]?.url.startsWith("camstack-local://") === true) throw new Error(`Custom model "${modelEntry.id}" (${format}) is not present in this node's models directory — distribute it to this node from Model Studio before selecting it here`);
-					this.log.info("Downloading model for step", { meta: {
-						modelId: step.modelId,
-						format,
-						step: step.addonId
-					} });
-					await this.downloadWithRetry(modelEntry, format, 3);
-				}
+		return needed;
+	}
+	/**
+	* Issue ONE load for a model set on a pool, queued behind the pool's
+	* previous load, and record its outcome with the governor. The set is
+	* re-derived once the queue reaches it: the load ahead may have brought
+	* some of it in.
+	*/
+	issueModelLoad(pool, perPool, setKey, steps, factory, format, deviceKey) {
+		const previous = this.modelLoadTail.get(pool) ?? Promise.resolve();
+		const flight = {
+			work: Promise.resolve(),
+			generation: null
+		};
+		flight.work = (async () => {
+			await previous;
+			const needed = this.stepsNeedingLoad(steps, factory);
+			if (needed.length === 0) return;
+			const models = needed.map(stepModelKey);
+			const refusal = this.loadRefusal(pool, deviceKey, models);
+			if (refusal !== null) throw refusal;
+			try {
+				await this.downloadNeededModels(needed, format);
+				this.log.info("Loading additional models for benchmark", { meta: {
+					count: needed.length,
+					models,
+					deviceKey
+				} });
+				await factory.loadAdditional(needed);
+			} catch (err) {
+				flight.generation = this.recordModelLoadFailure(pool, deviceKey, models, err);
+				throw err;
 			}
-			this.log.info("Loading additional models for benchmark", { meta: {
-				count: needed.length,
-				models: needed.map((s) => `${s.addonId}/${s.modelId}`)
-			} });
-			await factory.loadAdditional(needed);
+			this.loadGovernor.recordSuccess(pool, deviceKey, models);
 		})();
-		this.modelLoadInFlight = work;
-		try {
-			await work;
-		} finally {
-			if (this.modelLoadInFlight === work) this.modelLoadInFlight = null;
+		perPool.set(setKey, flight);
+		const tail = flight.work.catch(() => void 0);
+		this.modelLoadTail.set(pool, tail);
+		tail.then(() => {
+			if (perPool.get(setKey) === flight) perPool.delete(setKey);
+			if (this.modelLoadTail.get(pool) === tail) this.modelLoadTail.delete(pool);
+		});
+		return flight;
+	}
+	/** Charge a failed load to the model that failed; say so once if it is now refused. */
+	recordModelLoadFailure(pool, deviceKey, models, err) {
+		const failedModels = err instanceof StepVariantLoadError ? [`${err.stepId}/${err.modelId}`] : models;
+		const compileTimeout = err instanceof StepVariantLoadError && err.reason === "compile-timeout";
+		const poolModel = err instanceof StepVariantLoadError ? err.poolModel : null;
+		let generation = 0;
+		for (const model of failedModels) {
+			const outcome = this.loadGovernor.recordFailure(pool, deviceKey, model, require_dist.errMsg(err), compileTimeout, Date.now(), poolModel, err instanceof StepVariantLoadError ? err.reason : null);
+			generation = outcome.generation;
+			if (outcome.refusedNow) this.log.error("model REFUSED on this device — its compile timed out repeatedly", { meta: {
+				deviceKey,
+				model,
+				compileTimeouts: outcome.compileTimeouts,
+				error: require_dist.errMsg(err),
+				device: "keeps serving every other model",
+				rearm: "pipelineExecutor.rearmInferenceDevice"
+			} });
+		}
+		return generation;
+	}
+	/**
+	* One WARN per camera per state change of the model that cost it its frame.
+	* The ERROR for the failed load itself is written once, where the load
+	* failed (`Step variant load failed`); this line answers "which cameras
+	* did it cost?", tagged so it can be counted per camera.
+	*/
+	reportDispatchLoadLoss(context, deviceKey, models, model, state, error) {
+		if (!this.loadGovernor.shouldReport(context.deviceId, deviceKey, model, state)) return;
+		this.log.warn("model load for dispatch failed", {
+			...context.deviceId !== void 0 ? { tags: { deviceId: context.deviceId } } : {},
+			meta: {
+				deviceKey,
+				models,
+				model,
+				state,
+				error
+			}
+		});
+	}
+	/** Download any model of `needed` missing in `format` (the device's). */
+	async downloadNeededModels(needed, format) {
+		for (const step of needed) {
+			const modelEntry = await this.resolveModelEntry(step.addonId, step.modelId);
+			if (modelEntry && !(0, _camstack_system_addon_utils.isModelDownloaded)(this.modelsDir, modelEntry, format)) {
+				if (modelEntry.formats[format] === void 0) throw new Error(`Model "${modelEntry.id}" has no ${format} format build (available: ${Object.keys(modelEntry.formats).join(", ") || "none"}) — not retrying a permanent format mismatch`);
+				if (modelEntry.formats[format]?.url.startsWith("camstack-local://") === true) throw new Error(`Custom model "${modelEntry.id}" (${format}) is not present in this node's models directory — distribute it to this node from Model Studio before selecting it here`);
+				this.log.info("Downloading model for step", { meta: {
+					modelId: step.modelId,
+					format,
+					step: step.addonId
+				} });
+				await this.downloadWithRetry(modelEntry, format, 3);
+			}
 		}
 	}
 	/** Download a model with retry + exponential backoff */
@@ -8511,7 +9429,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		await this.ensureEngineFactory();
 		const factory = dispatchDeviceKey ? await this.resolveDeviceFactory(dispatchDeviceKey) : this.engineFactory;
 		if (!factory) throw new Error("runPipelineBatch: factory not initialised");
-		if (enabledSteps.filter((s) => factory.needsPoolUpdate(s)).length > 0) await this.ensureModelsForSteps(benchmarkSteps, factory, resolveFormat);
+		if (enabledSteps.filter((s) => factory.needsPoolUpdate(s)).length > 0) await this.ensureModelsForSteps(benchmarkSteps, factory, resolveFormat, { deviceKey: dispatchDeviceKey ?? deviceKeyOf(this.currentEngine) });
 		const canFastPath = singleRoot && allRaw && uniformDims && factory.supportsBatch();
 		this.log.info("runPipelineBatch path decision", { meta: {
 			phase: "batch",
@@ -8766,7 +9684,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 			if (this.engineFactory.isReady()) return;
 			const dead = this.engineFactory;
 			this.engineFactory = null;
-			this.noteDeviceDeath(defaultDeviceKey, "pool worker is not ready");
+			this.noteFactoryDeath(defaultDeviceKey, dead);
 			await dead.dispose().catch(() => void 0);
 			this.refuseIfDeviceUnusable(defaultDeviceKey);
 		}
@@ -8784,7 +9702,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				logger: this.log.child("engine"),
 				pythonPath: this.executorOptions.pythonPath ?? "",
 				provisioning: this.executorOptions.provisioning,
-				resolveCustomModel: this.customModelResolver
+				resolveCustomModel: this.customModelResolver,
+				onCompileFinishedLate: (event) => this.noteLateCompile(factory, defaultDeviceKey, event)
 			});
 			try {
 				await factory.initialize([]);
@@ -8904,15 +9823,35 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* dispose — which rejects whatever was still in flight on it with a reason
 	* rather than letting those requests sit until their own deadlines.
 	*/
-	async condemnDeviceFactory(deviceKey, factory, reason) {
+	async condemnDeviceFactory(deviceKey, factory) {
 		if (this.factoriesByDevice.get(deviceKey) === factory) {
 			this.factoriesByDevice.delete(deviceKey);
 			this.deviceReaper.cancel(deviceKey);
-			this.noteDeviceDeath(deviceKey, reason);
+			this.noteFactoryDeath(deviceKey, factory);
 		}
 		await factory.dispose().catch(() => void 0);
 	}
 	/**
+	* Charge one dead pool to its device — unless it died of a compile that was
+	* still running at its hard bound (D653, fix round 1). That death is not a
+	* crash of the DEVICE: the model that hung was already counted when its
+	* load answered `compile-timeout`, and the same model timing out again on
+	* the respawn is refused by name (`ModelLoadGovernor`), which is what bounds
+	* the respawns. A crash stays a crash.
+	*/
+	noteFactoryDeath(deviceKey, factory) {
+		const cause = factory.getDeathCause();
+		if (cause !== null && !cause.chargesDeviceBudget) {
+			this.log.warn("inference pool recycled after a compile hung — not charged to the device budget", { meta: {
+				deviceKey,
+				reason: cause.reason,
+				cause: cause.message
+			} });
+			return;
+		}
+		this.noteDeviceDeath(deviceKey, cause?.message ?? "pool worker is not ready");
+	}
+	/**
 	* Every inference device this node currently refuses, and why.
 	*
 	* The CHANNEL the 2026-08-26 analysis found missing. Pool health lived
@@ -8947,7 +9886,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				state: "backoff",
 				since: Date.now(),
 				deaths: 0,
-				lastError: "pool worker is not ready"
+				lastError: deathReasonOf(factory)
 			});
 		}
 		return { unhealthy: [...byKey.values()].toSorted((a, b) => a.deviceKey.localeCompare(b.deviceKey)) };
@@ -8964,10 +9903,13 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* no-op, reported as such.
 	*/
 	async rearmInferenceDevice(input) {
-		const rearmed = this.deviceLiveness.rearm(input.deviceKey);
+		const rearmedDevice = this.deviceLiveness.rearm(input.deviceKey);
+		const rearmedModels = this.loadGovernor.rearm(input.deviceKey);
+		const rearmed = rearmedDevice || rearmedModels > 0;
 		this.log.info("inference device re-armed by operator", { meta: {
 			deviceKey: input.deviceKey,
-			rearmed
+			rearmed,
+			rearmedModels
 		} });
 		return { rearmed };
 	}
@@ -8984,7 +9926,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				this.deviceReaper.touch(deviceKey);
 				return existing;
 			}
-			await this.condemnDeviceFactory(deviceKey, existing, "pool worker is not ready");
+			await this.condemnDeviceFactory(deviceKey, existing);
 			this.refuseIfDeviceUnusable(deviceKey);
 		}
 		const inflight = this.deviceFactoryInflight.get(deviceKey);
@@ -8997,7 +9939,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				logger: this.log.child(`engine:${deviceKey}`),
 				pythonPath: this.executorOptions.pythonPath ?? "",
 				provisioning: this.executorOptions.provisioning,
-				resolveCustomModel: this.customModelResolver
+				resolveCustomModel: this.customModelResolver,
+				onCompileFinishedLate: (event) => this.noteLateCompile(factory, deviceKey, event)
 			});
 			try {
 				await factory.initialize([]);