@camstack/addon-pipeline 1.2.294 → 1.2.296

package/dist/detection-pipeline/index.mjs

@@ -1,9 +1,9 @@
 import { n as __require } from "../chunk-DnnnRqeS.mjs";
-import { $ as defaultDeviceFor$1, $t as errMsg, A as PoolMemoryWatchdog, En as array, Et as overlayClusterStepSettings, Fn as string, Gt as runtimeDevices$1, In as union, Ln as EventCategory, Mn as object, Mt as pipelineExecutorCapability, Ot as pickClusterStepModels, Ut as resolvePoolMemoryPolicy, V as YAMNET_TO_MACRO, Xt as supportedRuntimes$1, _t as loadContributionCapability, bn as nodePin, ct as evaluateZoneRules, f as DEFAULT_CLUSTER_STEP_MODELS, fn as createEvent, gn as hydrateSchema, ht as inferModelProvider, kt as pickClusterStepSettings, on as BaseAddon, ot as enumerateInferenceDevices, p as DEFAULT_CLUSTER_STEP_SETTINGS, rt as detectionPipelineCapability, t as APPLE_SA_TO_MACRO, v as DEVICE_BACKEND_TO_FORMAT, wn as sleep, xn as parseJsonUnknown, zt as resolveClusterStepModelId } from "../dist-RXbmRAwP.mjs";
-import { s as readProcessCost } from "../node-atmRSHPk.mjs";
-import { a as localFrameRegistry, c as ALL_STEPS, d as getStepDefinition, f as resolveModelForFormat, l as getDefaultModelForFormat, m as landmarkPrecisionVerdict, n as resolveDefaultDetectionModel, r as macroGateVerdict, s as ALL_PIPELINE_STEPS, u as getStep } from "../default-detection-model-0dPKRKUD.mjs";
+import { At as pickClusterStepSettings, Cn as parseJsonUnknown, Dt as overlayClusterStepSettings, En as sleep, Gt as resolvePoolMemoryPolicy, H as YAMNET_TO_MACRO, Ln as string, Mt as pickInvalidClusterStepModels, On as array, Pn as object, Pt as pipelineExecutorCapability, Qt as supportedRuntimes$1, Rn as union, Sn as nodePin, Vt as resolveClusterStepModelId, cn as BaseAddon, et as defaultDeviceFor$1, gt as inferModelProvider, it as detectionPipelineCapability, j as PoolMemoryWatchdog, kt as pickClusterStepModels, lt as evaluateZoneRules, m as DEFAULT_CLUSTER_STEP_SETTINGS, mn as createEvent, p as DEFAULT_CLUSTER_STEP_MODELS, qt as runtimeDevices$1, st as enumerateInferenceDevices, t as APPLE_SA_TO_MACRO, tn as errMsg, vn as hydrateSchema, vt as loadContributionCapability, y as DEVICE_BACKEND_TO_FORMAT, zn as EventCategory } from "../dist-CCd0Q3nr.mjs";
+import { s as readProcessCost } from "../node-DgMSXSWP.mjs";
+import { a as localFrameRegistry, c as ALL_STEPS, d as getStepDefinition, f as resolveModelForFormat, l as getDefaultModelForFormat, m as landmarkPrecisionVerdict, n as resolveDefaultDetectionModel, r as macroGateVerdict, s as ALL_PIPELINE_STEPS, u as getStep } from "../default-detection-model-Co578D8C.mjs";
 import { t as getSharp } from "../lazy-sharp-DXsqwpph.mjs";
-import { n as pickNodePlatformArch, t as readProcessMemory } from "../process-memory-BgFOHFnx.mjs";
+import { n as pickNodePlatformArch, t as readProcessMemory } from "../process-memory-DFC_O5zE.mjs";
 import * as os from "node:os";
 import * as fs from "node:fs";
 import * as path$1 from "node:path";
@@ -415,6 +415,12 @@ var ClusterModelSource = class {
 			const next = pickClusterStepModels(view);
 			const nextSettings = pickClusterStepSettings(view);
 			this.lastReadAtMs = this.now();
+			const invalid = pickInvalidClusterStepModels(view);
+			if (invalid.length > 0) this.logger.warn("cluster step model row holds a value that is not a model id — running the catalog default for that step; no re-embed pass will run on it", { meta: {
+				owner: CLUSTER_MODEL_OWNER_ADDON_ID,
+				invalid,
+				inUse: Object.fromEntries(invalid.map((e) => [e.stepId, next[e.stepId]]))
+			} });
 			const changed = Object.keys(next).filter((stepId) => next[stepId] !== this.models[stepId]);
 			const settingsChanged = Object.keys(nextSettings).some((stepId) => {
 				const from = this.settings[stepId] ?? {};
@@ -562,6 +568,98 @@ var DeviceOverrideMirror = class {
 	}
 };
 //#endregion
+//#region src/detection-pipeline/engine/pool-worker-health.ts
+/**
+* The one table of load bounds (D653, fix round 1).
+*
+* The soft bound decides how soon the operator HEARS about a slow compile; the
+* hard bound decides when a compile is given up for hung. A single 120 s bound
+* that also killed the compile was wrong twice: a slow but finite compile was
+* killed before it could write its cache, so every respawn faced the same cold
+* compile and three of them walked the device to `failed`; and on CoreML the
+* bound was shorter than the cross-process compile-cache lock wait alone.
+*/
+var POOL_MODEL_LOAD_BOUNDS = {
+	openvino: {
+		softMs: 12e4,
+		hardMs: 6e5,
+		why: "A cold iGPU compile of the WHOLE model set measured ~25 s on the N100; one model is 1-3 s (YuNet 1.4 s on a fresh worker, 2026-09-26). 120 s is ~5x the worst measured set. No evidence of a legitimate compile past it; the hard bound gives one 5x more before the worker is recycled.",
+		gilMayBeHeldDuringCompile: false,
+		gilWhy: "The OpenVINO Python binding is believed to release the GIL around compile_model (gil_scoped_release). Not measured on the hub; the passive recipe in D653 checks it."
+	},
+	coreml: {
+		softMs: 3e5,
+		hardMs: 9e5,
+		why: "A load may first WAIT up to 180 s for another worker holding the compile-cache lock (`COMPILE_CACHE_LOCK_TIMEOUT_SEC` in inference_pool.py), then compile itself: 180 s of lock plus 120 s of compile. The old 120 s bound was shorter than the lock wait alone.",
+		gilMayBeHeldDuringCompile: true,
+		gilWhy: "Unverified whether coremltools releases the GIL while it compiles an mlpackage. If it does not, the loop freezes and nothing — not the soft timer, not mem_stats — can answer."
+	},
+	onnxruntime: {
+		softMs: 12e4,
+		hardMs: 6e5,
+		why: "Session creation, no persistent compile cache; CUDA/CoreML EP init is the slow case. Same numbers as OpenVINO for lack of any measurement saying otherwise.",
+		gilMayBeHeldDuringCompile: false,
+		gilWhy: "Not known to hold the GIL through session creation, and no frozen loop has been observed. Claiming the grace would blind stall detection during every load."
+	},
+	edgetpu: {
+		softMs: 12e4,
+		hardMs: 6e5,
+		why: "An Edge TPU model is precompiled; a load is an interpreter + delegate bind of seconds. Kept at the shared default rather than tightened without a measurement.",
+		gilMayBeHeldDuringCompile: false,
+		gilWhy: "Nothing is compiled: the load is an interpreter + delegate bind of seconds."
+	}
+};
+/**
+* The host-side deadline of a `load` / `replace`: the soft bound plus a margin,
+* so the worker's NAMED answer always lands first.
+*/
+var POOL_MODEL_LOAD_REPLY_MARGIN_MS = 3e4;
+function chargesDeviceBudget(reason) {
+	return reason !== "compile-hung";
+}
+/**
+* Tracks live inference outcomes on one worker and says when its executor has
+* produced nothing for too long. Consulted on demand — no timer.
+*
+* A SHED (`dropped`) does not count as a result: the Python loop answers sheds
+* itself, so a worker whose executor threads are hung keeps shedding happily.
+*/
+var InferStallDetector = class {
+	unanswered = 0;
+	lastResultAt;
+	/**
+	* When the current streak of unanswered requests began — the first one after
+	* a result. IDLE time is not silence: quiet cameras send nothing, and a
+	* worker that was asked nothing has failed nothing. Measuring from the last
+	* result let a 60 s lull plus the first 20 sheds of a burst (~8 s with six
+	* cameras) poison a healthy worker and charge the device.
+	*/
+	streakStartedAt = null;
+	constructor(now) {
+		this.lastResultAt = now;
+	}
+	/** A reply that carried a real result (or a real error) from the executor. */
+	noteResult(now) {
+		this.unanswered = 0;
+		this.lastResultAt = now;
+		this.streakStartedAt = null;
+	}
+	/**
+	* A live request that ended without a result (deadline expiry or a shed).
+	* Returns the stall when the worker has crossed both thresholds.
+	*/
+	noteUnanswered(now) {
+		this.unanswered += 1;
+		if (this.streakStartedAt === null) this.streakStartedAt = now;
+		const silentMs = now - Math.max(this.lastResultAt, this.streakStartedAt);
+		if (this.unanswered >= 20 && silentMs >= 3e4) return {
+			unanswered: this.unanswered,
+			silentMs
+		};
+		return null;
+	}
+};
+//#endregion
 //#region src/detection-pipeline/engine/shared-inference-pool.ts
 /**
 * SharedInferencePool — TypeScript wrapper for inference_pool.py.
@@ -603,6 +701,27 @@ var RAW_FMT_CODE = {
 	gray: 2
 };
 /**
+* A load the worker answered with a machine-readable failure class. The
+* provider tells a `compile-timeout` (counted against THAT MODEL on the
+* device) from an ordinary failure by `reason`, never by parsing text.
+*/
+var PoolModelLoadError = class extends Error {
+	reason;
+	/** The model's file stem, as the worker named it. */
+	model;
+	constructor(message, reason, model) {
+		super(message);
+		this.name = "PoolModelLoadError";
+		this.reason = reason;
+		this.model = model;
+	}
+};
+/**
+* Request id the worker uses for UNSOLICITED events — a reply to no request
+* (`compile-finished-late`). Never allocated to a request.
+*/
+var WORKER_EVENT_REQ_ID = 4294967295;
+/**
 * Per-inference-request reply timeout (ms). Turns a wedged request (worker
 * alive but no reply) into a rejection so every caller settles — runtime frame
 * dispatch drops the frame; the benchmark full-tree run rejects the wedged
@@ -611,6 +730,22 @@ var RAW_FMT_CODE = {
 * large frame) never trips; override via env for constrained hardware.
 */
 var POOL_INFER_TIMEOUT_MS = Math.max(1e3, Number(process.env["CAMSTACK_POOL_INFER_TIMEOUT_MS"]) || 6e4);
+/** Commands that compile a model — the only ones that get the load deadline. */
+var MODEL_LOAD_COMMANDS = new Set(["load", "replace"]);
+/**
+* Deadline of the liveness probe (`mem_stats`) sent after a command times out.
+* The worker answers `mem_stats` on its event loop, never behind a compile
+* (D653), so a healthy worker replies in milliseconds even mid-compile; ten
+* seconds only has to outlast a GC pause or a burst of inference replies.
+*/
+var POOL_LIVENESS_PROBE_TIMEOUT_MS = 1e4;
+/**
+* How long a poisoned worker may keep its in-flight LIVE requests before it is
+* killed anyway. A poisoned worker takes no new work, and every live request
+* carries a {@link POOL_LIVE_INFER_TIMEOUT_MS} deadline, so the drain is over
+* by then; the second of slack keeps the backstop from racing the last reply.
+*/
+var POOL_POISON_DRAIN_SLACK_MS = 1e3;
 /**
 * Max inference requests outstanding to ONE worker before new ones are SHED.
 *
@@ -798,6 +933,27 @@ var PoolWorker = class {
 	wireVersion = 1;
 	nextRequestId = 1;
 	ready = false;
+	/** Set once this worker is declared unusable while ALIVE (D653). Never cleared:
+	*  a poisoned worker is recycled, not revived. */
+	poisonVerdict = null;
+	/** The kill of a poisoned worker has been started. */
+	recycling = false;
+	/** Kills a poisoned worker whose drain did not finish in time. */
+	recycleBackstop = null;
+	/** A liveness probe is in flight — one at a time, never a probe storm. */
+	probeInFlight = false;
+	/** The child has exited (any cause). */
+	exited = false;
+	/** A compile past its soft bound, still running in the worker (D653). */
+	abandonedCompile = null;
+	/** Says when live inference has produced nothing for too long (D653). */
+	inferStall = new InferStallDetector(Date.now());
+	/**
+	* A load's HOST deadline expired with no reply since the last reply of any
+	* kind (D653 round 3): the loop missed its own soft bound, so the GIL grace
+	* is over for this worker until something — anything — comes back.
+	*/
+	loadDeadlineMissed = false;
 	log;
 	opts;
 	constructor(opts) {
@@ -822,6 +978,24 @@ var PoolWorker = class {
 	isReady() {
 		return this.ready;
 	}
+	/** Why this worker was recycled, typed for the provider's budget, or `null`. */
+	getDeathCause() {
+		const v = this.poisonVerdict;
+		const message = this.getPoisonDescription();
+		if (v === null || message === null) return null;
+		return {
+			reason: v.reason,
+			chargesDeviceBudget: chargesDeviceBudget(v.reason),
+			message
+		};
+	}
+	/** Why this live worker was declared unusable, or `null` (D653). */
+	getPoisonDescription() {
+		const v = this.poisonVerdict;
+		if (v === null) return null;
+		const target = v.command.model !== null ? ` of ${v.command.model}` : "";
+		return `${v.reason} on ${v.command.cmd}${target} (pid ${v.pid ?? "unknown"})`;
+	}
 	async initialize(initialModels) {
 		this.process = spawn(this.opts.pythonPath, [this.opts.scriptPath], { stdio: [
 			"pipe",
@@ -845,7 +1019,11 @@ var PoolWorker = class {
 		const spawnedProcess = this.process;
 		spawnedProcess.on("exit", (code, signal) => {
 			this.ready = false;
+			this.exited = true;
+			if (this.recycleBackstop) clearTimeout(this.recycleBackstop);
+			if (this.abandonedCompile) clearTimeout(this.abandonedCompile.hardTimer);
 			if (this.process !== spawnedProcess) return;
+			const poisonedBy = this.getPoisonDescription();
 			this.log.error("Worker process exited", { meta: {
 				worker: this.opts.workerLabel,
 				pid: spawnedProcess.pid ?? null,
@@ -853,9 +1031,10 @@ var PoolWorker = class {
 				device: this.opts.device ?? "default",
 				code,
 				signal,
-				inFlight: this.pending.size
+				inFlight: this.pending.size,
+				...poisonedBy !== null ? { poisonedBy } : {}
 			} });
-			this.rejectAll(/* @__PURE__ */ new Error(`PoolWorker[${this.opts.workerLabel}]: worker process exited (code=${code ?? "null"}, signal=${signal ?? "none"})`));
+			this.rejectAll(/* @__PURE__ */ new Error(`PoolWorker[${this.opts.workerLabel}]: worker process exited (code=${code ?? "null"}, signal=${signal ?? "none"})` + (poisonedBy !== null ? ` — recycled, poisoned by ${poisonedBy}` : "")));
 		});
 		this.process.stdout.on("data", (chunk) => {
 			const t0 = Date.now();
@@ -869,6 +1048,7 @@ var PoolWorker = class {
 			runtime: this.opts.poolRuntime,
 			concurrency: this.opts.concurrency,
 			protocolVersion: 2,
+			modelLoadTimeoutMs: POOL_MODEL_LOAD_BOUNDS[this.opts.poolRuntime].softMs,
 			models: initialModels.map((m) => serializeModelConfig(m))
 		};
 		if (this.opts.device) config["device"] = this.opts.device;
@@ -891,6 +1071,7 @@ var PoolWorker = class {
 					clearTimeout(timeout);
 					if (result["status"] === "ready") {
 						this.ready = true;
+						this.inferStall.noteResult(Date.now());
 						const loadedCount = result["models"];
 						const startupMs = result["startupMs"];
 						const workers = result["workers"] ?? 1;
@@ -913,7 +1094,7 @@ var PoolWorker = class {
 	async infer(modelByte, jpeg, deviceId) {
 		this.ensureReady();
 		const payload = Buffer.concat([Buffer.from([modelByte]), jpeg]);
-		return this.dispatch(MSG_INFER_JPEG, payload, deviceId);
+		return this.dispatch(MSG_INFER_JPEG, payload, { deviceId });
 	}
 	async inferRaw(modelByte, raw, width, height, format, deviceId) {
 		this.ensureReady();
@@ -966,12 +1147,18 @@ var PoolWorker = class {
 		const payload = Buffer.allocUnsafe(5);
 		payload[0] = modelByte;
 		payload.writeUInt32LE(frameId, 1);
-		return this.dispatch(MSG_INFER_CACHED, payload, deviceId);
+		return this.dispatch(MSG_INFER_CACHED, payload, { deviceId });
 	}
 	async sendCommand(cmd) {
 		this.ensureReady();
+		const command = describeCommand(cmd);
 		const payload = Buffer.from(JSON.stringify(cmd), "utf8");
-		return await this.dispatch(MSG_COMMAND, payload);
+		const sentAt = Date.now();
+		if (MODEL_LOAD_COMMANDS.has(command.cmd)) this.warnIfLoadQueued(command);
+		const raw = await this.dispatch(MSG_COMMAND, payload, { command });
+		if (MODEL_LOAD_COMMANDS.has(command.cmd)) this.inferStall.noteResult(Date.now());
+		this.inspectCommandReply(raw, command, sentAt);
+		return raw;
 	}
 	/** Command whose reply shape is NOT the load/unload/replace/status envelope
 	*  (e.g. `mem_stats`) — returns the raw JSON record, so no cast is needed
@@ -979,13 +1166,15 @@ var PoolWorker = class {
 	async sendRawCommand(cmd) {
 		this.ensureReady();
 		const payload = Buffer.from(JSON.stringify(cmd), "utf8");
-		return this.dispatch(MSG_COMMAND, payload);
+		return this.dispatch(MSG_COMMAND, payload, { command: describeCommand(cmd) });
 	}
 	async dispose() {
 		const proc = this.process;
 		if (!proc) return;
 		this.process = null;
 		this.ready = false;
+		if (this.recycleBackstop) clearTimeout(this.recycleBackstop);
+		if (this.abandonedCompile) clearTimeout(this.abandonedCompile.hardTimer);
 		await terminateChild(proc, POOL_WORKER_TERM_GRACE_MS);
 		this.rejectAll(/* @__PURE__ */ new Error(`PoolWorker[${this.opts.workerLabel}]: pool disposed while the request was in flight`));
 	}
@@ -1028,8 +1217,11 @@ var PoolWorker = class {
 	}
 	/** Live inference gets seconds; commands and model loads keep the long
 	*  timeout — see {@link POOL_LIVE_INFER_TIMEOUT_MS}. */
-	deadlineFor(msgType) {
-		return SHEDDABLE_MSG_TYPES.has(msgType) ? POOL_LIVE_INFER_TIMEOUT_MS : POOL_INFER_TIMEOUT_MS;
+	deadlineFor(msgType, opts = {}) {
+		if (opts.timeoutMs !== void 0) return opts.timeoutMs;
+		if (SHEDDABLE_MSG_TYPES.has(msgType)) return POOL_LIVE_INFER_TIMEOUT_MS;
+		if (opts.command !== void 0 && MODEL_LOAD_COMMANDS.has(opts.command.cmd)) return POOL_MODEL_LOAD_BOUNDS[this.opts.poolRuntime].softMs + POOL_MODEL_LOAD_REPLY_MARGIN_MS;
+		return POOL_INFER_TIMEOUT_MS;
 	}
 	/**
 	* The request's ABSOLUTE deadline for the wire (D350), or `null` when this
@@ -1046,16 +1238,18 @@ var PoolWorker = class {
 		stamp.writeBigUInt64LE(BigInt(Date.now() + this.deadlineFor(msgType)), 0);
 		return stamp;
 	}
-	dispatch(msgType, payload, deviceId) {
-		const shed = this.shedIfSaturated(msgType, deviceId);
+	dispatch(msgType, payload, opts = {}) {
+		const shed = this.shedIfSaturated(msgType, opts.deviceId);
 		if (shed) return Promise.resolve(shed);
 		const reqId = this.allocRequestId();
 		return new Promise((resolve, reject) => {
-			const timer = this.armRequestTimeout(reqId, msgType, resolve, reject, deviceId);
+			const timer = this.armRequestTimeout(reqId, msgType, resolve, reject, opts);
 			this.pending.set(reqId, {
 				resolve,
 				reject,
-				timer
+				timer,
+				...opts.command !== void 0 ? { command: opts.command } : {},
+				...SHEDDABLE_MSG_TYPES.has(msgType) ? { live: true } : {}
 			});
 			try {
 				const stamp = this.deadlineStamp(msgType);
@@ -1086,8 +1280,9 @@ var PoolWorker = class {
 	* - Anything else (commands, model loads) still REJECTS with the error the
 	*   dashboards grep for — a lost command is a fault, not flow control.
 	*/
-	armRequestTimeout(reqId, msgType, resolve, reject, deviceId) {
-		const timeoutMs = this.deadlineFor(msgType);
+	armRequestTimeout(reqId, msgType, resolve, reject, opts = {}) {
+		const { deviceId, command } = opts;
+		const timeoutMs = this.deadlineFor(msgType, opts);
 		const timer = setTimeout(() => {
 			if (this.pending.delete(reqId)) {
 				if (SHEDDABLE_MSG_TYPES.has(msgType)) {
@@ -1111,6 +1306,7 @@ var PoolWorker = class {
 						dropped: true,
 						shedReason: "deadline-expired"
 					});
+					this.noteInferUnanswered();
 					return;
 				}
 				this.timedOutCount++;
@@ -1123,10 +1319,20 @@ var PoolWorker = class {
 						device: this.opts.device ?? "default",
 						inFlight: this.pending.size,
 						reqId,
-						timeoutMs
+						timeoutMs,
+						...command !== void 0 ? {
+							command: command.cmd,
+							modelIndex: command.modelIndex,
+							model: command.model
+						} : {}
 					}
 				});
-				reject(/* @__PURE__ */ new Error(`PoolWorker[${this.opts.workerLabel}]: inference request ${reqId} timed out after ${timeoutMs}ms on ${this.opts.poolRuntime}:${this.opts.device ?? "default"} (worker alive, no reply)`));
+				const message = `PoolWorker[${this.opts.workerLabel}]: inference request ${reqId} timed out after ${timeoutMs}ms on ${this.opts.poolRuntime}:${this.opts.device ?? "default"} (worker alive, no reply)`;
+				if (command !== void 0 && MODEL_LOAD_COMMANDS.has(command.cmd)) {
+					this.loadDeadlineMissed = true;
+					reject(new PoolModelLoadError(`compile-timeout: ${message}`, "compile-timeout", command.model));
+				} else reject(new Error(message));
+				if (opts.probe !== true && command !== void 0) this.probeLiveness(command);
 			}
 		}, timeoutMs);
 		timer.unref?.();
@@ -1137,11 +1343,12 @@ var PoolWorker = class {
 		if (shed) return Promise.resolve(shed);
 		const reqId = this.allocRequestId();
 		return new Promise((resolve, reject) => {
-			const timer = this.armRequestTimeout(reqId, msgType, resolve, reject, deviceId);
+			const timer = this.armRequestTimeout(reqId, msgType, resolve, reject, { deviceId });
 			this.pending.set(reqId, {
 				resolve,
 				reject,
-				timer
+				timer,
+				...SHEDDABLE_MSG_TYPES.has(msgType) ? { live: true } : {}
 			});
 			try {
 				if (!this.process?.stdin) throw new Error("PoolWorker: not initialized");
@@ -1163,10 +1370,10 @@ var PoolWorker = class {
 	}
 	allocRequestId() {
 		let id = this.nextRequestId;
-		this.nextRequestId = id >= 4294967295 ? 1 : id + 1;
+		this.nextRequestId = id >= WORKER_EVENT_REQ_ID - 1 ? 1 : id + 1;
 		while (this.pending.has(id)) {
 			id = this.nextRequestId;
-			this.nextRequestId = id >= 4294967295 ? 1 : id + 1;
+			this.nextRequestId = id >= WORKER_EVENT_REQ_ID - 1 ? 1 : id + 1;
 		}
 		return id;
 	}
@@ -1181,6 +1388,8 @@ var PoolWorker = class {
 		this.process.stdin.write(payload);
 	}
 	ensureReady() {
+		const poisonedBy = this.getPoisonDescription();
+		if (poisonedBy !== null) throw new Error(`PoolWorker[${this.opts.workerLabel}]: poisoned (${poisonedBy}) — recycling, not taking work`);
 		if (!this.ready || !this.process?.stdin) throw new Error(`PoolWorker[${this.opts.workerLabel}]: not initialized`);
 	}
 	/** Time spent in `Buffer.concat` since the last report. */
@@ -1220,23 +1429,273 @@ var PoolWorker = class {
 			const reqId = this.receiveBuffer.readUInt32LE(4);
 			const jsonBytes = this.receiveBuffer.subarray(8, 4 + totalLen);
 			this.receiveBuffer = this.receiveBuffer.subarray(4 + totalLen);
+			if (reqId === WORKER_EVENT_REQ_ID) {
+				this.handleWorkerEvent(jsonBytes);
+				continue;
+			}
 			const entry = this.pending.get(reqId);
 			if (!entry) {
 				this.log.warn("Response for unknown request id", { meta: {
 					worker: this.opts.workerLabel,
 					reqId
 				} });
+				this.noteLateReply(jsonBytes);
 				continue;
 			}
 			this.pending.delete(reqId);
 			if (entry.timer) clearTimeout(entry.timer);
 			try {
 				const parsed = JSON.parse(jsonBytes.toString("utf8"));
+				if (entry.live === true) if (parsed["dropped"] === true) this.noteInferUnanswered();
+				else this.inferStall.noteResult(Date.now());
 				entry.resolve(parsed);
 			} catch (err) {
 				entry.reject(err instanceof Error ? err : new Error(String(err)));
 			}
 		}
+		this.noteAnyReply();
+		if (this.poisonVerdict !== null && this.pending.size === 0) this.recycle();
+	}
+	/**
+	* Something came back from the worker: its loop was alive just now, so the
+	* missed-deadline verdict is lifted.
+	*/
+	noteAnyReply() {
+		this.loadDeadlineMissed = false;
+	}
+	/**
+	* The invariant the load deadline depends on (D653 § 2): loads into one
+	* pool are serialised, so at most ONE load is outstanding per worker. The
+	* worker runs model commands in order and starts a load's soft bound only
+	* when it begins, while the host's deadline for it runs from the SEND. A
+	* load queued behind another could therefore see its host deadline fire
+	* before the worker's named answer. Nothing in the provider issues that; if
+	* anything ever does, this says so.
+	*/
+	warnIfLoadQueued(next) {
+		const ahead = [...this.pending.values()].map((p) => p.command).filter((c) => c !== void 0 && MODEL_LOAD_COMMANDS.has(c.cmd));
+		if (ahead.length === 0) return;
+		this.log.warn("model load queued behind another on one worker — its deadline may fire before the worker answers", { meta: {
+			worker: this.opts.workerLabel,
+			pid: this.getPid(),
+			runtime: this.opts.poolRuntime,
+			device: this.opts.device ?? "default",
+			model: next.model,
+			queuedBehind: ahead.map((c) => c.model)
+		} });
+	}
+	hasPendingLoad() {
+		for (const p of this.pending.values()) if (p.command !== void 0 && MODEL_LOAD_COMMANDS.has(p.command.cmd)) return true;
+		return false;
+	}
+	/**
+	* A load reply that says a compile outlived its SOFT bound (`compile-timeout`)
+	* or that an earlier one still has (`worker-poisoned`). The worker is not
+	* recycled for it (fix round 1): the compile keeps running so a slow but
+	* finite one writes its cache, the loaded models keep serving, and the worker
+	* starts no new load. Only a compile still running at the HARD bound costs
+	* the process.
+	*/
+	inspectCommandReply(raw, command, sentAt) {
+		const reason = raw["reason"];
+		if (reason !== "compile-timeout" && reason !== "worker-poisoned") return;
+		if (this.abandonedCompile !== null || this.poisonVerdict !== null || this.exited) return;
+		const model = raw["modelId"];
+		const abandoned = {
+			...command,
+			model: typeof model === "string" ? model : command.model
+		};
+		const bounds = POOL_MODEL_LOAD_BOUNDS[this.opts.poolRuntime];
+		const reported = raw["compileElapsedMs"];
+		const elapsedMs = typeof reported === "number" && Number.isFinite(reported) && reported >= 0 ? reported : Date.now() - sentAt;
+		const hardTimer = setTimeout(() => this.poison("compile-hung", abandoned, `the compile of ${abandoned.model ?? "unknown"} was still running at its hard bound (${bounds.hardMs}ms)`), Math.max(0, bounds.hardMs - elapsedMs));
+		hardTimer.unref?.();
+		this.abandonedCompile = {
+			command: abandoned,
+			since: Date.now(),
+			hardTimer
+		};
+		this.log.warn("model compile past its soft bound — left running so its cache can be written", { meta: {
+			worker: this.opts.workerLabel,
+			pid: this.getPid(),
+			runtime: this.opts.poolRuntime,
+			device: this.opts.device ?? "default",
+			model: abandoned.model,
+			modelIndex: abandoned.modelIndex,
+			softMs: bounds.softMs,
+			hardMs: bounds.hardMs,
+			loads: "refused until it returns"
+		} });
+	}
+	/** An unsolicited worker event (`compile-finished-late`). */
+	handleWorkerEvent(jsonBytes) {
+		let event;
+		try {
+			event = JSON.parse(jsonBytes.toString("utf8"));
+		} catch {
+			return;
+		}
+		if (event["event"] !== "compile-finished-late") return;
+		const abandoned = this.abandonedCompile;
+		if (abandoned !== null) clearTimeout(abandoned.hardTimer);
+		this.abandonedCompile = null;
+		this.inferStall.noteResult(Date.now());
+		const ok = event["ok"] === true;
+		const meta = {
+			worker: this.opts.workerLabel,
+			pid: this.getPid(),
+			runtime: this.opts.poolRuntime,
+			device: this.opts.device ?? "default",
+			model: typeof event["modelId"] === "string" ? event["modelId"] : null,
+			elapsedMs: typeof event["elapsedMs"] === "number" ? event["elapsedMs"] : null,
+			...typeof event["error"] === "string" ? { error: event["error"] } : {}
+		};
+		if (ok) this.log.info("slow model compile finished after its soft bound — cache written, loads re-enabled", { meta });
+		else this.log.warn("slow model compile failed after its soft bound — loads re-enabled", { meta });
+		this.opts.onCompileFinishedLate?.({
+			model: meta.model,
+			ok,
+			...typeof event["error"] === "string" ? { error: event["error"] } : {}
+		});
+	}
+	/**
+	* The GIL grace (D653 round 2): may a silent loop be a compile holding the
+	* GIL rather than a wedge? Only while a load is SENT AND UNANSWERED, and only
+	* on a runtime whose compile may hold the GIL (`POOL_MODEL_LOAD_BOUNDS`).
+	*
+	* NOT while a compile runs past its soft bound: the worker REPLIED
+	* `compile-timeout`, so its loop is proven alive, and an inference hang
+	* behind that compile — the 2026-09-26 incident exactly — must be seen on
+	* the normal rule, not ~600-900 s later at the hard bound.
+	*/
+	gilGraceActive() {
+		if (!POOL_MODEL_LOAD_BOUNDS[this.opts.poolRuntime].gilMayBeHeldDuringCompile) return false;
+		if (this.loadDeadlineMissed) return false;
+		return this.hasPendingLoad();
+	}
+	/**
+	* The reason a silent worker is recycled with: a freeze that began with a
+	* load missing its deadline is that compile's doing (`compile-hung`, charged
+	* to the model, not the device); anything else is the device's.
+	*/
+	silenceReason(fallback) {
+		return this.loadDeadlineMissed ? "compile-hung" : fallback;
+	}
+	/** A live request ended with no result: judge the executor (D653, item 4). */
+	noteInferUnanswered() {
+		const stall = this.inferStall.noteUnanswered(Date.now());
+		if (stall === null || this.gilGraceActive()) return;
+		this.poison(this.silenceReason("infer-unresponsive"), {
+			cmd: "infer",
+			modelIndex: null,
+			model: null
+		}, `${stall.unanswered} live requests in a row ended without a result, none for ${stall.silentMs}ms`);
+	}
+	/** A reply whose request was already abandoned: a real result still proves the executor runs. */
+	noteLateReply(jsonBytes) {
+		try {
+			const parsed = JSON.parse(jsonBytes.toString("utf8"));
+			if (parsed["dropped"] !== true && parsed["cmd"] === void 0) this.inferStall.noteResult(Date.now());
+		} catch {}
+	}
+	/**
+	* Ask a worker whose command just timed out whether its loop still answers.
+	* `mem_stats` is served on the loop, never behind a compile, so a worker
+	* that cannot answer it inside {@link POOL_LIVENESS_PROBE_TIMEOUT_MS} is not
+	* slow — it is wedged. That is the 2026-09-26 shape exactly: 123 of 123
+	* `mem_stats` lost while the process looked alive.
+	*/
+	probeLiveness(timedOut) {
+		if (this.probeInFlight || this.poisonVerdict !== null || this.exited) return;
+		this.probeInFlight = true;
+		this.dispatch(MSG_COMMAND, Buffer.from(JSON.stringify({ cmd: "mem_stats" }), "utf8"), {
+			command: {
+				cmd: "mem_stats",
+				modelIndex: null,
+				model: null
+			},
+			timeoutMs: POOL_LIVENESS_PROBE_TIMEOUT_MS,
+			probe: true
+		}).then(() => {
+			this.log.warn("pool command timed out but the worker answers its liveness probe — slow, not wedged", { meta: {
+				worker: this.opts.workerLabel,
+				pid: this.getPid(),
+				runtime: this.opts.poolRuntime,
+				device: this.opts.device ?? "default",
+				command: timedOut.cmd,
+				modelIndex: timedOut.modelIndex,
+				model: timedOut.model
+			} });
+		}, (err) => {
+			if (this.gilGraceActive()) {
+				this.log.warn("liveness probe unanswered while a model load is outstanding — not poisoning yet", { meta: {
+					worker: this.opts.workerLabel,
+					pid: this.getPid(),
+					runtime: this.opts.poolRuntime,
+					device: this.opts.device ?? "default",
+					command: timedOut.cmd,
+					model: timedOut.model
+				} });
+				return;
+			}
+			this.poison(this.silenceReason("unresponsive"), timedOut, `${timedOut.cmd} timed out, then the liveness probe failed: ${err instanceof Error ? err.message : String(err)}`);
+		}).finally(() => {
+			this.probeInFlight = false;
+		});
+	}
+	/**
+	* Declare this LIVE worker unusable, say so once at ERROR, stop taking work,
+	* and recycle it once it has drained.
+	*
+	* `ready = false` is what hands the pool back to the provider: its next
+	* dispatch finds the factory not ready and condemns it under the per-device
+	* restart budget (3 deaths in 10 min, then a terminal `failed` the balancer
+	* excludes) — the same path a crashed worker takes, except that a
+	* `compile-hung` death is not charged to the device (see PoolPoisonReason).
+	* An unresponsive worker is killed at once (it will answer nothing it
+	* holds); a compile-hung one keeps serving its loaded models until its
+	* in-flight requests are answered, bounded by the live deadline.
+	*/
+	poison(reason, command, detail) {
+		if (this.poisonVerdict !== null || this.exited || this.process === null) return;
+		this.poisonVerdict = {
+			reason,
+			command,
+			detail,
+			pid: this.getPid()
+		};
+		this.ready = false;
+		const inFlightCommands = [...this.pending.values()].map((p) => p.command).filter((c) => c !== void 0).map((c) => c.model !== null ? `${c.cmd}:${c.model}` : c.cmd);
+		this.log.error("pool worker POISONED — recycling it", { meta: {
+			worker: this.opts.workerLabel,
+			pid: this.getPid(),
+			runtime: this.opts.poolRuntime,
+			device: this.opts.device ?? "default",
+			reason,
+			command: command.cmd,
+			modelIndex: command.modelIndex,
+			model: command.model,
+			detail,
+			inFlight: this.pending.size,
+			inFlightCommands
+		} });
+		if (reason !== "compile-hung" || this.pending.size === 0) {
+			this.recycle();
+			return;
+		}
+		this.recycleBackstop = setTimeout(() => this.recycle(), POOL_LIVE_INFER_TIMEOUT_MS + POOL_POISON_DRAIN_SLACK_MS);
+		this.recycleBackstop.unref?.();
+	}
+	/**
+	* Kill a poisoned worker WITHOUT nulling `this.process`, so its `exit` is
+	* reported and rejects whatever it still held — a deliberate dispose would
+	* silence both, and this is not one.
+	*/
+	recycle() {
+		if (this.recycling || this.exited || this.process === null) return;
+		this.recycling = true;
+		if (this.recycleBackstop) clearTimeout(this.recycleBackstop);
+		terminateChild(this.process, POOL_WORKER_TERM_GRACE_MS);
 	}
 	rejectAll(err) {
 		const entries = [...this.pending.values()];
@@ -1283,8 +1742,10 @@ var SharedInferencePool = class {
 		this.tuning = options.tuning ?? null;
 		this.numWorkers = Math.max(1, options.numWorkers ?? 1);
 		this.device = options.device;
+		this.onCompileFinishedLate = options.onCompileFinishedLate;
 	}
 	device;
+	onCompileFinishedLate;
 	/** Pid of the first worker (for legacy callers). Use `getPids()` for all. */
 	/** Summed backlog and shed count across the pool's workers. A rising
 	*  `inFlight` with a rising `shed` is a worker falling behind; a rising
@@ -1323,7 +1784,8 @@ var SharedInferencePool = class {
 			tuning: this.tuning,
 			logger: this.log,
 			workerLabel: `w${i}`,
-			...this.device ? { device: this.device } : {}
+			...this.device ? { device: this.device } : {},
+			...this.onCompileFinishedLate ? { onCompileFinishedLate: this.onCompileFinishedLate } : {}
 		}));
 		const t0 = performance.now();
 		const results = await Promise.all(this.workers.map((w) => w.initialize(initialModels)));
@@ -1418,7 +1880,7 @@ var SharedInferencePool = class {
 			index,
 			config: serializeModelConfig(config)
 		})));
-		for (const resp of responses) if (resp.status !== "ok") throw new Error(`Failed to load model at index ${index}: ${resp.error ?? "unknown"}`);
+		for (const resp of responses) if (resp.status !== "ok") throw new PoolModelLoadError(`Failed to load model at index ${index}: ${describeCommandFailure(resp)}`, resp.reason ?? null, resp.modelId ?? null);
 		if (index >= this.nextFreeIndex) this.nextFreeIndex = index + 1;
 		return { loadMs: Math.max(...responses.map((r) => r.loadMs ?? 0)) };
 	}
@@ -1454,7 +1916,7 @@ var SharedInferencePool = class {
 			index,
 			config: serializeModelConfig(config)
 		})));
-		for (const resp of responses) if (resp.status !== "ok") throw new Error(`Failed to replace model at index ${index}: ${resp.error ?? "unknown"}`);
+		for (const resp of responses) if (resp.status !== "ok") throw new PoolModelLoadError(`Failed to replace model at index ${index}: ${describeCommandFailure(resp)}`, resp.reason ?? null, resp.modelId ?? null);
 		return { loadMs: Math.max(...responses.map((r) => r.loadMs ?? 0)) };
 	}
 	/**
@@ -1504,9 +1966,35 @@ var SharedInferencePool = class {
 	allocateIndex() {
 		return this.nextFreeIndex++;
 	}
+	/**
+	* Give back an index whose load FAILED, so the retry reuses it. Only the most
+	* recent allocation can be returned. Loads into one pool are serialised by
+	* the provider, so a failed load is normally the latest one; anything else
+	* is left allocated rather than risk handing out a live slot twice.
+	*/
+	releaseIndex(index) {
+		if (index === this.nextFreeIndex - 1) this.nextFreeIndex = index;
+	}
 	isReady() {
 		return this.workers.length > 0 && this.workers.every((w) => w.isReady());
 	}
+	/**
+	* Why a worker of this pool was declared unusable while alive (D653), or
+	* `null`. The provider charges the restart budget with THIS — or, for a
+	* `compile-hung` death, does not charge the device at all — so the
+	* `inference device FAILED` line names the cause instead of "pool worker is
+	* not ready".
+	*/
+	getDeathCause() {
+		for (const w of this.workers) {
+			const cause = w.getDeathCause();
+			if (cause !== null) return {
+				...cause,
+				message: `worker ${cause.message}`
+			};
+		}
+		return null;
+	}
 	async dispose() {
 		await Promise.all(this.workers.map((w) => w.dispose()));
 		this.workers.length = 0;
@@ -1571,6 +2059,23 @@ var SharedInferencePool = class {
 		return found;
 	}
 };
+/** A failed command's reply as one message: its reason class first, when it has one. */
+function describeCommandFailure(resp) {
+	const error = resp.error ?? "unknown";
+	return resp.reason !== void 0 ? `${resp.reason}: ${error}` : error;
+}
+/** Name a command for the lines that must say which one hung (D653). */
+function describeCommand(cmd) {
+	const name = typeof cmd["cmd"] === "string" ? cmd["cmd"] : "unknown";
+	const index = cmd["index"];
+	const config = cmd["config"];
+	const modelPath = typeof config === "object" && config !== null && "path" in config ? config.path : void 0;
+	return {
+		cmd: name,
+		modelIndex: typeof index === "number" ? index : null,
+		model: typeof modelPath === "string" && modelPath.length > 0 ? path$1.basename(modelPath, path$1.extname(modelPath)) : null
+	};
+}
 function serializeModelConfig(config) {
 	const result = {
 		path: config.path,
@@ -1696,12 +2201,13 @@ function resolveEffectivePostprocessorForEntry(definition, entry) {
 //#region src/detection-pipeline/registry/model-knob-applicability.ts
 /** Decodes that run NMS in the pipeline (and hence consult `nmsIouThreshold`).
 *  Mirrors the Python postprocessors that read the knob — yolo.py,
-*  yolo_seg.py, rfdetr.py, scrfd.py. */
+*  yolo_seg.py, rfdetr.py, scrfd.py, yunet.py. */
 var NMS_CONSUMING_POSTPROCESSORS = new Set([
 	"yolo",
 	"yolo-seg",
 	"rfdetr",
-	"scrfd"
+	"scrfd",
+	"yunet"
 ]);
 /**
 * Does this decode consult `nmsIouThreshold`?
@@ -1857,6 +2363,28 @@ function poolDecodeSettingsEqual(a, b) {
 }
 //#endregion
 //#region src/detection-pipeline/engine/pipeline-model-manager.ts
+/**
+* A (step, model) load the pool rejected — carrying the pool's failure class
+* (`compile-timeout`, …) so the provider can count a timeout against THAT
+* model without parsing a message (D653).
+*/
+var StepVariantLoadError = class extends Error {
+	stepId;
+	modelId;
+	poolIndex;
+	reason;
+	/** The model's file stem as the pool names it (`camstack-yunet-2023mar`). */
+	poolModel;
+	constructor(stepId, modelId, poolIndex, cause) {
+		super(cause instanceof Error ? cause.message : String(cause));
+		this.name = "StepVariantLoadError";
+		this.stepId = stepId;
+		this.modelId = modelId;
+		this.poolIndex = poolIndex;
+		this.reason = cause instanceof PoolModelLoadError ? cause.reason : null;
+		this.poolModel = cause instanceof PoolModelLoadError ? cause.model : null;
+	}
+};
 var PipelineModelManager = class {
 	pool;
 	source;
@@ -1868,11 +2396,20 @@ var PipelineModelManager = class {
 	lruClock = 0;
 	log;
 	maxModelsPerStep;
+	deviceKey;
+	/**
+	* Loads issued and not yet answered, keyed `stepId::modelId`. A second
+	* caller for the same pair JOINS the pending load instead of issuing its own
+	* (D653): on 2026-09-26 43 identical YuNet loads queued behind one hung GPU
+	* compile, each allocating a pool index of its own.
+	*/
+	pendingLoads = /* @__PURE__ */ new Map();
 	constructor(pool, source, logger, options) {
 		this.pool = pool;
 		this.source = source;
 		this.log = logger;
 		this.maxModelsPerStep = options?.maxModelsPerStep ?? 4;
+		this.deviceKey = options?.deviceKey ?? null;
 	}
 	/**
 	* Apply a new pipeline configuration — driven by the runtime config
@@ -2022,6 +2559,23 @@ var PipelineModelManager = class {
 			await this.reconcileDecode(existing, settings);
 			return existing;
 		}
+		const key = `${stepId}::${modelId}`;
+		const pending = this.pendingLoads.get(key);
+		if (pending) {
+			const joined = await pending;
+			await this.reconcileDecode(joined, settings);
+			return joined;
+		}
+		const load = this.issueLoad(perStep, stepId, modelId, settings);
+		this.pendingLoads.set(key, load);
+		try {
+			return await load;
+		} finally {
+			this.pendingLoads.delete(key);
+		}
+	}
+	/** Evict if at capacity, then send ONE load command and record the result. */
+	async issueLoad(perStep, stepId, modelId, settings) {
 		while (perStep.size >= this.maxModelsPerStep) {
 			const evicted = this.pickEvictionTarget(stepId);
 			if (!evicted) break;
@@ -2033,16 +2587,31 @@ var PipelineModelManager = class {
 				cap: this.maxModelsPerStep
 			} });
 		}
-		const index = this.pool.allocateIndex();
 		const config = this.source.buildConfig(stepId, modelId, settings);
 		const decode = poolDecodeSettingsOf(config);
+		const index = this.pool.allocateIndex();
 		this.log.info("Loading step variant", { meta: {
 			step: stepId,
 			modelId,
 			poolIndex: index,
+			deviceKey: this.deviceKey,
 			...decode
 		} });
-		const { loadMs } = await this.pool.loadModel(index, config);
+		let loadMs;
+		try {
+			({loadMs} = await this.pool.loadModel(index, config));
+		} catch (err) {
+			this.pool.releaseIndex(index);
+			this.log.error("Step variant load failed", { meta: {
+				step: stepId,
+				modelId,
+				poolIndex: index,
+				deviceKey: this.deviceKey,
+				reason: err instanceof PoolModelLoadError ? err.reason : null,
+				error: err instanceof Error ? err.message : String(err)
+			} });
+			throw new StepVariantLoadError(stepId, modelId, index, err);
+		}
 		this.log.info("Step variant loaded", { meta: {
 			step: stepId,
 			modelId,
@@ -2412,6 +2981,15 @@ var EngineFactory = class {
 	isReady() {
 		return this.pool?.isReady() ?? false;
 	}
+	/**
+	* Why this factory's pool stopped being usable while its process was alive —
+	* a compile that never returned, or a worker that answered nothing (D653) —
+	* or `null`. A pool that merely crashed says nothing here; its `Worker
+	* process exited` line already names the signal.
+	*/
+	getDeathCause() {
+		return this.pool?.getDeathCause() ?? null;
+	}
 	/** Native pid of the underlying Python pool, if any. */
 	getPoolPid() {
 		return this.pool?.getPid() ?? null;
@@ -2538,12 +3116,13 @@ var EngineFactory = class {
 			concurrency,
 			tuning: resolvedTuning,
 			numWorkers,
-			...this.opts.engine.device ? { device: this.opts.engine.device } : {}
+			...this.opts.engine.device ? { device: this.opts.engine.device } : {},
+			...this.opts.onCompileFinishedLate ? { onCompileFinishedLate: this.opts.onCompileFinishedLate } : {}
 		});
 		this.poolManager = new PipelineModelManager(this.pool, {
 			buildConfig: (stepId, modelId, settings) => this.buildPoolModelConfig(stepId, modelId, poolRuntime, settings),
 			resolveDecode: (stepId, modelId, settings) => this.resolveDecode(stepId, modelId, settings)
-		}, this.log.child("model-mgr"));
+		}, this.log.child("model-mgr"), { deviceKey: this.deviceKey });
 		await this.pool.initialize([]);
 		await this.poolManager.applyConfig(steps);
 	}
@@ -2659,6 +3238,175 @@ function buildPoolModelConfigForStep(inputs) {
 	};
 }
 //#endregion
+//#region src/detection-pipeline/engine/model-load-governor.ts
+/**
+* What the dispatch path may ask of a pool's model loads, and what it must say
+* when it may not (D653, fix round 1).
+*
+* Three pieces of state, consulted ON DEMAND by the dispatch that needs a
+* model — there is no timer anywhere in here:
+*
+*  1. A NEGATIVE CACHE per (pool, model). `needsPoolUpdate` stays true after a
+*     failed load, so without it every frame of every camera re-issued the
+*     failing load and wrote two ERRORs. After a failure the model is not
+*     re-issued on that pool for an interval that doubles per consecutive
+*     failure up to a cap; a success clears it.
+*  2. COMPILE TIMEOUTS per (device, model). A load that outlived its soft
+*     bound is not a crash of the device. The SAME model timing out
+*     {@link COMPILE_TIMEOUTS_BEFORE_REFUSAL} times on one device — which,
+*     since a slow compile now runs on and writes its cache, only a compile
+*     that never finishes can do — refuses THAT MODEL there, by name. The
+*     device keeps serving every other model.
+*  3. Which camera has already been TOLD about the current state of a model,
+*     so each camera gets one line per state change, never one per frame.
+*/
+/** The first back-off after a failed load. */
+var MODEL_LOAD_BACKOFF_INITIAL_MS = 5e3;
+/** The back-off doubles per consecutive failure up to this. */
+var MODEL_LOAD_BACKOFF_MAX_MS = 5 * 6e4;
+/** Timeouts older than this no longer count toward a refusal. */
+var COMPILE_TIMEOUT_WINDOW_MS = 60 * 6e4;
+var ModelLoadGovernor = class {
+	/** Keyed by the pool OBJECT, so a respawned pool starts clean. */
+	backoff = /* @__PURE__ */ new WeakMap();
+	timeouts = /* @__PURE__ */ new Map();
+	reported = /* @__PURE__ */ new Map();
+	generation = 0;
+	/** May this dispatch load `models` on `pool` (a device `deviceKey`) now? */
+	check(pool, deviceKey, models, now = Date.now()) {
+		for (const model of models) {
+			const t = this.timeouts.get(timeoutKey(deviceKey, model));
+			if (t?.refused === true) return {
+				kind: "refused",
+				model,
+				timeouts: t.at.length,
+				error: t.lastError
+			};
+		}
+		const perPool = this.backoff.get(pool);
+		if (perPool === void 0) return { kind: "go" };
+		for (const model of models) {
+			const b = perPool.get(model);
+			if (b !== void 0 && now < b.retryAtMs) return {
+				kind: "backoff",
+				model,
+				retryInMs: b.retryAtMs - now,
+				failures: b.failures,
+				error: b.error,
+				generation: b.generation
+			};
+		}
+		return { kind: "go" };
+	}
+	/** One load of `model` on `pool` failed. */
+	recordFailure(pool, deviceKey, model, error, compileTimeout, now = Date.now(), poolModel = null, reason = null) {
+		let perPool = this.backoff.get(pool);
+		if (perPool === void 0) {
+			perPool = /* @__PURE__ */ new Map();
+			this.backoff.set(pool, perPool);
+		}
+		const previousEntry = perPool.get(model);
+		const failures = (previousEntry?.failures ?? 0) + 1;
+		const retryInMs = Math.min(MODEL_LOAD_BACKOFF_MAX_MS, MODEL_LOAD_BACKOFF_INITIAL_MS * 2 ** (failures - 1));
+		this.generation += 1;
+		const generation = this.generation;
+		perPool.set(model, {
+			failures,
+			retryAtMs: now + retryInMs,
+			error,
+			generation,
+			poolModel: poolModel ?? previousEntry?.poolModel ?? null,
+			reason
+		});
+		if (!compileTimeout) return {
+			generation,
+			retryInMs,
+			refusedNow: false,
+			compileTimeouts: 0
+		};
+		const key = timeoutKey(deviceKey, model);
+		const previous = this.timeouts.get(key);
+		const at = [...(previous?.at ?? []).filter((t) => t >= now - COMPILE_TIMEOUT_WINDOW_MS), now];
+		const refused = at.length >= 2;
+		this.timeouts.set(key, {
+			at,
+			refused,
+			lastError: error
+		});
+		return {
+			generation,
+			retryInMs,
+			refusedNow: refused && previous?.refused !== true,
+			compileTimeouts: at.length
+		};
+	}
+	/**
+	* A compile that outlived its bound came back on `pool` (D653 round 3).
+	*
+	* `ok`: THAT model's cache is written and the worker loads again, so its
+	* back-off is lifted — and only its: another model's failure on the same
+	* pool is not the compile that just finished. Not ok: it is one more
+	* failure of that model, and its back-off grows. Refusals by name are never
+	* lifted here; a model refused for timing out twice stays refused until the
+	* operator re-arms. Returns the governor keys it touched.
+	*/
+	settleLateCompile(pool, deviceKey, poolModel, ok, error, now = Date.now()) {
+		const perPool = this.backoff.get(pool);
+		if (perPool === void 0) return [];
+		const refusedMeanwhile = [...perPool].filter(([, b]) => b.reason === "worker-poisoned" && b.poolModel !== poolModel).map(([key]) => key);
+		for (const key of refusedMeanwhile) perPool.delete(key);
+		const own = poolModel === null ? [] : [...perPool].filter(([, b]) => b.poolModel === poolModel).map(([key]) => key);
+		for (const key of own) if (ok) perPool.delete(key);
+		else this.recordFailure(pool, deviceKey, key, error, false, now, poolModel);
+		return [...refusedMeanwhile, ...own];
+	}
+	/** `models` loaded on `pool`: forget their failures there. */
+	recordSuccess(pool, deviceKey, models) {
+		const perPool = this.backoff.get(pool);
+		for (const model of models) {
+			perPool?.delete(model);
+			this.timeouts.delete(timeoutKey(deviceKey, model));
+		}
+	}
+	/**
+	* Has `camera` been told about this `state` of `model` on `deviceKey` yet?
+	* Returns `true` exactly once per state change.
+	*/
+	shouldReport(camera, deviceKey, model, state) {
+		const key = `${camera ?? "-"}|${deviceKey}|${model}`;
+		if (this.reported.get(key) === state) return false;
+		this.reported.set(key, state);
+		return true;
+	}
+	/** Every model refused on some device. */
+	refused() {
+		const out = [];
+		for (const [key, t] of this.timeouts) {
+			if (!t.refused) continue;
+			const [deviceKey = "", model = ""] = key.split("\0");
+			out.push({
+				deviceKey,
+				model,
+				timeouts: t.at.length,
+				lastError: t.lastError
+			});
+		}
+		return out;
+	}
+	/** Operator re-arm: forget the refusals on `deviceKey`. Returns how many. */
+	rearm(deviceKey) {
+		let cleared = 0;
+		for (const key of [...this.timeouts.keys()]) if (key.startsWith(`${deviceKey}\u0000`)) {
+			this.timeouts.delete(key);
+			cleared += 1;
+		}
+		return cleared;
+	}
+};
+function timeoutKey(deviceKey, model) {
+	return `${deviceKey}\u0000${model}`;
+}
+//#endregion
 //#region src/detection-pipeline/engine/idle-pool-reaper.ts
 var IdlePoolReaper = class {
 	lastUsed = /* @__PURE__ */ new Map();
@@ -3158,6 +3906,52 @@ function createInferenceTimeoutGuard(deps) {
 	};
 }
 //#endregion
+//#region src/detection-pipeline/model-pin.ts
+function requestedModels(steps) {
+	const out = /* @__PURE__ */ new Map();
+	const walk = (nodes) => {
+		for (const step of nodes) {
+			if (!step.enabled) continue;
+			out.set(step.addonId, step.modelId);
+			if (step.children !== void 0) walk(step.children);
+		}
+	};
+	walk(steps);
+	return out;
+}
+function checkReplayPin(input) {
+	if (input.requestedDeviceKey !== void 0 && input.dispatchDeviceKey !== input.requestedDeviceKey) return {
+		kind: "refused",
+		reason: "device-not-servable",
+		detail: `device pool ${input.requestedDeviceKey} cannot run this tree (the capability gate would fall back to ${input.dispatchDeviceKey ?? "the node default"})`
+	};
+	const asked = requestedModels(input.requested);
+	for (const step of input.executing) {
+		const want = asked.get(step.addonId);
+		if (want === void 0 || want.length === 0) return {
+			kind: "refused",
+			reason: "model-not-servable",
+			detail: `${step.addonId}: no model pinned by the caller (would run ${step.modelId}, ${input.format})`
+		};
+		if (want !== step.modelId) return {
+			kind: "refused",
+			reason: "model-not-servable",
+			detail: `${step.addonId}: asked for ${want}, this pool would run ${step.modelId} (${input.format})`
+		};
+	}
+	return { kind: "pinned" };
+}
+var PRIMARY_ROOT_STEP = "object-detection";
+/**
+* The root model a frame result reports: `object-detection` when it runs (it
+* is the detector the tracks come from), else the first enabled video root.
+* `undefined` when no root runs — never a fabricated id.
+*/
+function resolveRootModelId(roots) {
+	const enabled = roots.filter((s) => s.enabled && s.slot !== "audio-classifier");
+	return (enabled.find((s) => s.addonId === PRIMARY_ROOT_STEP) ?? enabled[0])?.modelId;
+}
+//#endregion
 //#region src/detection-pipeline/engine-provisioner.ts
 /** Incremental backoff growing to a ~5 min cap; retries indefinitely at cap. */
 var BACKOFF_SCHEDULE_MS = [
@@ -3428,6 +4222,7 @@ var KNOWN_POSTPROCESSORS = new Set([
 	"rfdetr",
 	"yolo-seg",
 	"scrfd",
+	"yunet",
 	"arcface",
 	"clip",
 	"softmax",
@@ -3529,9 +4324,12 @@ var DropSampler = class {
 	* accumulates and a summary is released once per interval — carrying the
 	* number suppressed, so nothing about the RATE is lost even though the lines
 	* are.
+	*
+	* `channel` separates occasions that share a device — a replay's drops are
+	* sampled apart from the live ones.
 	*/
-	record(deviceId, macroClass) {
-		const key = `${deviceId}:${macroClass}`;
+	record(deviceId, macroClass, channel = "live") {
+		const key = `${channel}:${deviceId}:${macroClass}`;
 		const at = this.now();
 		const prev = this.seen.get(key);
 		if (prev === void 0) {
@@ -5317,6 +6115,11 @@ var PipelineExecutor = class {
 	async run(tree, rootInput, fullFrameJpegProvider, imageWidth, imageHeight, deviceId, runOpts, nativeCropProvider, cropZoneBbox, rootInputViewProvider) {
 		const startMs = Date.now();
 		const verbosity = runOpts?.traceVerbosity ?? "off";
+		const occasion = runOpts?.occasion ?? "live";
+		const logTags = occasion === "replay" ? {
+			deviceId,
+			occasion
+		} : { deviceId };
 		const traceBuilder = new ExecutionTraceBuilder(verbosity, deviceId, imageWidth, imageHeight, this.opts.engineRuntime);
 		const firstLevel = [];
 		const details = [];
@@ -5332,9 +6135,9 @@ var PipelineExecutor = class {
 			idGen,
 			details,
 			onClassifierClassDropped: (drop) => {
-				const sample = this.classFilterDropSampler.record(deviceId, `${drop.stepId}:${drop.droppedClass}`);
+				const sample = this.classFilterDropSampler.record(deviceId, `${drop.stepId}:${drop.droppedClass}`, occasion);
 				if (sample.logExample) this.opts.logger?.info("classifier winner excluded by class filter — label dropped", {
-					tags: { deviceId },
+					tags: logTags,
 					meta: {
 						step: drop.stepId,
 						model: drop.modelId,
@@ -5345,7 +6148,7 @@ var PipelineExecutor = class {
 					}
 				});
 				if (sample.summaryCount !== null) this.opts.logger?.info("classifier class filter: still dropping", {
-					tags: { deviceId },
+					tags: logTags,
 					meta: {
 						step: drop.stepId,
 						droppedClass: drop.droppedClass,
@@ -5378,7 +6181,7 @@ var PipelineExecutor = class {
 						await this.executeChildren(rootStep.children, mutable, rootView?.jpegProvider ?? fullFrameJpegProvider, imageWidth, imageHeight, traceBuilder, stepTimings, ctx, poolAgg, runOpts?.plane, deviceId, nativeCropProvider);
 					} catch (err) {
 						this.opts.logger?.warn("Pipeline child execution failed — keeping parent detection", {
-							tags: { deviceId },
+							tags: logTags,
 							meta: {
 								rootStepId: rootStep.stepId,
 								parentClass: mutable.macroClass,
@@ -5432,9 +6235,9 @@ var PipelineExecutor = class {
 				const fullFrameBar = rootStep.settings?.["fullFrameGuardMaxConfidence"];
 				const fullFrameHardArea = rootStep.settings?.["fullFrameGuardHardAreaRatio"];
 				if (isFullFramePhantomDetection(mutable.bbox, imageWidth, imageHeight, mutable.score, typeof fullFrameBar === "number" ? fullFrameBar : void 0, typeof fullFrameHardArea === "number" ? fullFrameHardArea : void 0)) {
-					const sample = this.dropSampler.record(deviceId, String(mutable.macroClass));
+					const sample = this.dropSampler.record(deviceId, String(mutable.macroClass), occasion);
 					if (sample.summaryCount !== null) this.opts.logger?.info("full-frame phantom guard: still dropping", {
-						tags: { deviceId },
+						tags: logTags,
 						meta: {
 							macroClass: mutable.macroClass,
 							suppressed: sample.summaryCount,
@@ -5443,7 +6246,7 @@ var PipelineExecutor = class {
 						}
 					});
 					if (sample.logExample) this.opts.logger?.info("full-frame phantom guard: detection dropped", {
-						tags: { deviceId },
+						tags: logTags,
 						meta: {
 							macroClass: mutable.macroClass,
 							score: mutable.score,
@@ -5469,7 +6272,7 @@ var PipelineExecutor = class {
 					continue;
 				}
 				if (isFullFrameSurvivor(mutable.bbox, imageWidth, imageHeight, mutable.score, typeof fullFrameBar === "number" ? fullFrameBar : void 0)) this.opts.logger?.info("full-frame box SURVIVED the guard on score", {
-					tags: { deviceId },
+					tags: logTags,
 					meta: {
 						macroClass: mutable.macroClass,
 						score: mutable.score,
@@ -5483,7 +6286,7 @@ var PipelineExecutor = class {
 					await this.executeChildren(rootStep.children, mutable, rootView?.jpegProvider ?? fullFrameJpegProvider, imageWidth, imageHeight, traceBuilder, stepTimings, ctx, poolAgg, runOpts?.plane, deviceId, nativeCropProvider);
 				} catch (err) {
 					this.opts.logger?.warn("Pipeline child execution failed — keeping parent detection", {
-						tags: { deviceId },
+						tags: logTags,
 						meta: {
 							rootStepId: rootStep.stepId,
 							parentClass: mutable.macroClass,
@@ -6500,6 +7303,34 @@ function parseWavToAudioChunk(filePath) {
 function enginesEqual(a, b) {
 	return a.runtime === b.runtime && a.backend === b.backend && a.format === b.format && (a.device ?? null) === (b.device ?? null);
 }
+/** A dispatch's load refused before it was issued: a back-off, or a refusal by name. */
+var ModelLoadRefusedError = class extends Error {
+	model;
+	state;
+	constructor(message, model, state) {
+		super(message);
+		this.name = "ModelLoadRefusedError";
+		this.model = model;
+		this.state = state;
+	}
+};
+/** `step/model` — how the governor and the log lines name a model. */
+function stepModelKey(step) {
+	return `${step.addonId}/${step.modelId}`;
+}
+/** The reporting state a refusal verdict stands for. */
+function stateOf(verdict) {
+	return verdict.kind === "refused" ? "refused" : `failed:${verdict.generation}`;
+}
+/**
+* The reason a condemned pool is charged against its device's restart budget.
+* A pool recycled because it HUNG (D653) names the hang — `inference device
+* FAILED … reason: pool worker is not ready` could not tell a GPU compile that
+* never returned from a crash.
+*/
+function deathReasonOf(factory) {
+	return factory.getDeathCause()?.message ?? "pool worker is not ready";
+}
 /** Build a `RuntimeEnv` from the running process + probed hardware. */
 function runtimeEnvFromProcess(hardware) {
 	return {
@@ -6541,25 +7372,6 @@ var PROBE_READY_TIMEOUT_MS = 12e4;
 * race bound keeps a wedged transport from wedging `setApi`.
 */
 var PROBE_DIRECT_RPC_TIMEOUT_MS = 6e4;
-/**
-* Build the onnx-cpu floor pick using `pickBestRuntime` with a null hardware
-* env.  Used wherever the old `detectBestEngine()` sync probe fell back — the
-* result is identical (onnx / cpu) but is now derived through the shared rules
-* instead of duplicated inline.
-*/
-/**
-* Is `backend` even POSSIBLE on this node's OS/arch (ignoring gpu detail)?
-* coreml ⇒ darwin only; openvino ⇒ x64 non-darwin only; onnx ⇒ anywhere. Used to
-* reject a persisted/global engine choice that the node's PLATFORM fundamentally
-* cannot run (e.g. the cluster's OpenVINO default landing on a Mac) — distinct
-* from the gpu-dependent support (linux without a probed Intel iGPU still keeps
-* openvino as a valid platform choice; the device falls back to cpu).
-*/
-function backendPossibleOnPlatform(backend) {
-	if (backend === "coreml") return process.platform === "darwin";
-	if (backend === "openvino") return process.arch === "x64" && process.platform !== "darwin";
-	return true;
-}
 var DetectionPipelineProvider = class DetectionPipelineProvider {
 	modelsDir;
 	eventBus;
@@ -6699,22 +7511,6 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	*/
 	needsAutoPick = false;
 	/**
-	* Warm cache for benchmark engine-override runs.
-	*
-	* Each override rebuild costs a full Python pool spin-up (~300-500ms)
-	* plus the per-model load. Benchmark tabs iterate up to 800× against
-	* the same override (e.g. yolov9s / coreml / all) — paying that spin-up
-	* each iteration dwarfs actual inference time.
-	*
-	* When the same override engine is requested within `OVERRIDE_CACHE_TTL_MS`,
-	* we reuse the prior transient factory. Hitting a different override
-	* or exceeding the TTL disposes the cache and spins a new factory.
-	* The prior "main" factory (`priorFactory`) is still restored so
-	* runtime dispatch (camera frames) keeps its own resident engine.
-	*/
-	overrideCache = null;
-	overrideCacheTimer = null;
-	/**
 	* Idle-TTL for a per-device inference pool (multi-device C3). A device pool
 	* with no dispatch for this long is disposed and recreated on next use — the
 	* memory guardrail that keeps `factoriesByDevice` from accumulating warm pools
@@ -6747,7 +7543,6 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 			error: err instanceof Error ? err.message : String(err)
 		} })
 	});
-	static OVERRIDE_CACHE_TTL_MS = 6e4;
 	/**
 	* RSS bound + periodic memory telemetry for the Python inference pools
 	* (2026-08-18 OOM audit: the pools, not the recorder, were the host's OOM
@@ -6884,15 +7679,14 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	/**
 	* Lazy-install the pip requirements file matching `engine.backend` into
 	* the embedded Python before the inference pool spawns. No-op for
-	* `runtime: 'node'`, for backends with no requirements file on disk,
-	* or before the addon context has been wired (warm-pool race window).
+	* backends with no requirements file on disk, or before the addon
+	* context has been wired (warm-pool race window).
 	*
 	* Idempotent — `installPythonRequirements` short-circuits on a hash
 	* marker, so back-to-back EngineFactory rebuilds (benchmark overrides,
 	* tuning respawns) skip the pip subprocess after the first install.
 	*/
 	async ensureBackendDeps(engine) {
-		if (engine.runtime !== "python") return;
 		if (!this.addonCtx) return;
 		const pythonAddonDir = this.executorOptions.pythonAddonDir;
 		if (!pythonAddonDir) return;
@@ -7457,8 +8251,22 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* result with a warning logged, same posture as `recomputeConfigIssues`.
 	*/
 	async validatePipeline(input) {
-		const format = this.currentEngine.format;
+		let format = this.currentEngine.format;
 		try {
+			if (input.deviceKey !== void 0) try {
+				format = resolveDeviceEngine(input.deviceKey).format;
+			} catch (err) {
+				return {
+					ok: false,
+					issues: [{
+						addonId: "*",
+						kind: "unknown-device",
+						detail: errMsg(err)
+					}],
+					substitutions: [],
+					format
+				};
+			}
 			const steps = input.steps;
 			const enabledSteps = this.pruneToEnabledSteps(steps);
 			const substitutions = collectSubstitutions(enabledSteps, format);
@@ -7783,7 +8591,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		const nodeId = this.addonCtx?.kernel?.localNodeId ?? "hub";
 		const sessionId = input.sessionId ?? `run-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
 		const isRuntime = Boolean(input.frame || input.frameRef);
-		const progressToBus = !isRuntime || input.sessionId !== void 0;
+		const progressToBus = !isRuntime && input.replay !== true || input.sessionId !== void 0;
 		const emit = (message, extra) => {
 			onProgress?.(message);
 			if (!progressToBus) return;
@@ -7925,7 +8733,6 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		const dispatchResolution = this.resolveStepsForDispatch({
 			steps: input.steps,
 			deviceKey: input.deviceKey,
-			engineOverride: input.engine,
 			deviceId: input.deviceId,
 			plane: input.plane
 		});
@@ -7945,322 +8752,153 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		}
 		const enabledSteps = flattenSteps(benchmarkSteps);
 		emit(`Pipeline: ${enabledSteps.length} step(s) — ${enabledSteps.map((s) => s.addonId).join(" → ")}`);
-		await this.ensureEngineFactory();
-		const priorEngine = this.currentEngine;
-		const priorFactory = this.engineFactory;
-		const priorExecutor = this.executor;
-		const runEngine = input.engine ? {
-			runtime: input.engine.runtime,
-			backend: input.engine.backend,
-			format: input.engine.format,
-			...input.engine.device || priorEngine.device ? { device: input.engine.device ?? priorEngine.device } : {}
-		} : priorEngine;
-		const differs = input.engine && (priorEngine.runtime !== runEngine.runtime || priorEngine.backend !== runEngine.backend || priorEngine.format !== runEngine.format || (priorEngine.device ?? "") !== (runEngine.device ?? ""));
-		if (differs) if (this.overrideCache && enginesEqual(this.overrideCache.engine, runEngine)) {
-			const cached = this.overrideCache;
-			if (cached.initPromise) await cached.initPromise;
-			this.log.debug("Benchmark: reusing warm override factory", { meta: { engine: `${runEngine.runtime}/${runEngine.backend}/${runEngine.device ?? "default"}` } });
-			this.currentEngine = runEngine;
-			this.engineFactory = cached.factory;
-			this.executor = cached.executor;
-		} else {
-			if (this.overrideCache) {
-				try {
-					await this.overrideCache.factory.dispose();
-				} catch {}
-				this.overrideCache = null;
-			}
-			this.log.info("Benchmark: engine override", { meta: {
-				from: `${priorEngine.runtime}/${priorEngine.backend}`,
-				to: `${runEngine.runtime}/${runEngine.backend}`
-			} });
-			await this.ensureBackendDeps(runEngine);
-			const newFactory = new EngineFactory({
-				engine: runEngine,
-				modelsDir: this.modelsDir,
-				logger: this.log.child("engine-override"),
-				pythonPath: this.executorOptions.pythonPath ?? "",
-				provisioning: this.executorOptions.provisioning,
-				resolveCustomModel: this.customModelResolver
-			});
-			const initPromise = newFactory.initialize([]);
-			this.overrideCache = {
-				engine: runEngine,
-				factory: newFactory,
-				executor: null,
-				lastUsedMs: Date.now(),
-				initPromise
-			};
-			this.scheduleOverrideCacheEviction();
-			try {
-				await initPromise;
-				this.overrideCache.initPromise = null;
-			} catch (err) {
-				this.overrideCache = null;
-				if (this.overrideCacheTimer) {
-					clearTimeout(this.overrideCacheTimer);
-					this.overrideCacheTimer = null;
-				}
-				throw err;
-			}
-			this.currentEngine = runEngine;
-			this.engineFactory = newFactory;
-			this.executor = null;
-		}
-		try {
-			const deviceLabel = runEngine.device ?? "default";
-			emit(`Engine: ${runEngine.runtime}/${runEngine.backend}/${deviceLabel}${input.engine ? " (override)" : ""}`, {
-				step: "engine",
-				engine: {
-					runtime: runEngine.runtime,
-					backend: runEngine.backend,
-					device: deviceLabel
-				}
-			});
-			const runtimeStr = `${runEngine.runtime}+${runEngine.backend}`;
-			const executor = this.executor ?? new PipelineExecutor({
-				engineRuntime: runtimeStr,
-				logger: this.log
+		const rootModelId = resolveRootModelId(benchmarkSteps);
+		const stampRoot = (r) => rootModelId === void 0 ? r : {
+			...r,
+			rootModelId
+		};
+		if (input.replay === true) {
+			const verdict = checkReplayPin({
+				requested: input.steps,
+				executing: input.plane === "frame" ? collectFramePlaneSteps(benchmarkSteps, isDetailPlaneStep) : enabledSteps,
+				requestedDeviceKey: input.deviceKey,
+				dispatchDeviceKey,
+				format: dispatchDeviceKey ? resolveDeviceEngine(dispatchDeviceKey).format : this.currentEngine.format
 			});
-			this.executor = executor;
-			const dispatchEngine = dispatchDeviceKey ? resolveDeviceEngine(dispatchDeviceKey) : runEngine;
-			const dispatchFactory = dispatchDeviceKey ? await this.resolveDeviceFactory(dispatchDeviceKey) : this.engineFactory;
-			const needed = enabledSteps.filter((s) => dispatchFactory.needsPoolUpdate(s));
-			if (needed.length > 0) {
-				this.log.info("Benchmark: models to load", { meta: {
-					count: needed.length,
-					models: needed.map((s) => `${s.addonId}/${s.modelId}`)
-				} });
-				for (const s of needed) emit(`Loading model: ${s.addonName} (${s.modelId})...`, {
-					step: s.addonId,
-					addonId: s.addonId,
-					modelId: s.modelId
+			if (verdict.kind === "refused") {
+				this.log.warn("replay refused — the node cannot run exactly what was pinned", {
+					...input.deviceId !== void 0 ? { tags: { deviceId: input.deviceId } } : {},
+					meta: {
+						reason: verdict.reason,
+						detail: verdict.detail,
+						deviceKey: input.deviceKey
+					}
 				});
-				await this.ensureModelsForSteps(benchmarkSteps, dispatchFactory, dispatchEngine.format);
-				emit(`All models loaded`);
-			}
-			emit("Running inference...");
-			const tree = buildExecutableTree(benchmarkSteps, (stepId) => dispatchFactory.getEngine(stepId), this.customModelResolver);
-			setupMs = performance.now() - wallT0 - decodeMs;
-			const effectiveDeviceId = input.deviceId ?? 0;
-			const deviceOverrides = effectiveDeviceId > 0 ? await this.deviceOverrides.resolve(effectiveDeviceId) : {};
-			const effectiveTree = Object.keys(deviceOverrides).length > 0 ? applyDeviceOverridesToTree(tree, "object-detection", deviceOverrides) : tree;
-			const nativeCropProvider = this.buildNativeCropProviderFromRef(input.nativeCropRef) ?? this.buildNativeFaceCropProvider(input.frameHandle);
-			let cropZoneBbox;
-			if (isRuntime && effectiveDeviceId > 0 && effectiveTree.roots.some((r) => r.definition.extractMode === "crop-zone")) {
-				await this.ensureDeviceProxy(effectiveDeviceId);
-				const proxy = this.deviceProxies.get(effectiveDeviceId);
-				if (proxy) cropZoneBbox = resolvePackageCropBbox(proxy.state.zones.value?.zones ?? [], proxy.state.zoneRules.value?.package ?? [], imageWidth, imageHeight) ?? void 0;
-			}
-			let frameViewResolver;
-			let rootInputViewProvider;
-			if (runtimeFrameRef) {
-				frameViewResolver = createRootFrameViewResolver(localFrameRegistry, runtimeFrameRef);
-				rootInputViewProvider = async (root, crop) => {
-					const view = await frameViewResolver(root, crop ? {
-						left: crop[0],
-						top: crop[1],
-						width: crop[2] - crop[0],
-						height: crop[3] - crop[1]
-					} : void 0);
-					const data = Buffer.from(view.data.buffer, view.data.byteOffset, view.data.byteLength);
-					return {
-						input: {
-							kind: "jpeg",
-							data
-						},
-						jpegProvider: async () => data,
-						width: view.width,
-						height: view.height,
-						geometry: view.geometry
-					};
-				};
-			}
-			let executionSucceeded = false;
-			let execution;
-			try {
-				execution = await executor.run(effectiveTree, rootInput, jpegProvider, imageWidth, imageHeight, effectiveDeviceId, {
-					traceVerbosity: isRuntime ? this.eventBus ? "summary" : "off" : "full",
-					plane: input.plane
-				}, nativeCropProvider, cropZoneBbox, rootInputViewProvider);
-				executionSucceeded = true;
-			} finally {
-				frameViewResolver?.release(executionSucceeded ? "success" : "error");
-			}
-			const { result, trace } = execution;
-			if (isRuntime) {
-				if (trace && this.eventBus) this.eventBus.emit(createEvent(EventCategory.PipelineTrace, {
-					type: "device",
-					id: trace.deviceId,
-					nodeId: "hub"
-				}, trace));
-				if (effectiveDeviceId > 0) {
-					await this.ensureDeviceProxy(effectiveDeviceId);
-					return this.gateDetectionsByZoneRules(effectiveDeviceId, result);
-				}
-				return result;
-			}
-			for (const t of result.debug?.stepTimings ?? []) emit(`${t.source}${t.modelId ? ` (${t.modelId})` : ""} → ${t.ms}ms`, {
-				step: t.source,
-				addonId: t.source,
-				modelId: t.modelId ?? void 0,
-				ms: t.ms
-			});
-			emit(`Done — ${result.detections.length} detection(s) in ${result.debug?.totalInferenceMs ?? 0}ms`, { ms: result.debug?.totalInferenceMs });
-			const wallMs = performance.now() - wallT0;
-			const inferMs = result.debug?.totalInferenceMs ?? 0;
-			const overheadMs = Math.max(0, wallMs - decodeMs - setupMs - inferMs);
-			return {
-				...result,
-				debug: {
-					...result.debug,
-					decodeMs: Math.round(decodeMs * 100) / 100,
-					setupMs: Math.round(setupMs * 100) / 100,
-					wallMs: Math.round(wallMs * 100) / 100,
-					overheadMs: Math.round(overheadMs * 100) / 100
-				}
-			};
-		} finally {
-			if (differs && this.overrideCache) {
-				this.overrideCache.executor = this.executor;
-				this.overrideCache.lastUsedMs = Date.now();
-				this.scheduleOverrideCacheEviction();
+				throw new Error(`${verdict.reason}: ${verdict.detail}`);
 			}
-			if (differs) {
-				this.currentEngine = priorEngine;
-				this.engineFactory = priorFactory;
-				this.executor = priorExecutor;
-			}
-		}
-	}
-	/**
-	* Apply a benchmark engine override (mirroring the body of `runPipeline`
-	* lines 863-953). Swaps `currentEngine` / `engineFactory` / `executor`
-	* to the override's warm factory and returns a restore function the
-	* caller MUST invoke in `finally`. When `override` is undefined or
-	* matches the current engine, this is a no-op and the restore function
-	* does nothing — same shape so callers don't need a conditional path.
-	*
-	* Used by both `runPipeline` (single-frame benchmark) and
-	* `runPipelineBatch` (batched fast path). Without this, the batch
-	* path silently ignored `input.engine` and ran the user's override
-	* against the addon's saved engine — making the override matrix in
-	* the bench UI a no-op for batchSize > 1.
-	*/
-	async applyEngineOverride(override) {
-		const priorEngine = this.currentEngine;
-		const priorFactory = this.engineFactory;
-		const priorExecutor = this.executor;
-		if (!override) return () => {};
-		if (!backendPossibleOnPlatform(override.backend)) {
-			this.log.warn("Engine override impossible on this platform — running on own engine", { meta: {
-				override: `${override.runtime}/${override.backend}/${override.device ?? "default"}`,
-				platform: process.platform,
-				arch: process.arch,
-				engine: `${priorEngine.runtime}/${priorEngine.backend}`
-			} });
-			return () => {};
 		}
-		const runEngine = {
-			runtime: override.runtime,
-			backend: override.backend,
-			format: override.format,
-			...override.device || priorEngine.device ? { device: override.device ?? priorEngine.device } : {}
-		};
-		if (!(priorEngine.runtime !== runEngine.runtime || priorEngine.backend !== runEngine.backend || priorEngine.format !== runEngine.format || (priorEngine.device ?? "") !== (runEngine.device ?? ""))) return () => {};
-		if (this.overrideCache && enginesEqual(this.overrideCache.engine, runEngine)) {
-			const cached = this.overrideCache;
-			if (cached.initPromise) await cached.initPromise;
-			this.log.debug("Benchmark: reusing warm override factory", { meta: { engine: `${runEngine.runtime}/${runEngine.backend}/${runEngine.device ?? "default"}` } });
-			this.currentEngine = runEngine;
-			this.engineFactory = cached.factory;
-			this.executor = cached.executor;
-		} else {
-			if (this.overrideCache) {
-				try {
-					await this.overrideCache.factory.dispose();
-				} catch {}
-				this.overrideCache = null;
+		const appliesCameraGates = isRuntime || input.replay === true;
+		await this.ensureEngineFactory();
+		const runEngine = this.currentEngine;
+		const deviceLabel = runEngine.device ?? "default";
+		emit(`Engine: ${runEngine.runtime}/${runEngine.backend}/${deviceLabel}`, {
+			step: "engine",
+			engine: {
+				runtime: runEngine.runtime,
+				backend: runEngine.backend,
+				device: deviceLabel
 			}
-			this.log.info("Benchmark: engine override", { meta: {
-				from: `${priorEngine.runtime}/${priorEngine.backend}`,
-				to: `${runEngine.runtime}/${runEngine.backend}`
+		});
+		const runtimeStr = `${runEngine.runtime}+${runEngine.backend}`;
+		const executor = this.executor ?? new PipelineExecutor({
+			engineRuntime: runtimeStr,
+			logger: this.log
+		});
+		this.executor = executor;
+		const dispatchEngine = dispatchDeviceKey ? resolveDeviceEngine(dispatchDeviceKey) : runEngine;
+		const dispatchFactory = dispatchDeviceKey ? await this.resolveDeviceFactory(dispatchDeviceKey) : this.engineFactory;
+		const needed = enabledSteps.filter((s) => dispatchFactory.needsPoolUpdate(s));
+		if (needed.length > 0) {
+			this.log.info("Benchmark: models to load", { meta: {
+				count: needed.length,
+				models: needed.map((s) => `${s.addonId}/${s.modelId}`)
 			} });
-			await this.ensureBackendDeps(runEngine);
-			const newFactory = new EngineFactory({
-				engine: runEngine,
-				modelsDir: this.modelsDir,
-				logger: this.log.child("engine-override"),
-				pythonPath: this.executorOptions.pythonPath ?? "",
-				provisioning: this.executorOptions.provisioning,
-				resolveCustomModel: this.customModelResolver
+			for (const s of needed) emit(`Loading model: ${s.addonName} (${s.modelId})...`, {
+				step: s.addonId,
+				addonId: s.addonId,
+				modelId: s.modelId
 			});
-			const initPromise = newFactory.initialize([]);
-			this.overrideCache = {
-				engine: runEngine,
-				factory: newFactory,
-				executor: null,
-				lastUsedMs: Date.now(),
-				initPromise
+			await this.ensureModelsForSteps(benchmarkSteps, dispatchFactory, dispatchEngine.format, {
+				...input.deviceId !== void 0 ? { deviceId: input.deviceId } : {},
+				deviceKey: deviceKeyOf(dispatchEngine)
+			});
+			emit(`All models loaded`);
+		}
+		emit("Running inference...");
+		const tree = buildExecutableTree(benchmarkSteps, (stepId) => dispatchFactory.getEngine(stepId), this.customModelResolver);
+		setupMs = performance.now() - wallT0 - decodeMs;
+		const effectiveDeviceId = input.deviceId ?? 0;
+		const deviceOverrides = effectiveDeviceId > 0 ? await this.deviceOverrides.resolve(effectiveDeviceId) : {};
+		const effectiveTree = Object.keys(deviceOverrides).length > 0 ? applyDeviceOverridesToTree(tree, "object-detection", deviceOverrides) : tree;
+		const nativeCropProvider = this.buildNativeCropProviderFromRef(input.nativeCropRef) ?? this.buildNativeFaceCropProvider(input.frameHandle);
+		let cropZoneBbox;
+		if (appliesCameraGates && effectiveDeviceId > 0 && effectiveTree.roots.some((r) => r.definition.extractMode === "crop-zone")) {
+			await this.ensureDeviceProxy(effectiveDeviceId);
+			const proxy = this.deviceProxies.get(effectiveDeviceId);
+			if (proxy) cropZoneBbox = resolvePackageCropBbox(proxy.state.zones.value?.zones ?? [], proxy.state.zoneRules.value?.package ?? [], imageWidth, imageHeight) ?? void 0;
+		}
+		let frameViewResolver;
+		let rootInputViewProvider;
+		if (runtimeFrameRef) {
+			frameViewResolver = createRootFrameViewResolver(localFrameRegistry, runtimeFrameRef);
+			rootInputViewProvider = async (root, crop) => {
+				const view = await frameViewResolver(root, crop ? {
+					left: crop[0],
+					top: crop[1],
+					width: crop[2] - crop[0],
+					height: crop[3] - crop[1]
+				} : void 0);
+				const data = Buffer.from(view.data.buffer, view.data.byteOffset, view.data.byteLength);
+				return {
+					input: {
+						kind: "jpeg",
+						data
+					},
+					jpegProvider: async () => data,
+					width: view.width,
+					height: view.height,
+					geometry: view.geometry
+				};
 			};
-			this.scheduleOverrideCacheEviction();
-			try {
-				await initPromise;
-				this.overrideCache.initPromise = null;
-			} catch (err) {
-				this.overrideCache = null;
-				if (this.overrideCacheTimer) {
-					clearTimeout(this.overrideCacheTimer);
-					this.overrideCacheTimer = null;
-				}
-				throw err;
+		}
+		let executionSucceeded = false;
+		let execution;
+		try {
+			execution = await executor.run(effectiveTree, rootInput, jpegProvider, imageWidth, imageHeight, effectiveDeviceId, {
+				traceVerbosity: isRuntime ? this.eventBus ? "summary" : "off" : "full",
+				plane: input.plane,
+				occasion: input.replay === true ? "replay" : "live"
+			}, nativeCropProvider, cropZoneBbox, rootInputViewProvider);
+			executionSucceeded = true;
+		} finally {
+			frameViewResolver?.release(executionSucceeded ? "success" : "error");
+		}
+		const { result, trace } = execution;
+		if (isRuntime) {
+			if (trace && this.eventBus) this.eventBus.emit(createEvent(EventCategory.PipelineTrace, {
+				type: "device",
+				id: trace.deviceId,
+				nodeId: "hub"
+			}, trace));
+			if (effectiveDeviceId > 0) {
+				await this.ensureDeviceProxy(effectiveDeviceId);
+				return stampRoot(this.gateDetectionsByZoneRules(effectiveDeviceId, result));
 			}
-			this.currentEngine = runEngine;
-			this.engineFactory = newFactory;
-			this.executor = null;
+			return stampRoot(result);
 		}
-		return () => {
-			if (this.overrideCache) {
-				this.overrideCache.executor = this.executor;
-				this.overrideCache.lastUsedMs = Date.now();
-				this.scheduleOverrideCacheEviction();
+		for (const t of result.debug?.stepTimings ?? []) emit(`${t.source}${t.modelId ? ` (${t.modelId})` : ""} → ${t.ms}ms`, {
+			step: t.source,
+			addonId: t.source,
+			modelId: t.modelId ?? void 0,
+			ms: t.ms
+		});
+		emit(`Done — ${result.detections.length} detection(s) in ${result.debug?.totalInferenceMs ?? 0}ms`, { ms: result.debug?.totalInferenceMs });
+		const wallMs = performance.now() - wallT0;
+		const inferMs = result.debug?.totalInferenceMs ?? 0;
+		const overheadMs = Math.max(0, wallMs - decodeMs - setupMs - inferMs);
+		const gated = input.replay === true && effectiveDeviceId > 0 ? await this.ensureDeviceProxy(effectiveDeviceId).then(() => this.gateDetectionsByZoneRules(effectiveDeviceId, result)) : result;
+		return {
+			...stampRoot(gated),
+			debug: {
+				...gated.debug,
+				decodeMs: Math.round(decodeMs * 100) / 100,
+				setupMs: Math.round(setupMs * 100) / 100,
+				wallMs: Math.round(wallMs * 100) / 100,
+				overheadMs: Math.round(overheadMs * 100) / 100
 			}
-			this.currentEngine = priorEngine;
-			this.engineFactory = priorFactory;
-			this.executor = priorExecutor;
 		};
 	}
 	/**
-	* Schedule eviction of the override cache after `OVERRIDE_CACHE_TTL_MS`
-	* of idleness. Rescheduling resets the timer so an active benchmark
-	* keeps its warm factory alive until iterations stop arriving.
-	*/
-	scheduleOverrideCacheEviction() {
-		if (this.overrideCacheTimer) clearTimeout(this.overrideCacheTimer);
-		this.overrideCacheTimer = setTimeout(() => {
-			this.evictOverrideCache("idle TTL");
-		}, DetectionPipelineProvider.OVERRIDE_CACHE_TTL_MS);
-	}
-	async evictOverrideCache(reason) {
-		if (!this.overrideCache) return;
-		const entry = this.overrideCache;
-		this.overrideCache = null;
-		if (this.overrideCacheTimer) {
-			clearTimeout(this.overrideCacheTimer);
-			this.overrideCacheTimer = null;
-		}
-		try {
-			await entry.factory.dispose();
-			this.log.info("Override cache evicted", { meta: {
-				reason,
-				engine: `${entry.engine.runtime}/${entry.engine.backend}/${entry.engine.device ?? "default"}`,
-				ageMs: Date.now() - entry.lastUsedMs
-			} });
-		} catch (err) {
-			this.log.warn("Override cache dispose failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
-		}
-	}
-	/**
 	* Convert PipelineStepInput[] → PipelineDefaultStep[] by enriching from
 	* StepDefinitions. This is the LIVE per-camera dispatch path — it runs
 	* once per decoded frame (up to detectionFps × N cameras/sec) — so it
@@ -8276,13 +8914,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* `resolveStepModels` (see its doc comment); a second writer here would
 	* corrupt it.
 	*
-	* `format` defaults to `this.currentEngine.format` (the LIVE per-camera
-	* dispatch path never passes it explicitly). Benchmark call sites with a
-	* per-run engine override (`input.engine`) MUST pass the override's format
-	* explicitly — `this.currentEngine` only gets swapped to the override
-	* LATER in `runPipeline`/`runPipelineBatch`, so relying on the default here
-	* would resolve models against the node's persisted format instead of the
-	* format actually being benchmarked.
+	* `format` defaults to `this.currentEngine.format` (the node-default
+	* pool); a `deviceKey` dispatch passes that device's format explicitly.
 	*/
 	inputStepsToPipelineSteps(steps, format = this.currentEngine.format, engine = {
 		backend: this.currentEngine.backend,
@@ -8364,8 +8997,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	*    devices — so the whole-tree gate stays.
 	*
 	* Fallback is strictly NODE-LOCAL and ordered: the node's own selected
-	* engine (or the benchmark's explicit `engineOverride`) is the one and only
-	* fallback — frames never migrate to another node from here (that is the
+	* engine is the one and only fallback — frames never migrate to another node from here (that is the
 	* orchestrator's tier). The fallback is logged ONCE per distinct
 	* (deviceKey, steps, format) signature, with `tags: { deviceId }` when the
 	* dispatch is camera-scoped.
@@ -8376,12 +9008,9 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* configuration must stay loud, not vanish into a silent fallback chain.
 	*/
 	resolveStepsForDispatch(args) {
-		const { deviceKey, engineOverride, deviceId } = args;
-		const nodeFormat = engineOverride?.format ?? this.currentEngine.format;
-		const nodeEngine = engineOverride ? {
-			backend: engineOverride.backend,
-			device: engineOverride.device ?? null
-		} : {
+		const { deviceKey, deviceId } = args;
+		const nodeFormat = this.currentEngine.format;
+		const nodeEngine = {
 			backend: this.currentEngine.backend,
 			device: this.currentEngine.device ?? null
 		};
@@ -8479,53 +9108,193 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		throw new Error(message);
 	}
 	/**
-	* Single-flight gate around `ensureModelsForSteps`. Concurrent
-	* callers (every camera that fires motion in the same window calls
-	* `runPipeline` → `ensureModelsForSteps`) share the same in-flight
-	* promise instead of each racing into `loadAdditional`. Without this
-	* the pool gets the SAME model loaded N times under N distinct
-	* pool indices because every caller saw `isLoadedWithModel === false`
-	* before any of them committed via `stepToLoaded.set`.
-	*/
-	modelLoadInFlight = null;
+	* In-flight model loads, per POOL and per MODEL SET (D653). Two dispatches
+	* needing the same models on the same pool share ONE load and its outcome;
+	* a dispatch needing a different set is never handed another set's failure
+	* — it queues behind the pool's current load ({@link modelLoadTail}) and
+	* then decides for itself. Keyed by pool, not node-wide: one GPU compile
+	* that never returned held EVERY load on the node — the NPU's and the CPU's
+	* too — behind it.
+	*/
+	modelLoadInFlight = /* @__PURE__ */ new Map();
+	/** The last load issued on each pool — loads into one pool run one at a time. */
+	modelLoadTail = /* @__PURE__ */ new Map();
+	/** Stands in for "the node-default pool" before it exists. */
+	nullPoolKey = {};
+	/** Negative cache, per-model compile-timeout refusals, per-camera reporting (D653). */
+	loadGovernor = new ModelLoadGovernor();
 	/** Ensure all models needed by steps are downloaded and loaded in the engine pool.
 	* `factory`/`format` default to the node's engine; a per-device dispatch passes
-	* that device's factory + format (Phase 2 multi-device). */
-	async ensureModelsForSteps(steps, factory = this.engineFactory, format = this.currentEngine?.format ?? "onnx") {
-		while (this.modelLoadInFlight) await this.modelLoadInFlight;
-		const allEnabled = flattenSteps(steps);
+	* that device's factory + format (Phase 2 multi-device). `context` names the
+	* camera and the accelerator on the line a rejected load writes. */
+	async ensureModelsForSteps(steps, factory = this.engineFactory, format = this.currentEngine?.format ?? "onnx", context = {}) {
+		const needed = this.stepsNeedingLoad(steps, factory);
+		if (needed.length === 0) return;
+		const pool = factory ?? this.nullPoolKey;
+		const deviceKey = context.deviceKey ?? deviceKeyOf(this.currentEngine);
+		const models = needed.map(stepModelKey);
+		const refusal = this.loadRefusal(pool, deviceKey, models);
+		if (refusal !== null) {
+			this.reportDispatchLoadLoss(context, deviceKey, models, refusal.model, refusal.state, refusal.message);
+			throw refusal;
+		}
+		const setKey = [...models].sort().join(",");
+		let perPool = this.modelLoadInFlight.get(pool);
+		if (perPool === void 0) {
+			perPool = /* @__PURE__ */ new Map();
+			this.modelLoadInFlight.set(pool, perPool);
+		}
+		const flight = perPool.get(setKey) ?? this.issueModelLoad(pool, perPool, setKey, steps, factory, format, deviceKey);
+		try {
+			await flight.work;
+		} catch (err) {
+			if (err instanceof ModelLoadRefusedError) {
+				this.reportDispatchLoadLoss(context, deviceKey, models, err.model, err.state, err.message);
+				throw err;
+			}
+			const failed = err instanceof StepVariantLoadError ? `${err.stepId}/${err.modelId}` : models[0] ?? "";
+			this.reportDispatchLoadLoss(context, deviceKey, models, failed, `failed:${flight.generation ?? "unknown"}`, errMsg(err));
+			throw err;
+		}
+	}
+	/**
+	* Why `models` may not be loaded on `pool` right now — a back-off after a
+	* failure, or a refusal by name after repeated compile timeouts — or `null`.
+	*/
+	loadRefusal(pool, deviceKey, models) {
+		const verdict = this.loadGovernor.check(pool, deviceKey, models);
+		if (verdict.kind === "go") return null;
+		return new ModelLoadRefusedError(verdict.kind === "refused" ? `model ${verdict.model} is refused on ${deviceKey}: its compile timed out ${verdict.timeouts} times (${verdict.error}) — re-arm with pipelineExecutor.rearmInferenceDevice` : `model ${verdict.model} failed to load on ${deviceKey} ${verdict.failures} time(s); not retried for ${verdict.retryInMs}ms (${verdict.error})`, verdict.model, stateOf(verdict));
+	}
+	/**
+	* A compile that outlived its soft bound came back on `factory`'s pool
+	* (D653 rounds 2-3). A SUCCESS wrote that model's cache and the worker loads
+	* again, so that model's back-off is lifted at once rather than waited out;
+	* no other model's is. A FAILURE is one more failure of that model, and its
+	* back-off grows.
+	*/
+	noteLateCompile(factory, deviceKey, event) {
+		const models = this.loadGovernor.settleLateCompile(factory, deviceKey, event.model, event.ok, event.error ?? "late compile failed");
+		if (event.ok) {
+			this.log.info("model loadable again after late compile", { meta: {
+				deviceKey,
+				model: event.model,
+				backoffLifted: models
+			} });
+			return;
+		}
+		this.log.warn("late compile failed — the model stays backed off", { meta: {
+			deviceKey,
+			model: event.model,
+			backoffExtended: models,
+			error: event.error ?? null
+		} });
+	}
+	/** The enabled steps the factory's pool does not reflect yet. */
+	stepsNeedingLoad(steps, factory) {
 		const needed = [];
-		for (const step of allEnabled) {
+		for (const step of flattenSteps(steps)) {
 			if (!step.enabled) continue;
 			if (factory !== null && !factory.needsPoolUpdate(step)) continue;
 			needed.push(step);
 		}
-		if (needed.length === 0) return;
-		const work = (async () => {
-			for (const step of needed) {
-				const modelEntry = await this.resolveModelEntry(step.addonId, step.modelId);
-				if (modelEntry && !isModelDownloaded(this.modelsDir, modelEntry, format)) {
-					if (modelEntry.formats[format] === void 0) throw new Error(`Model "${modelEntry.id}" has no ${format} format build (available: ${Object.keys(modelEntry.formats).join(", ") || "none"}) — not retrying a permanent format mismatch`);
-					if (modelEntry.formats[format]?.url.startsWith("camstack-local://") === true) throw new Error(`Custom model "${modelEntry.id}" (${format}) is not present in this node's models directory — distribute it to this node from Model Studio before selecting it here`);
-					this.log.info("Downloading model for step", { meta: {
-						modelId: step.modelId,
-						format,
-						step: step.addonId
-					} });
-					await this.downloadWithRetry(modelEntry, format, 3);
-				}
+		return needed;
+	}
+	/**
+	* Issue ONE load for a model set on a pool, queued behind the pool's
+	* previous load, and record its outcome with the governor. The set is
+	* re-derived once the queue reaches it: the load ahead may have brought
+	* some of it in.
+	*/
+	issueModelLoad(pool, perPool, setKey, steps, factory, format, deviceKey) {
+		const previous = this.modelLoadTail.get(pool) ?? Promise.resolve();
+		const flight = {
+			work: Promise.resolve(),
+			generation: null
+		};
+		flight.work = (async () => {
+			await previous;
+			const needed = this.stepsNeedingLoad(steps, factory);
+			if (needed.length === 0) return;
+			const models = needed.map(stepModelKey);
+			const refusal = this.loadRefusal(pool, deviceKey, models);
+			if (refusal !== null) throw refusal;
+			try {
+				await this.downloadNeededModels(needed, format);
+				this.log.info("Loading additional models for benchmark", { meta: {
+					count: needed.length,
+					models,
+					deviceKey
+				} });
+				await factory.loadAdditional(needed);
+			} catch (err) {
+				flight.generation = this.recordModelLoadFailure(pool, deviceKey, models, err);
+				throw err;
 			}
-			this.log.info("Loading additional models for benchmark", { meta: {
-				count: needed.length,
-				models: needed.map((s) => `${s.addonId}/${s.modelId}`)
-			} });
-			await factory.loadAdditional(needed);
+			this.loadGovernor.recordSuccess(pool, deviceKey, models);
 		})();
-		this.modelLoadInFlight = work;
-		try {
-			await work;
-		} finally {
-			if (this.modelLoadInFlight === work) this.modelLoadInFlight = null;
+		perPool.set(setKey, flight);
+		const tail = flight.work.catch(() => void 0);
+		this.modelLoadTail.set(pool, tail);
+		tail.then(() => {
+			if (perPool.get(setKey) === flight) perPool.delete(setKey);
+			if (this.modelLoadTail.get(pool) === tail) this.modelLoadTail.delete(pool);
+		});
+		return flight;
+	}
+	/** Charge a failed load to the model that failed; say so once if it is now refused. */
+	recordModelLoadFailure(pool, deviceKey, models, err) {
+		const failedModels = err instanceof StepVariantLoadError ? [`${err.stepId}/${err.modelId}`] : models;
+		const compileTimeout = err instanceof StepVariantLoadError && err.reason === "compile-timeout";
+		const poolModel = err instanceof StepVariantLoadError ? err.poolModel : null;
+		let generation = 0;
+		for (const model of failedModels) {
+			const outcome = this.loadGovernor.recordFailure(pool, deviceKey, model, errMsg(err), compileTimeout, Date.now(), poolModel, err instanceof StepVariantLoadError ? err.reason : null);
+			generation = outcome.generation;
+			if (outcome.refusedNow) this.log.error("model REFUSED on this device — its compile timed out repeatedly", { meta: {
+				deviceKey,
+				model,
+				compileTimeouts: outcome.compileTimeouts,
+				error: errMsg(err),
+				device: "keeps serving every other model",
+				rearm: "pipelineExecutor.rearmInferenceDevice"
+			} });
+		}
+		return generation;
+	}
+	/**
+	* One WARN per camera per state change of the model that cost it its frame.
+	* The ERROR for the failed load itself is written once, where the load
+	* failed (`Step variant load failed`); this line answers "which cameras
+	* did it cost?", tagged so it can be counted per camera.
+	*/
+	reportDispatchLoadLoss(context, deviceKey, models, model, state, error) {
+		if (!this.loadGovernor.shouldReport(context.deviceId, deviceKey, model, state)) return;
+		this.log.warn("model load for dispatch failed", {
+			...context.deviceId !== void 0 ? { tags: { deviceId: context.deviceId } } : {},
+			meta: {
+				deviceKey,
+				models,
+				model,
+				state,
+				error
+			}
+		});
+	}
+	/** Download any model of `needed` missing in `format` (the device's). */
+	async downloadNeededModels(needed, format) {
+		for (const step of needed) {
+			const modelEntry = await this.resolveModelEntry(step.addonId, step.modelId);
+			if (modelEntry && !isModelDownloaded(this.modelsDir, modelEntry, format)) {
+				if (modelEntry.formats[format] === void 0) throw new Error(`Model "${modelEntry.id}" has no ${format} format build (available: ${Object.keys(modelEntry.formats).join(", ") || "none"}) — not retrying a permanent format mismatch`);
+				if (modelEntry.formats[format]?.url.startsWith("camstack-local://") === true) throw new Error(`Custom model "${modelEntry.id}" (${format}) is not present in this node's models directory — distribute it to this node from Model Studio before selecting it here`);
+				this.log.info("Downloading model for step", { meta: {
+					modelId: step.modelId,
+					format,
+					step: step.addonId
+				} });
+				await this.downloadWithRetry(modelEntry, format, 3);
+			}
 		}
 	}
 	/** Download a model with retry + exponential backoff */
@@ -8626,12 +9395,11 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		const dispatchResolution = this.resolveStepsForDispatch({
 			steps: input.steps,
 			deviceKey: input.deviceKey,
-			engineOverride: input.engine,
 			deviceId: input.deviceId,
 			plane: void 0
 		});
 		const dispatchDeviceKey = dispatchResolution.deviceKey;
-		const resolveFormat = dispatchDeviceKey ? resolveDeviceEngine(dispatchDeviceKey).format : input.engine?.format ?? this.currentEngine.format;
+		const resolveFormat = dispatchDeviceKey ? resolveDeviceEngine(dispatchDeviceKey).format : this.currentEngine.format;
 		const benchmarkSteps = dispatchResolution.steps;
 		const enabledSteps = flattenEnabledVideoSteps(benchmarkSteps);
 		if (enabledSteps.length === 0) throw new Error("runPipelineBatch: no enabled steps");
@@ -8652,53 +9420,48 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		const uniformDims = input.frames.every((f) => f.width === firstFrame.width && f.height === firstFrame.height);
 		const singleRoot = enabledSteps.length === 1;
 		await this.ensureEngineFactory();
-		const restoreEngine = await this.applyEngineOverride(input.engine);
-		try {
-			const factory = dispatchDeviceKey ? await this.resolveDeviceFactory(dispatchDeviceKey) : this.engineFactory;
-			if (!factory) throw new Error("runPipelineBatch: factory not initialised");
-			if (enabledSteps.filter((s) => factory.needsPoolUpdate(s)).length > 0) await this.ensureModelsForSteps(benchmarkSteps, factory, resolveFormat);
-			const canFastPath = singleRoot && allRaw && uniformDims && factory.supportsBatch();
-			this.log.info("runPipelineBatch path decision", { meta: {
-				phase: "batch",
-				canFastPath,
-				singleRoot,
-				allRaw,
-				uniformDims,
-				supportsBatch: factory.supportsBatch()
-			} });
-			if (!canFastPath) return { results: await Promise.all(input.frames.map((frame) => this.runPipeline({
-				steps: input.steps,
-				frame,
-				deviceId: input.deviceId,
-				sessionId: input.sessionId,
-				deviceKey: dispatchDeviceKey
-			}))) };
-			const items = input.frames.map((frame) => ({
-				raw: Buffer.from(frame.data),
-				width: frame.width,
-				height: frame.height,
-				format: frame.format
-			}));
-			this.log.info("runPipelineBatch fast path dispatch", { meta: {
-				phase: "batch",
-				stepId: rootStep.addonId,
-				modelId: rootStep.modelId,
-				items: items.length,
-				totalRawBytes: items.reduce((s, it) => s + it.raw.length, 0)
-			} });
-			const start = performance.now();
-			const rawResults = await factory.batchInferRaw(rootStep.addonId, items, rootStep.modelId, input.frameId ?? 0);
-			const totalMs = performance.now() - start;
-			this.log.info("runPipelineBatch fast path returned", { meta: {
-				phase: "batch",
-				resultCount: rawResults.length,
-				ms: Math.round(totalMs)
-			} });
-			const perFrameMs = totalMs / rawResults.length;
-			return { results: rawResults.map((raw, i) => assembleBatchedFrameResult(raw, input.frames[i], rootStep.addonId, input.deviceId ?? 0, perFrameMs, perFrameMs)) };
-		} finally {
-			restoreEngine();
-		}
+		const factory = dispatchDeviceKey ? await this.resolveDeviceFactory(dispatchDeviceKey) : this.engineFactory;
+		if (!factory) throw new Error("runPipelineBatch: factory not initialised");
+		if (enabledSteps.filter((s) => factory.needsPoolUpdate(s)).length > 0) await this.ensureModelsForSteps(benchmarkSteps, factory, resolveFormat, { deviceKey: dispatchDeviceKey ?? deviceKeyOf(this.currentEngine) });
+		const canFastPath = singleRoot && allRaw && uniformDims && factory.supportsBatch();
+		this.log.info("runPipelineBatch path decision", { meta: {
+			phase: "batch",
+			canFastPath,
+			singleRoot,
+			allRaw,
+			uniformDims,
+			supportsBatch: factory.supportsBatch()
+		} });
+		if (!canFastPath) return { results: await Promise.all(input.frames.map((frame) => this.runPipeline({
+			steps: input.steps,
+			frame,
+			deviceId: input.deviceId,
+			sessionId: input.sessionId,
+			deviceKey: dispatchDeviceKey
+		}))) };
+		const items = input.frames.map((frame) => ({
+			raw: Buffer.from(frame.data),
+			width: frame.width,
+			height: frame.height,
+			format: frame.format
+		}));
+		this.log.info("runPipelineBatch fast path dispatch", { meta: {
+			phase: "batch",
+			stepId: rootStep.addonId,
+			modelId: rootStep.modelId,
+			items: items.length,
+			totalRawBytes: items.reduce((s, it) => s + it.raw.length, 0)
+		} });
+		const start = performance.now();
+		const rawResults = await factory.batchInferRaw(rootStep.addonId, items, rootStep.modelId, input.frameId ?? 0);
+		const totalMs = performance.now() - start;
+		this.log.info("runPipelineBatch fast path returned", { meta: {
+			phase: "batch",
+			resultCount: rawResults.length,
+			ms: Math.round(totalMs)
+		} });
+		const perFrameMs = totalMs / rawResults.length;
+		return { results: rawResults.map((raw, i) => assembleBatchedFrameResult(raw, input.frames[i], rootStep.addonId, input.deviceId ?? 0, perFrameMs, perFrameMs)) };
 	}
 	async listReferenceImages() {
 		const dir = this.resolveRefImagesDir();
@@ -8803,7 +9566,6 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		this.poolMemoryGuard = null;
 		this.inferenceTimeoutGuard?.stop();
 		this.inferenceTimeoutGuard = null;
-		await this.evictOverrideCache("shutdown");
 		if (this.engineFactory) {
 			await this.engineFactory.dispose();
 			this.engineFactory = null;
@@ -8885,13 +9647,22 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* own runtime's row, so this is not "the node's tuning" — the per-pool truth
 	* is on each pool's `Python inference pool` log line and in `listLoadedEngines`.
 	*/
-	async getEffectiveTuning() {
-		const p = resolvePoolProvisioning(this.currentEngine, this.executorOptions.provisioning);
+	/**
+	* One pool's provisioning — the device pool named by `deviceKey`, else the
+	* node default. Resolved with the SAME override `resolveDeviceFactory` and
+	* `ensureEngineFactory` build the pool with, so the answer describes the
+	* pool that ran the work (D646).
+	*/
+	async getEffectiveTuning(input = {}) {
+		const p = resolvePoolProvisioning(input.deviceKey !== void 0 ? resolveDeviceEngine(input.deviceKey) : this.currentEngine, this.executorOptions.provisioning);
 		return {
+			deviceKey: input.deviceKey ?? null,
+			runtime: p.runtime,
 			batchMode: p.batchMode,
 			windowMs: p.windowMs,
 			maxBatchSize: p.maxBatchSize,
-			concurrency: p.concurrency
+			concurrency: p.concurrency,
+			numWorkers: p.numWorkers
 		};
 	}
 	/**
@@ -8906,7 +9677,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 			if (this.engineFactory.isReady()) return;
 			const dead = this.engineFactory;
 			this.engineFactory = null;
-			this.noteDeviceDeath(defaultDeviceKey, "pool worker is not ready");
+			this.noteFactoryDeath(defaultDeviceKey, dead);
 			await dead.dispose().catch(() => void 0);
 			this.refuseIfDeviceUnusable(defaultDeviceKey);
 		}
@@ -8924,7 +9695,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				logger: this.log.child("engine"),
 				pythonPath: this.executorOptions.pythonPath ?? "",
 				provisioning: this.executorOptions.provisioning,
-				resolveCustomModel: this.customModelResolver
+				resolveCustomModel: this.customModelResolver,
+				onCompileFinishedLate: (event) => this.noteLateCompile(factory, defaultDeviceKey, event)
 			});
 			try {
 				await factory.initialize([]);
@@ -9044,15 +9816,35 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* dispose — which rejects whatever was still in flight on it with a reason
 	* rather than letting those requests sit until their own deadlines.
 	*/
-	async condemnDeviceFactory(deviceKey, factory, reason) {
+	async condemnDeviceFactory(deviceKey, factory) {
 		if (this.factoriesByDevice.get(deviceKey) === factory) {
 			this.factoriesByDevice.delete(deviceKey);
 			this.deviceReaper.cancel(deviceKey);
-			this.noteDeviceDeath(deviceKey, reason);
+			this.noteFactoryDeath(deviceKey, factory);
 		}
 		await factory.dispose().catch(() => void 0);
 	}
 	/**
+	* Charge one dead pool to its device — unless it died of a compile that was
+	* still running at its hard bound (D653, fix round 1). That death is not a
+	* crash of the DEVICE: the model that hung was already counted when its
+	* load answered `compile-timeout`, and the same model timing out again on
+	* the respawn is refused by name (`ModelLoadGovernor`), which is what bounds
+	* the respawns. A crash stays a crash.
+	*/
+	noteFactoryDeath(deviceKey, factory) {
+		const cause = factory.getDeathCause();
+		if (cause !== null && !cause.chargesDeviceBudget) {
+			this.log.warn("inference pool recycled after a compile hung — not charged to the device budget", { meta: {
+				deviceKey,
+				reason: cause.reason,
+				cause: cause.message
+			} });
+			return;
+		}
+		this.noteDeviceDeath(deviceKey, cause?.message ?? "pool worker is not ready");
+	}
+	/**
 	* Every inference device this node currently refuses, and why.
 	*
 	* The CHANNEL the 2026-08-26 analysis found missing. Pool health lived
@@ -9087,7 +9879,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				state: "backoff",
 				since: Date.now(),
 				deaths: 0,
-				lastError: "pool worker is not ready"
+				lastError: deathReasonOf(factory)
 			});
 		}
 		return { unhealthy: [...byKey.values()].toSorted((a, b) => a.deviceKey.localeCompare(b.deviceKey)) };
@@ -9104,10 +9896,13 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* no-op, reported as such.
 	*/
 	async rearmInferenceDevice(input) {
-		const rearmed = this.deviceLiveness.rearm(input.deviceKey);
+		const rearmedDevice = this.deviceLiveness.rearm(input.deviceKey);
+		const rearmedModels = this.loadGovernor.rearm(input.deviceKey);
+		const rearmed = rearmedDevice || rearmedModels > 0;
 		this.log.info("inference device re-armed by operator", { meta: {
 			deviceKey: input.deviceKey,
-			rearmed
+			rearmed,
+			rearmedModels
 		} });
 		return { rearmed };
 	}
@@ -9124,7 +9919,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				this.deviceReaper.touch(deviceKey);
 				return existing;
 			}
-			await this.condemnDeviceFactory(deviceKey, existing, "pool worker is not ready");
+			await this.condemnDeviceFactory(deviceKey, existing);
 			this.refuseIfDeviceUnusable(deviceKey);
 		}
 		const inflight = this.deviceFactoryInflight.get(deviceKey);
@@ -9137,7 +9932,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				logger: this.log.child(`engine:${deviceKey}`),
 				pythonPath: this.executorOptions.pythonPath ?? "",
 				provisioning: this.executorOptions.provisioning,
-				resolveCustomModel: this.customModelResolver
+				resolveCustomModel: this.customModelResolver,
+				onCompileFinishedLate: (event) => this.noteLateCompile(factory, deviceKey, event)
 			});
 			try {
 				await factory.initialize([]);
@@ -9279,9 +10075,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	}
 	async listLoadedEngines() {
 		const out = [];
-		const overrideFactory = this.overrideCache?.factory ?? null;
-		const runtimeFactoryIsOverride = this.engineFactory !== null && this.engineFactory === overrideFactory;
-		if (this.engineFactory && !runtimeFactoryIsOverride) {
+		if (this.engineFactory) {
 			const eng = this.currentEngine;
 			const engineKey = `${eng.runtime}/${eng.backend}/${eng.device ?? "default"}`;
 			const loaded = this.engineFactory.listLoaded();
@@ -9296,22 +10090,6 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				idleTtlMs: null
 			});
 		}
-		if (this.overrideCache) {
-			const eng = this.overrideCache.engine;
-			const engineKey = `${eng.runtime}/${eng.backend}/${eng.device ?? "default"} (warm)`;
-			const loaded = this.overrideCache.factory.listLoaded();
-			const idleMs = Math.max(0, Date.now() - this.overrideCache.lastUsedMs);
-			if (idleMs <= DetectionPipelineProvider.OVERRIDE_CACHE_TTL_MS) out.push({
-				engineKey,
-				engine: eng,
-				modelsLoaded: loaded.map((l) => `${l.stepId}/${l.modelId}`),
-				inUseByCameras: [],
-				kind: "warm-override",
-				poolPid: this.overrideCache.factory.getPoolPid(),
-				idleMs,
-				idleTtlMs: DetectionPipelineProvider.OVERRIDE_CACHE_TTL_MS
-			});
-		}
 		for (const [deviceKey, factory] of this.factoriesByDevice) {
 			const eng = resolveDeviceEngine(deviceKey);
 			const loaded = factory.listLoaded();
@@ -9340,14 +10118,6 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		return { success: true };
 	}
 	async killEngine(input) {
-		if (this.overrideCache && enginesEqual(this.overrideCache.engine, input.engine)) {
-			await this.evictOverrideCache("killEngine");
-			this.log.info("Override engine killed", { meta: {
-				...input.engine,
-				force: input.force ?? false
-			} });
-			return { success: true };
-		}
 		if (!this.engineFactory) return {
 			success: false,
 			reason: "not loaded"
@@ -9865,7 +10635,7 @@ var DetectionPipelineAddon = class extends BaseAddon {
 	/**
 	* Embedded Python path resolved once at boot via
 	* `ctx.deps.ensurePython()`. Empty string means the download failed
-	* or `runtime: 'node'` was selected — the provider's
+	* — the provider's
 	* `ensureBackendDeps` and EngineFactory's `initPythonPool` raise a
 	* clear error in that case.
 	*/