@camstack/addon-pipeline 1.2.295 → 1.2.297

package/dist/detection-pipeline/index.js

@@ -3,11 +3,11 @@ Object.defineProperties(exports, {
 	[Symbol.toStringTag]: { value: "Module" }
 });
 const require_chunk = require("../chunk-emK7D4bc.js");
-const require_dist = require("../dist-BsVcf5wO.js");
-const require_node = require("../node-UFk6I2f6.js");
-const require_default_detection_model = require("../default-detection-model-wNEISVA9.js");
+const require_dist = require("../dist-DUqr1zyq.js");
+const require_node = require("../node-Dx6DLQ1j.js");
+const require_default_detection_model = require("../default-detection-model-B2n_SFXt.js");
 const require_lazy_sharp = require("../lazy-sharp-BuwKlQLR.js");
-const require_process_memory = require("../process-memory-CJV29sPf.js");
+const require_process_memory = require("../process-memory-CMJ3NY-s.js");
 let node_os = require("node:os");
 node_os = require_chunk.__toESM(node_os);
 let node_fs = require("node:fs");
@@ -367,12 +367,14 @@ function normalizeEngineNodeId(rawNodeId) {
 * builders eventually reaches ten of them, and the cameras attached by the
 * forgotten one quietly write a foreign feature space into the shared index.
 *
-* ## Why the failure mode keeps the last value
+* ## Why the failure mode keeps the last value — and refuses before the first
 *
 * A failed read must never revert to the registry default MID-PASS: that would
 * silently change what every remaining vector of that dispatch means, which is
 * the exact class of harm the scope exists to prevent. So a failure keeps the
-* last known row and says so, loudly, once per failure.
+* last known row and says so, once per change. Before the FIRST answer there is
+* no row to keep: the step is refused by name (D658) — a default run then made
+* the default model the pool's active variant for the process lifetime.
 */
 /**
 * The addon whose GLOBAL settings own the cluster row. Same owner as the crop
@@ -381,35 +383,58 @@ function normalizeEngineNodeId(rawNodeId) {
 * enforces it live — neither should own a value the other depends on).
 */
 var CLUSTER_MODEL_OWNER_ADDON_ID = "pipeline-orchestrator";
-/** TTL-cached read of the cluster model row. */
+/**
+* TTL-cached read of the cluster model row.
+*
+* A step's model is KNOWN only once the owner has answered for it (`stored` or
+* `absent`). Until then it is UNKNOWN, and a cluster-scoped step is refused by
+* name rather than run with the registry default (D49, D658): a default run
+* during the owner's boot window made that model the pool's active variant for
+* the whole process, and the face default was ArcFace until D651. Once known,
+* an unanswered read keeps the LAST KNOWN model — never reverts to a default.
+*/
 var ClusterModelSource = class {
-	models = require_dist.DEFAULT_CLUSTER_STEP_MODELS;
+	models = {};
+	unknown = new Map(require_dist.CLUSTER_MODEL_SCOPED_STEPS.map((s) => [s.stepId, "not-yet-asked"]));
 	settings = require_dist.DEFAULT_CLUSTER_STEP_SETTINGS;
 	lastReadAtMs = Number.NEGATIVE_INFINITY;
 	inFlight = null;
 	logger;
 	now;
 	ttlMs;
+	unknownRetryMs;
+	/** The last "not known" state logged, so it is said once per change. */
+	lastUnknownLogKey = "";
 	constructor(options) {
 		this.logger = options.logger;
 		this.now = options.now ?? (() => Date.now());
 		this.ttlMs = options.ttlMs ?? 6e4;
+		this.unknownRetryMs = options.unknownRetryMs ?? 5e3;
 	}
-	/** The row in force right now — the registry defaults until a read lands. */
+	/**
+	* The KNOWN models, `stepId → modelId`. A step absent here is unknown (see
+	* {@link unknownSteps}) and must be refused, never defaulted.
+	*/
 	current() {
 		return this.models;
 	}
+	/** Cluster-scoped steps whose model the owner has not answered yet, with why. */
+	unknownSteps() {
+		return this.unknown;
+	}
 	/** Cluster-scoped step knobs in force right now (admission floors, …). */
 	currentSettings() {
 		return this.settings;
 	}
 	/**
-	* Re-read if the cached row has expired. Concurrent callers share one
-	* in-flight read; a failure keeps the LAST KNOWN row.
+	* Re-read if the cached row has expired (sooner while a step is unknown).
+	* Concurrent callers share one in-flight read; a failure keeps the LAST
+	* KNOWN row.
 	*/
 	async refresh(api) {
 		if (api === void 0) return this.models;
-		if (this.now() - this.lastReadAtMs < this.ttlMs) return this.models;
+		const ttl = this.unknown.size > 0 ? Math.min(this.ttlMs, this.unknownRetryMs) : this.ttlMs;
+		if (this.now() - this.lastReadAtMs < ttl) return this.models;
 		this.inFlight ??= this.read(api).finally(() => {
 			this.inFlight = null;
 		});
@@ -417,38 +442,186 @@ var ClusterModelSource = class {
 		return this.models;
 	}
 	async read(api) {
-		try {
-			const view = await api.addonSettings.getGlobalSettings.query({ addonId: CLUSTER_MODEL_OWNER_ADDON_ID });
-			const next = require_dist.pickClusterStepModels(view);
-			const nextSettings = require_dist.pickClusterStepSettings(view);
-			this.lastReadAtMs = this.now();
-			const changed = Object.keys(next).filter((stepId) => next[stepId] !== this.models[stepId]);
-			const settingsChanged = Object.keys(nextSettings).some((stepId) => {
-				const from = this.settings[stepId] ?? {};
-				const to = nextSettings[stepId] ?? {};
-				return [...new Set([...Object.keys(from), ...Object.keys(to)])].some((key) => from[key] !== to[key]);
-			});
-			if (changed.length > 0) {
-				this.logger.info("cluster step model changed — vectors already stored by the previous model need a re-embed pass", { meta: { changed: changed.map((stepId) => ({
-					stepId,
-					from: this.models[stepId],
-					to: next[stepId]
-				})) } });
-				this.models = next;
-			}
-			if (settingsChanged) {
-				this.logger.info("cluster step settings changed", { meta: {
-					from: { ...this.settings },
-					to: { ...nextSettings }
-				} });
-				this.settings = nextSettings;
+		this.lastReadAtMs = this.now();
+		await Promise.all([this.readModels(api), this.readSettings(api)]);
+	}
+	async readModels(api) {
+		const answers = await Promise.all(require_dist.CLUSTER_MODEL_SCOPED_STEPS.map(async (step) => ({
+			stepId: step.stepId,
+			answer: await askOwner(api, step.stepId)
+		})));
+		const nextModels = { ...this.models };
+		const nextUnknown = /* @__PURE__ */ new Map();
+		const unanswered = [];
+		for (const { stepId, answer } of answers) {
+			if (answer.kind === "model") {
+				nextModels[stepId] = answer.modelId;
+				continue;
 			}
+			const keeping = this.models[stepId] ?? null;
+			unanswered.push({
+				stepId,
+				reason: answer.reason,
+				keeping
+			});
+			if (keeping === null) nextUnknown.set(stepId, answer.reason);
+		}
+		const changed = Object.keys(nextModels).filter((stepId) => this.models[stepId] !== void 0 && nextModels[stepId] !== this.models[stepId]);
+		if (changed.length > 0) this.logger.info("cluster step model changed — vectors already stored by the previous model need a re-embed pass", { meta: { changed: changed.map((stepId) => ({
+			stepId,
+			from: this.models[stepId],
+			to: nextModels[stepId]
+		})) } });
+		const learned = Object.keys(nextModels).filter((stepId) => this.models[stepId] === void 0);
+		if (learned.length > 0) this.logger.info("cluster step model known — cluster-scoped steps run it", { meta: { known: Object.fromEntries(learned.map((id) => [id, nextModels[id]])) } });
+		this.models = nextModels;
+		this.unknown = nextUnknown;
+		this.reportUnanswered(unanswered);
+	}
+	/** Once per change of the unanswered set — never once per retry. */
+	reportUnanswered(unanswered) {
+		const key = JSON.stringify(unanswered);
+		if (key === this.lastUnknownLogKey) return;
+		this.lastUnknownLogKey = key;
+		if (unanswered.length === 0) return;
+		this.logger.warn("cluster step model not answered by its owner — a step never known is REFUSED, a known one keeps its last model", { meta: {
+			owner: CLUSTER_MODEL_OWNER_ADDON_ID,
+			unanswered: unanswered.map((u) => ({
+				stepId: u.stepId,
+				reason: u.reason,
+				...u.keeping !== null ? { keeping: u.keeping } : { refused: true }
+			}))
+		} });
+	}
+	async readSettings(api) {
+		let view;
+		try {
+			view = await api.addonSettings.getGlobalSettings.query({ addonId: CLUSTER_MODEL_OWNER_ADDON_ID });
 		} catch (err) {
-			this.logger.warn("cluster step model read failed — keeping the last known row", { meta: {
+			this.logger.warn("cluster step settings read failed — keeping the last known knobs", { meta: {
 				owner: CLUSTER_MODEL_OWNER_ADDON_ID,
-				inUse: { ...this.models },
 				error: err instanceof Error ? err.message : String(err)
 			} });
+			return;
+		}
+		if (view === null) return;
+		const nextSettings = require_dist.pickClusterStepSettings(view);
+		if (Object.keys(nextSettings).some((stepId) => {
+			const from = this.settings[stepId] ?? {};
+			const to = nextSettings[stepId] ?? {};
+			return [...new Set([...Object.keys(from), ...Object.keys(to)])].some((key) => from[key] !== to[key]);
+		})) {
+			this.logger.info("cluster step settings changed", { meta: {
+				from: { ...this.settings },
+				to: { ...nextSettings }
+			} });
+			this.settings = nextSettings;
+		}
+	}
+};
+/** One step's model from the owner's three-way answer; a throw is no answer. */
+async function askOwner(api, stepId) {
+	try {
+		const choice = await api.pipelineOrchestrator.getClusterModelChoice.query({ stepId });
+		switch (choice.kind) {
+			case "stored": return {
+				kind: "model",
+				modelId: choice.modelId
+			};
+			case "absent": return {
+				kind: "model",
+				modelId: choice.defaultModelId
+			};
+			case "unreadable": return {
+				kind: "none",
+				reason: choice.reason
+			};
+		}
+	} catch (err) {
+		return {
+			kind: "none",
+			reason: `read-failed: ${err instanceof Error ? err.message : String(err)}`
+		};
+	}
+}
+//#endregion
+//#region src/detection-pipeline/cluster-model-switch-log.ts
+function* clusterScopedSteps(steps) {
+	for (const step of steps) {
+		if (step.definition.modelScope === "cluster") yield step;
+		yield* clusterScopedSteps(step.children);
+	}
+}
+var ClusterModelSwitchLog = class {
+	logger;
+	/** `deviceId|stepId` → the model that step last ran for that camera. */
+	lastRun = /* @__PURE__ */ new Map();
+	constructor(logger) {
+		this.logger = logger;
+	}
+	/** Record what `tree` runs for `deviceId`; say so when a cluster step changed model. */
+	note(deviceId, tree) {
+		if (deviceId <= 0) return;
+		for (const step of clusterScopedSteps(tree.roots)) {
+			const key = `${String(deviceId)}|${step.stepId}`;
+			const previous = this.lastRun.get(key);
+			if (previous === step.modelId) continue;
+			this.lastRun.set(key, step.modelId);
+			if (previous === void 0) continue;
+			this.logger.info("cluster-scoped step re-provisioned for this camera — now runs the new model", {
+				tags: { deviceId },
+				meta: {
+					stepId: step.stepId,
+					from: previous,
+					to: step.modelId
+				}
+			});
+		}
+	}
+};
+function* enabledStepIds(steps) {
+	for (const step of steps) {
+		if (!step.enabled) continue;
+		yield step.addonId;
+		if (step.children) yield* enabledStepIds(step.children);
+	}
+}
+/** The enabled steps of `steps` (any depth) whose cluster model is unknown. */
+function unknownStepsNamed(steps, unknown) {
+	if (unknown.size === 0) return [];
+	return [...enabledStepIds(steps)].filter((id) => unknown.has(id));
+}
+/**
+* One line per (camera, step) while a cluster-scoped step is REFUSED because
+* the owner has not answered which model the cluster runs (D658). Said once per
+* episode: when every step becomes known the memory is cleared, so the next
+* episode is said again.
+*/
+var ClusterUnknownRefusalLog = class {
+	logger;
+	said = /* @__PURE__ */ new Set();
+	constructor(logger) {
+		this.logger = logger;
+	}
+	note(deviceId, steps, unknown) {
+		if (unknown.size === 0) {
+			if (this.said.size > 0) this.said = /* @__PURE__ */ new Set();
+			return;
+		}
+		if (deviceId <= 0) return;
+		for (const stepId of enabledStepIds(steps)) {
+			const reason = unknown.get(stepId);
+			if (reason === void 0) continue;
+			const key = `${String(deviceId)}|${stepId}`;
+			if (this.said.has(key)) continue;
+			this.said = new Set(this.said).add(key);
+			this.logger.warn("cluster-scoped step refused for this camera — the owner has not answered which model the cluster runs", {
+				tags: { deviceId },
+				meta: {
+					stepId,
+					reason
+				}
+			});
 		}
 	}
 };
@@ -569,6 +742,98 @@ var DeviceOverrideMirror = class {
 	}
 };
 //#endregion
+//#region src/detection-pipeline/engine/pool-worker-health.ts
+/**
+* The one table of load bounds (D653, fix round 1).
+*
+* The soft bound decides how soon the operator HEARS about a slow compile; the
+* hard bound decides when a compile is given up for hung. A single 120 s bound
+* that also killed the compile was wrong twice: a slow but finite compile was
+* killed before it could write its cache, so every respawn faced the same cold
+* compile and three of them walked the device to `failed`; and on CoreML the
+* bound was shorter than the cross-process compile-cache lock wait alone.
+*/
+var POOL_MODEL_LOAD_BOUNDS = {
+	openvino: {
+		softMs: 12e4,
+		hardMs: 6e5,
+		why: "A cold iGPU compile of the WHOLE model set measured ~25 s on the N100; one model is 1-3 s (YuNet 1.4 s on a fresh worker, 2026-09-26). 120 s is ~5x the worst measured set. No evidence of a legitimate compile past it; the hard bound gives one 5x more before the worker is recycled.",
+		gilMayBeHeldDuringCompile: false,
+		gilWhy: "The OpenVINO Python binding is believed to release the GIL around compile_model (gil_scoped_release). Not measured on the hub; the passive recipe in D653 checks it."
+	},
+	coreml: {
+		softMs: 3e5,
+		hardMs: 9e5,
+		why: "A load may first WAIT up to 180 s for another worker holding the compile-cache lock (`COMPILE_CACHE_LOCK_TIMEOUT_SEC` in inference_pool.py), then compile itself: 180 s of lock plus 120 s of compile. The old 120 s bound was shorter than the lock wait alone.",
+		gilMayBeHeldDuringCompile: true,
+		gilWhy: "Unverified whether coremltools releases the GIL while it compiles an mlpackage. If it does not, the loop freezes and nothing — not the soft timer, not mem_stats — can answer."
+	},
+	onnxruntime: {
+		softMs: 12e4,
+		hardMs: 6e5,
+		why: "Session creation, no persistent compile cache; CUDA/CoreML EP init is the slow case. Same numbers as OpenVINO for lack of any measurement saying otherwise.",
+		gilMayBeHeldDuringCompile: false,
+		gilWhy: "Not known to hold the GIL through session creation, and no frozen loop has been observed. Claiming the grace would blind stall detection during every load."
+	},
+	edgetpu: {
+		softMs: 12e4,
+		hardMs: 6e5,
+		why: "An Edge TPU model is precompiled; a load is an interpreter + delegate bind of seconds. Kept at the shared default rather than tightened without a measurement.",
+		gilMayBeHeldDuringCompile: false,
+		gilWhy: "Nothing is compiled: the load is an interpreter + delegate bind of seconds."
+	}
+};
+/**
+* The host-side deadline of a `load` / `replace`: the soft bound plus a margin,
+* so the worker's NAMED answer always lands first.
+*/
+var POOL_MODEL_LOAD_REPLY_MARGIN_MS = 3e4;
+function chargesDeviceBudget(reason) {
+	return reason !== "compile-hung";
+}
+/**
+* Tracks live inference outcomes on one worker and says when its executor has
+* produced nothing for too long. Consulted on demand — no timer.
+*
+* A SHED (`dropped`) does not count as a result: the Python loop answers sheds
+* itself, so a worker whose executor threads are hung keeps shedding happily.
+*/
+var InferStallDetector = class {
+	unanswered = 0;
+	lastResultAt;
+	/**
+	* When the current streak of unanswered requests began — the first one after
+	* a result. IDLE time is not silence: quiet cameras send nothing, and a
+	* worker that was asked nothing has failed nothing. Measuring from the last
+	* result let a 60 s lull plus the first 20 sheds of a burst (~8 s with six
+	* cameras) poison a healthy worker and charge the device.
+	*/
+	streakStartedAt = null;
+	constructor(now) {
+		this.lastResultAt = now;
+	}
+	/** A reply that carried a real result (or a real error) from the executor. */
+	noteResult(now) {
+		this.unanswered = 0;
+		this.lastResultAt = now;
+		this.streakStartedAt = null;
+	}
+	/**
+	* A live request that ended without a result (deadline expiry or a shed).
+	* Returns the stall when the worker has crossed both thresholds.
+	*/
+	noteUnanswered(now) {
+		this.unanswered += 1;
+		if (this.streakStartedAt === null) this.streakStartedAt = now;
+		const silentMs = now - Math.max(this.lastResultAt, this.streakStartedAt);
+		if (this.unanswered >= 20 && silentMs >= 3e4) return {
+			unanswered: this.unanswered,
+			silentMs
+		};
+		return null;
+	}
+};
+//#endregion
 //#region src/detection-pipeline/engine/shared-inference-pool.ts
 /**
 * SharedInferencePool — TypeScript wrapper for inference_pool.py.
@@ -610,6 +875,27 @@ var RAW_FMT_CODE = {
 	gray: 2
 };
 /**
+* A load the worker answered with a machine-readable failure class. The
+* provider tells a `compile-timeout` (counted against THAT MODEL on the
+* device) from an ordinary failure by `reason`, never by parsing text.
+*/
+var PoolModelLoadError = class extends Error {
+	reason;
+	/** The model's file stem, as the worker named it. */
+	model;
+	constructor(message, reason, model) {
+		super(message);
+		this.name = "PoolModelLoadError";
+		this.reason = reason;
+		this.model = model;
+	}
+};
+/**
+* Request id the worker uses for UNSOLICITED events — a reply to no request
+* (`compile-finished-late`). Never allocated to a request.
+*/
+var WORKER_EVENT_REQ_ID = 4294967295;
+/**
 * Per-inference-request reply timeout (ms). Turns a wedged request (worker
 * alive but no reply) into a rejection so every caller settles — runtime frame
 * dispatch drops the frame; the benchmark full-tree run rejects the wedged
@@ -618,6 +904,22 @@ var RAW_FMT_CODE = {
 * large frame) never trips; override via env for constrained hardware.
 */
 var POOL_INFER_TIMEOUT_MS = Math.max(1e3, Number(process.env["CAMSTACK_POOL_INFER_TIMEOUT_MS"]) || 6e4);
+/** Commands that compile a model — the only ones that get the load deadline. */
+var MODEL_LOAD_COMMANDS = new Set(["load", "replace"]);
+/**
+* Deadline of the liveness probe (`mem_stats`) sent after a command times out.
+* The worker answers `mem_stats` on its event loop, never behind a compile
+* (D653), so a healthy worker replies in milliseconds even mid-compile; ten
+* seconds only has to outlast a GC pause or a burst of inference replies.
+*/
+var POOL_LIVENESS_PROBE_TIMEOUT_MS = 1e4;
+/**
+* How long a poisoned worker may keep its in-flight LIVE requests before it is
+* killed anyway. A poisoned worker takes no new work, and every live request
+* carries a {@link POOL_LIVE_INFER_TIMEOUT_MS} deadline, so the drain is over
+* by then; the second of slack keeps the backstop from racing the last reply.
+*/
+var POOL_POISON_DRAIN_SLACK_MS = 1e3;
 /**
 * Max inference requests outstanding to ONE worker before new ones are SHED.
 *
@@ -805,6 +1107,27 @@ var PoolWorker = class {
 	wireVersion = 1;
 	nextRequestId = 1;
 	ready = false;
+	/** Set once this worker is declared unusable while ALIVE (D653). Never cleared:
+	*  a poisoned worker is recycled, not revived. */
+	poisonVerdict = null;
+	/** The kill of a poisoned worker has been started. */
+	recycling = false;
+	/** Kills a poisoned worker whose drain did not finish in time. */
+	recycleBackstop = null;
+	/** A liveness probe is in flight — one at a time, never a probe storm. */
+	probeInFlight = false;
+	/** The child has exited (any cause). */
+	exited = false;
+	/** A compile past its soft bound, still running in the worker (D653). */
+	abandonedCompile = null;
+	/** Says when live inference has produced nothing for too long (D653). */
+	inferStall = new InferStallDetector(Date.now());
+	/**
+	* A load's HOST deadline expired with no reply since the last reply of any
+	* kind (D653 round 3): the loop missed its own soft bound, so the GIL grace
+	* is over for this worker until something — anything — comes back.
+	*/
+	loadDeadlineMissed = false;
 	log;
 	opts;
 	constructor(opts) {
@@ -829,6 +1152,24 @@ var PoolWorker = class {
 	isReady() {
 		return this.ready;
 	}
+	/** Why this worker was recycled, typed for the provider's budget, or `null`. */
+	getDeathCause() {
+		const v = this.poisonVerdict;
+		const message = this.getPoisonDescription();
+		if (v === null || message === null) return null;
+		return {
+			reason: v.reason,
+			chargesDeviceBudget: chargesDeviceBudget(v.reason),
+			message
+		};
+	}
+	/** Why this live worker was declared unusable, or `null` (D653). */
+	getPoisonDescription() {
+		const v = this.poisonVerdict;
+		if (v === null) return null;
+		const target = v.command.model !== null ? ` of ${v.command.model}` : "";
+		return `${v.reason} on ${v.command.cmd}${target} (pid ${v.pid ?? "unknown"})`;
+	}
 	async initialize(initialModels) {
 		this.process = (0, node_child_process.spawn)(this.opts.pythonPath, [this.opts.scriptPath], { stdio: [
 			"pipe",
@@ -852,7 +1193,11 @@ var PoolWorker = class {
 		const spawnedProcess = this.process;
 		spawnedProcess.on("exit", (code, signal) => {
 			this.ready = false;
+			this.exited = true;
+			if (this.recycleBackstop) clearTimeout(this.recycleBackstop);
+			if (this.abandonedCompile) clearTimeout(this.abandonedCompile.hardTimer);
 			if (this.process !== spawnedProcess) return;
+			const poisonedBy = this.getPoisonDescription();
 			this.log.error("Worker process exited", { meta: {
 				worker: this.opts.workerLabel,
 				pid: spawnedProcess.pid ?? null,
@@ -860,9 +1205,10 @@ var PoolWorker = class {
 				device: this.opts.device ?? "default",
 				code,
 				signal,
-				inFlight: this.pending.size
+				inFlight: this.pending.size,
+				...poisonedBy !== null ? { poisonedBy } : {}
 			} });
-			this.rejectAll(/* @__PURE__ */ new Error(`PoolWorker[${this.opts.workerLabel}]: worker process exited (code=${code ?? "null"}, signal=${signal ?? "none"})`));
+			this.rejectAll(/* @__PURE__ */ new Error(`PoolWorker[${this.opts.workerLabel}]: worker process exited (code=${code ?? "null"}, signal=${signal ?? "none"})` + (poisonedBy !== null ? ` — recycled, poisoned by ${poisonedBy}` : "")));
 		});
 		this.process.stdout.on("data", (chunk) => {
 			const t0 = Date.now();
@@ -876,6 +1222,7 @@ var PoolWorker = class {
 			runtime: this.opts.poolRuntime,
 			concurrency: this.opts.concurrency,
 			protocolVersion: 2,
+			modelLoadTimeoutMs: POOL_MODEL_LOAD_BOUNDS[this.opts.poolRuntime].softMs,
 			models: initialModels.map((m) => serializeModelConfig(m))
 		};
 		if (this.opts.device) config["device"] = this.opts.device;
@@ -898,6 +1245,7 @@ var PoolWorker = class {
 					clearTimeout(timeout);
 					if (result["status"] === "ready") {
 						this.ready = true;
+						this.inferStall.noteResult(Date.now());
 						const loadedCount = result["models"];
 						const startupMs = result["startupMs"];
 						const workers = result["workers"] ?? 1;
@@ -920,7 +1268,7 @@ var PoolWorker = class {
 	async infer(modelByte, jpeg, deviceId) {
 		this.ensureReady();
 		const payload = Buffer.concat([Buffer.from([modelByte]), jpeg]);
-		return this.dispatch(MSG_INFER_JPEG, payload, deviceId);
+		return this.dispatch(MSG_INFER_JPEG, payload, { deviceId });
 	}
 	async inferRaw(modelByte, raw, width, height, format, deviceId) {
 		this.ensureReady();
@@ -973,12 +1321,18 @@ var PoolWorker = class {
 		const payload = Buffer.allocUnsafe(5);
 		payload[0] = modelByte;
 		payload.writeUInt32LE(frameId, 1);
-		return this.dispatch(MSG_INFER_CACHED, payload, deviceId);
+		return this.dispatch(MSG_INFER_CACHED, payload, { deviceId });
 	}
 	async sendCommand(cmd) {
 		this.ensureReady();
+		const command = describeCommand(cmd);
 		const payload = Buffer.from(JSON.stringify(cmd), "utf8");
-		return await this.dispatch(MSG_COMMAND, payload);
+		const sentAt = Date.now();
+		if (MODEL_LOAD_COMMANDS.has(command.cmd)) this.warnIfLoadQueued(command);
+		const raw = await this.dispatch(MSG_COMMAND, payload, { command });
+		if (MODEL_LOAD_COMMANDS.has(command.cmd)) this.inferStall.noteResult(Date.now());
+		this.inspectCommandReply(raw, command, sentAt);
+		return raw;
 	}
 	/** Command whose reply shape is NOT the load/unload/replace/status envelope
 	*  (e.g. `mem_stats`) — returns the raw JSON record, so no cast is needed
@@ -986,13 +1340,15 @@ var PoolWorker = class {
 	async sendRawCommand(cmd) {
 		this.ensureReady();
 		const payload = Buffer.from(JSON.stringify(cmd), "utf8");
-		return this.dispatch(MSG_COMMAND, payload);
+		return this.dispatch(MSG_COMMAND, payload, { command: describeCommand(cmd) });
 	}
 	async dispose() {
 		const proc = this.process;
 		if (!proc) return;
 		this.process = null;
 		this.ready = false;
+		if (this.recycleBackstop) clearTimeout(this.recycleBackstop);
+		if (this.abandonedCompile) clearTimeout(this.abandonedCompile.hardTimer);
 		await terminateChild(proc, POOL_WORKER_TERM_GRACE_MS);
 		this.rejectAll(/* @__PURE__ */ new Error(`PoolWorker[${this.opts.workerLabel}]: pool disposed while the request was in flight`));
 	}
@@ -1035,8 +1391,11 @@ var PoolWorker = class {
 	}
 	/** Live inference gets seconds; commands and model loads keep the long
 	*  timeout — see {@link POOL_LIVE_INFER_TIMEOUT_MS}. */
-	deadlineFor(msgType) {
-		return SHEDDABLE_MSG_TYPES.has(msgType) ? POOL_LIVE_INFER_TIMEOUT_MS : POOL_INFER_TIMEOUT_MS;
+	deadlineFor(msgType, opts = {}) {
+		if (opts.timeoutMs !== void 0) return opts.timeoutMs;
+		if (SHEDDABLE_MSG_TYPES.has(msgType)) return POOL_LIVE_INFER_TIMEOUT_MS;
+		if (opts.command !== void 0 && MODEL_LOAD_COMMANDS.has(opts.command.cmd)) return POOL_MODEL_LOAD_BOUNDS[this.opts.poolRuntime].softMs + POOL_MODEL_LOAD_REPLY_MARGIN_MS;
+		return POOL_INFER_TIMEOUT_MS;
 	}
 	/**
 	* The request's ABSOLUTE deadline for the wire (D350), or `null` when this
@@ -1053,16 +1412,18 @@ var PoolWorker = class {
 		stamp.writeBigUInt64LE(BigInt(Date.now() + this.deadlineFor(msgType)), 0);
 		return stamp;
 	}
-	dispatch(msgType, payload, deviceId) {
-		const shed = this.shedIfSaturated(msgType, deviceId);
+	dispatch(msgType, payload, opts = {}) {
+		const shed = this.shedIfSaturated(msgType, opts.deviceId);
 		if (shed) return Promise.resolve(shed);
 		const reqId = this.allocRequestId();
 		return new Promise((resolve, reject) => {
-			const timer = this.armRequestTimeout(reqId, msgType, resolve, reject, deviceId);
+			const timer = this.armRequestTimeout(reqId, msgType, resolve, reject, opts);
 			this.pending.set(reqId, {
 				resolve,
 				reject,
-				timer
+				timer,
+				...opts.command !== void 0 ? { command: opts.command } : {},
+				...SHEDDABLE_MSG_TYPES.has(msgType) ? { live: true } : {}
 			});
 			try {
 				const stamp = this.deadlineStamp(msgType);
@@ -1093,8 +1454,9 @@ var PoolWorker = class {
 	* - Anything else (commands, model loads) still REJECTS with the error the
 	*   dashboards grep for — a lost command is a fault, not flow control.
 	*/
-	armRequestTimeout(reqId, msgType, resolve, reject, deviceId) {
-		const timeoutMs = this.deadlineFor(msgType);
+	armRequestTimeout(reqId, msgType, resolve, reject, opts = {}) {
+		const { deviceId, command } = opts;
+		const timeoutMs = this.deadlineFor(msgType, opts);
 		const timer = setTimeout(() => {
 			if (this.pending.delete(reqId)) {
 				if (SHEDDABLE_MSG_TYPES.has(msgType)) {
@@ -1118,6 +1480,7 @@ var PoolWorker = class {
 						dropped: true,
 						shedReason: "deadline-expired"
 					});
+					this.noteInferUnanswered();
 					return;
 				}
 				this.timedOutCount++;
@@ -1130,10 +1493,20 @@ var PoolWorker = class {
 						device: this.opts.device ?? "default",
 						inFlight: this.pending.size,
 						reqId,
-						timeoutMs
+						timeoutMs,
+						...command !== void 0 ? {
+							command: command.cmd,
+							modelIndex: command.modelIndex,
+							model: command.model
+						} : {}
 					}
 				});
-				reject(/* @__PURE__ */ new Error(`PoolWorker[${this.opts.workerLabel}]: inference request ${reqId} timed out after ${timeoutMs}ms on ${this.opts.poolRuntime}:${this.opts.device ?? "default"} (worker alive, no reply)`));
+				const message = `PoolWorker[${this.opts.workerLabel}]: inference request ${reqId} timed out after ${timeoutMs}ms on ${this.opts.poolRuntime}:${this.opts.device ?? "default"} (worker alive, no reply)`;
+				if (command !== void 0 && MODEL_LOAD_COMMANDS.has(command.cmd)) {
+					this.loadDeadlineMissed = true;
+					reject(new PoolModelLoadError(`compile-timeout: ${message}`, "compile-timeout", command.model));
+				} else reject(new Error(message));
+				if (opts.probe !== true && command !== void 0) this.probeLiveness(command);
 			}
 		}, timeoutMs);
 		timer.unref?.();
@@ -1144,11 +1517,12 @@ var PoolWorker = class {
 		if (shed) return Promise.resolve(shed);
 		const reqId = this.allocRequestId();
 		return new Promise((resolve, reject) => {
-			const timer = this.armRequestTimeout(reqId, msgType, resolve, reject, deviceId);
+			const timer = this.armRequestTimeout(reqId, msgType, resolve, reject, { deviceId });
 			this.pending.set(reqId, {
 				resolve,
 				reject,
-				timer
+				timer,
+				...SHEDDABLE_MSG_TYPES.has(msgType) ? { live: true } : {}
 			});
 			try {
 				if (!this.process?.stdin) throw new Error("PoolWorker: not initialized");
@@ -1170,10 +1544,10 @@ var PoolWorker = class {
 	}
 	allocRequestId() {
 		let id = this.nextRequestId;
-		this.nextRequestId = id >= 4294967295 ? 1 : id + 1;
+		this.nextRequestId = id >= WORKER_EVENT_REQ_ID - 1 ? 1 : id + 1;
 		while (this.pending.has(id)) {
 			id = this.nextRequestId;
-			this.nextRequestId = id >= 4294967295 ? 1 : id + 1;
+			this.nextRequestId = id >= WORKER_EVENT_REQ_ID - 1 ? 1 : id + 1;
 		}
 		return id;
 	}
@@ -1188,6 +1562,8 @@ var PoolWorker = class {
 		this.process.stdin.write(payload);
 	}
 	ensureReady() {
+		const poisonedBy = this.getPoisonDescription();
+		if (poisonedBy !== null) throw new Error(`PoolWorker[${this.opts.workerLabel}]: poisoned (${poisonedBy}) — recycling, not taking work`);
 		if (!this.ready || !this.process?.stdin) throw new Error(`PoolWorker[${this.opts.workerLabel}]: not initialized`);
 	}
 	/** Time spent in `Buffer.concat` since the last report. */
@@ -1227,23 +1603,273 @@ var PoolWorker = class {
 			const reqId = this.receiveBuffer.readUInt32LE(4);
 			const jsonBytes = this.receiveBuffer.subarray(8, 4 + totalLen);
 			this.receiveBuffer = this.receiveBuffer.subarray(4 + totalLen);
+			if (reqId === WORKER_EVENT_REQ_ID) {
+				this.handleWorkerEvent(jsonBytes);
+				continue;
+			}
 			const entry = this.pending.get(reqId);
 			if (!entry) {
 				this.log.warn("Response for unknown request id", { meta: {
 					worker: this.opts.workerLabel,
 					reqId
 				} });
+				this.noteLateReply(jsonBytes);
 				continue;
 			}
 			this.pending.delete(reqId);
 			if (entry.timer) clearTimeout(entry.timer);
 			try {
 				const parsed = JSON.parse(jsonBytes.toString("utf8"));
+				if (entry.live === true) if (parsed["dropped"] === true) this.noteInferUnanswered();
+				else this.inferStall.noteResult(Date.now());
 				entry.resolve(parsed);
 			} catch (err) {
 				entry.reject(err instanceof Error ? err : new Error(String(err)));
 			}
 		}
+		this.noteAnyReply();
+		if (this.poisonVerdict !== null && this.pending.size === 0) this.recycle();
+	}
+	/**
+	* Something came back from the worker: its loop was alive just now, so the
+	* missed-deadline verdict is lifted.
+	*/
+	noteAnyReply() {
+		this.loadDeadlineMissed = false;
+	}
+	/**
+	* The invariant the load deadline depends on (D653 § 2): loads into one
+	* pool are serialised, so at most ONE load is outstanding per worker. The
+	* worker runs model commands in order and starts a load's soft bound only
+	* when it begins, while the host's deadline for it runs from the SEND. A
+	* load queued behind another could therefore see its host deadline fire
+	* before the worker's named answer. Nothing in the provider issues that; if
+	* anything ever does, this says so.
+	*/
+	warnIfLoadQueued(next) {
+		const ahead = [...this.pending.values()].map((p) => p.command).filter((c) => c !== void 0 && MODEL_LOAD_COMMANDS.has(c.cmd));
+		if (ahead.length === 0) return;
+		this.log.warn("model load queued behind another on one worker — its deadline may fire before the worker answers", { meta: {
+			worker: this.opts.workerLabel,
+			pid: this.getPid(),
+			runtime: this.opts.poolRuntime,
+			device: this.opts.device ?? "default",
+			model: next.model,
+			queuedBehind: ahead.map((c) => c.model)
+		} });
+	}
+	hasPendingLoad() {
+		for (const p of this.pending.values()) if (p.command !== void 0 && MODEL_LOAD_COMMANDS.has(p.command.cmd)) return true;
+		return false;
+	}
+	/**
+	* A load reply that says a compile outlived its SOFT bound (`compile-timeout`)
+	* or that an earlier one still has (`worker-poisoned`). The worker is not
+	* recycled for it (fix round 1): the compile keeps running so a slow but
+	* finite one writes its cache, the loaded models keep serving, and the worker
+	* starts no new load. Only a compile still running at the HARD bound costs
+	* the process.
+	*/
+	inspectCommandReply(raw, command, sentAt) {
+		const reason = raw["reason"];
+		if (reason !== "compile-timeout" && reason !== "worker-poisoned") return;
+		if (this.abandonedCompile !== null || this.poisonVerdict !== null || this.exited) return;
+		const model = raw["modelId"];
+		const abandoned = {
+			...command,
+			model: typeof model === "string" ? model : command.model
+		};
+		const bounds = POOL_MODEL_LOAD_BOUNDS[this.opts.poolRuntime];
+		const reported = raw["compileElapsedMs"];
+		const elapsedMs = typeof reported === "number" && Number.isFinite(reported) && reported >= 0 ? reported : Date.now() - sentAt;
+		const hardTimer = setTimeout(() => this.poison("compile-hung", abandoned, `the compile of ${abandoned.model ?? "unknown"} was still running at its hard bound (${bounds.hardMs}ms)`), Math.max(0, bounds.hardMs - elapsedMs));
+		hardTimer.unref?.();
+		this.abandonedCompile = {
+			command: abandoned,
+			since: Date.now(),
+			hardTimer
+		};
+		this.log.warn("model compile past its soft bound — left running so its cache can be written", { meta: {
+			worker: this.opts.workerLabel,
+			pid: this.getPid(),
+			runtime: this.opts.poolRuntime,
+			device: this.opts.device ?? "default",
+			model: abandoned.model,
+			modelIndex: abandoned.modelIndex,
+			softMs: bounds.softMs,
+			hardMs: bounds.hardMs,
+			loads: "refused until it returns"
+		} });
+	}
+	/** An unsolicited worker event (`compile-finished-late`). */
+	handleWorkerEvent(jsonBytes) {
+		let event;
+		try {
+			event = JSON.parse(jsonBytes.toString("utf8"));
+		} catch {
+			return;
+		}
+		if (event["event"] !== "compile-finished-late") return;
+		const abandoned = this.abandonedCompile;
+		if (abandoned !== null) clearTimeout(abandoned.hardTimer);
+		this.abandonedCompile = null;
+		this.inferStall.noteResult(Date.now());
+		const ok = event["ok"] === true;
+		const meta = {
+			worker: this.opts.workerLabel,
+			pid: this.getPid(),
+			runtime: this.opts.poolRuntime,
+			device: this.opts.device ?? "default",
+			model: typeof event["modelId"] === "string" ? event["modelId"] : null,
+			elapsedMs: typeof event["elapsedMs"] === "number" ? event["elapsedMs"] : null,
+			...typeof event["error"] === "string" ? { error: event["error"] } : {}
+		};
+		if (ok) this.log.info("slow model compile finished after its soft bound — cache written, loads re-enabled", { meta });
+		else this.log.warn("slow model compile failed after its soft bound — loads re-enabled", { meta });
+		this.opts.onCompileFinishedLate?.({
+			model: meta.model,
+			ok,
+			...typeof event["error"] === "string" ? { error: event["error"] } : {}
+		});
+	}
+	/**
+	* The GIL grace (D653 round 2): may a silent loop be a compile holding the
+	* GIL rather than a wedge? Only while a load is SENT AND UNANSWERED, and only
+	* on a runtime whose compile may hold the GIL (`POOL_MODEL_LOAD_BOUNDS`).
+	*
+	* NOT while a compile runs past its soft bound: the worker REPLIED
+	* `compile-timeout`, so its loop is proven alive, and an inference hang
+	* behind that compile — the 2026-09-26 incident exactly — must be seen on
+	* the normal rule, not ~600-900 s later at the hard bound.
+	*/
+	gilGraceActive() {
+		if (!POOL_MODEL_LOAD_BOUNDS[this.opts.poolRuntime].gilMayBeHeldDuringCompile) return false;
+		if (this.loadDeadlineMissed) return false;
+		return this.hasPendingLoad();
+	}
+	/**
+	* The reason a silent worker is recycled with: a freeze that began with a
+	* load missing its deadline is that compile's doing (`compile-hung`, charged
+	* to the model, not the device); anything else is the device's.
+	*/
+	silenceReason(fallback) {
+		return this.loadDeadlineMissed ? "compile-hung" : fallback;
+	}
+	/** A live request ended with no result: judge the executor (D653, item 4). */
+	noteInferUnanswered() {
+		const stall = this.inferStall.noteUnanswered(Date.now());
+		if (stall === null || this.gilGraceActive()) return;
+		this.poison(this.silenceReason("infer-unresponsive"), {
+			cmd: "infer",
+			modelIndex: null,
+			model: null
+		}, `${stall.unanswered} live requests in a row ended without a result, none for ${stall.silentMs}ms`);
+	}
+	/** A reply whose request was already abandoned: a real result still proves the executor runs. */
+	noteLateReply(jsonBytes) {
+		try {
+			const parsed = JSON.parse(jsonBytes.toString("utf8"));
+			if (parsed["dropped"] !== true && parsed["cmd"] === void 0) this.inferStall.noteResult(Date.now());
+		} catch {}
+	}
+	/**
+	* Ask a worker whose command just timed out whether its loop still answers.
+	* `mem_stats` is served on the loop, never behind a compile, so a worker
+	* that cannot answer it inside {@link POOL_LIVENESS_PROBE_TIMEOUT_MS} is not
+	* slow — it is wedged. That is the 2026-09-26 shape exactly: 123 of 123
+	* `mem_stats` lost while the process looked alive.
+	*/
+	probeLiveness(timedOut) {
+		if (this.probeInFlight || this.poisonVerdict !== null || this.exited) return;
+		this.probeInFlight = true;
+		this.dispatch(MSG_COMMAND, Buffer.from(JSON.stringify({ cmd: "mem_stats" }), "utf8"), {
+			command: {
+				cmd: "mem_stats",
+				modelIndex: null,
+				model: null
+			},
+			timeoutMs: POOL_LIVENESS_PROBE_TIMEOUT_MS,
+			probe: true
+		}).then(() => {
+			this.log.warn("pool command timed out but the worker answers its liveness probe — slow, not wedged", { meta: {
+				worker: this.opts.workerLabel,
+				pid: this.getPid(),
+				runtime: this.opts.poolRuntime,
+				device: this.opts.device ?? "default",
+				command: timedOut.cmd,
+				modelIndex: timedOut.modelIndex,
+				model: timedOut.model
+			} });
+		}, (err) => {
+			if (this.gilGraceActive()) {
+				this.log.warn("liveness probe unanswered while a model load is outstanding — not poisoning yet", { meta: {
+					worker: this.opts.workerLabel,
+					pid: this.getPid(),
+					runtime: this.opts.poolRuntime,
+					device: this.opts.device ?? "default",
+					command: timedOut.cmd,
+					model: timedOut.model
+				} });
+				return;
+			}
+			this.poison(this.silenceReason("unresponsive"), timedOut, `${timedOut.cmd} timed out, then the liveness probe failed: ${err instanceof Error ? err.message : String(err)}`);
+		}).finally(() => {
+			this.probeInFlight = false;
+		});
+	}
+	/**
+	* Declare this LIVE worker unusable, say so once at ERROR, stop taking work,
+	* and recycle it once it has drained.
+	*
+	* `ready = false` is what hands the pool back to the provider: its next
+	* dispatch finds the factory not ready and condemns it under the per-device
+	* restart budget (3 deaths in 10 min, then a terminal `failed` the balancer
+	* excludes) — the same path a crashed worker takes, except that a
+	* `compile-hung` death is not charged to the device (see PoolPoisonReason).
+	* An unresponsive worker is killed at once (it will answer nothing it
+	* holds); a compile-hung one keeps serving its loaded models until its
+	* in-flight requests are answered, bounded by the live deadline.
+	*/
+	poison(reason, command, detail) {
+		if (this.poisonVerdict !== null || this.exited || this.process === null) return;
+		this.poisonVerdict = {
+			reason,
+			command,
+			detail,
+			pid: this.getPid()
+		};
+		this.ready = false;
+		const inFlightCommands = [...this.pending.values()].map((p) => p.command).filter((c) => c !== void 0).map((c) => c.model !== null ? `${c.cmd}:${c.model}` : c.cmd);
+		this.log.error("pool worker POISONED — recycling it", { meta: {
+			worker: this.opts.workerLabel,
+			pid: this.getPid(),
+			runtime: this.opts.poolRuntime,
+			device: this.opts.device ?? "default",
+			reason,
+			command: command.cmd,
+			modelIndex: command.modelIndex,
+			model: command.model,
+			detail,
+			inFlight: this.pending.size,
+			inFlightCommands
+		} });
+		if (reason !== "compile-hung" || this.pending.size === 0) {
+			this.recycle();
+			return;
+		}
+		this.recycleBackstop = setTimeout(() => this.recycle(), POOL_LIVE_INFER_TIMEOUT_MS + POOL_POISON_DRAIN_SLACK_MS);
+		this.recycleBackstop.unref?.();
+	}
+	/**
+	* Kill a poisoned worker WITHOUT nulling `this.process`, so its `exit` is
+	* reported and rejects whatever it still held — a deliberate dispose would
+	* silence both, and this is not one.
+	*/
+	recycle() {
+		if (this.recycling || this.exited || this.process === null) return;
+		this.recycling = true;
+		if (this.recycleBackstop) clearTimeout(this.recycleBackstop);
+		terminateChild(this.process, POOL_WORKER_TERM_GRACE_MS);
 	}
 	rejectAll(err) {
 		const entries = [...this.pending.values()];
@@ -1290,8 +1916,10 @@ var SharedInferencePool = class {
 		this.tuning = options.tuning ?? null;
 		this.numWorkers = Math.max(1, options.numWorkers ?? 1);
 		this.device = options.device;
+		this.onCompileFinishedLate = options.onCompileFinishedLate;
 	}
 	device;
+	onCompileFinishedLate;
 	/** Pid of the first worker (for legacy callers). Use `getPids()` for all. */
 	/** Summed backlog and shed count across the pool's workers. A rising
 	*  `inFlight` with a rising `shed` is a worker falling behind; a rising
@@ -1330,7 +1958,8 @@ var SharedInferencePool = class {
 			tuning: this.tuning,
 			logger: this.log,
 			workerLabel: `w${i}`,
-			...this.device ? { device: this.device } : {}
+			...this.device ? { device: this.device } : {},
+			...this.onCompileFinishedLate ? { onCompileFinishedLate: this.onCompileFinishedLate } : {}
 		}));
 		const t0 = performance.now();
 		const results = await Promise.all(this.workers.map((w) => w.initialize(initialModels)));
@@ -1425,7 +2054,7 @@ var SharedInferencePool = class {
 			index,
 			config: serializeModelConfig(config)
 		})));
-		for (const resp of responses) if (resp.status !== "ok") throw new Error(`Failed to load model at index ${index}: ${resp.error ?? "unknown"}`);
+		for (const resp of responses) if (resp.status !== "ok") throw new PoolModelLoadError(`Failed to load model at index ${index}: ${describeCommandFailure(resp)}`, resp.reason ?? null, resp.modelId ?? null);
 		if (index >= this.nextFreeIndex) this.nextFreeIndex = index + 1;
 		return { loadMs: Math.max(...responses.map((r) => r.loadMs ?? 0)) };
 	}
@@ -1461,7 +2090,7 @@ var SharedInferencePool = class {
 			index,
 			config: serializeModelConfig(config)
 		})));
-		for (const resp of responses) if (resp.status !== "ok") throw new Error(`Failed to replace model at index ${index}: ${resp.error ?? "unknown"}`);
+		for (const resp of responses) if (resp.status !== "ok") throw new PoolModelLoadError(`Failed to replace model at index ${index}: ${describeCommandFailure(resp)}`, resp.reason ?? null, resp.modelId ?? null);
 		return { loadMs: Math.max(...responses.map((r) => r.loadMs ?? 0)) };
 	}
 	/**
@@ -1511,9 +2140,35 @@ var SharedInferencePool = class {
 	allocateIndex() {
 		return this.nextFreeIndex++;
 	}
+	/**
+	* Give back an index whose load FAILED, so the retry reuses it. Only the most
+	* recent allocation can be returned. Loads into one pool are serialised by
+	* the provider, so a failed load is normally the latest one; anything else
+	* is left allocated rather than risk handing out a live slot twice.
+	*/
+	releaseIndex(index) {
+		if (index === this.nextFreeIndex - 1) this.nextFreeIndex = index;
+	}
 	isReady() {
 		return this.workers.length > 0 && this.workers.every((w) => w.isReady());
 	}
+	/**
+	* Why a worker of this pool was declared unusable while alive (D653), or
+	* `null`. The provider charges the restart budget with THIS — or, for a
+	* `compile-hung` death, does not charge the device at all — so the
+	* `inference device FAILED` line names the cause instead of "pool worker is
+	* not ready".
+	*/
+	getDeathCause() {
+		for (const w of this.workers) {
+			const cause = w.getDeathCause();
+			if (cause !== null) return {
+				...cause,
+				message: `worker ${cause.message}`
+			};
+		}
+		return null;
+	}
 	async dispose() {
 		await Promise.all(this.workers.map((w) => w.dispose()));
 		this.workers.length = 0;
@@ -1578,6 +2233,23 @@ var SharedInferencePool = class {
 		return found;
 	}
 };
+/** A failed command's reply as one message: its reason class first, when it has one. */
+function describeCommandFailure(resp) {
+	const error = resp.error ?? "unknown";
+	return resp.reason !== void 0 ? `${resp.reason}: ${error}` : error;
+}
+/** Name a command for the lines that must say which one hung (D653). */
+function describeCommand(cmd) {
+	const name = typeof cmd["cmd"] === "string" ? cmd["cmd"] : "unknown";
+	const index = cmd["index"];
+	const config = cmd["config"];
+	const modelPath = typeof config === "object" && config !== null && "path" in config ? config.path : void 0;
+	return {
+		cmd: name,
+		modelIndex: typeof index === "number" ? index : null,
+		model: typeof modelPath === "string" && modelPath.length > 0 ? node_path.basename(modelPath, node_path.extname(modelPath)) : null
+	};
+}
 function serializeModelConfig(config) {
 	const result = {
 		path: config.path,
@@ -1649,7 +2321,8 @@ function collectFramePlaneSteps(steps, isDetailStep) {
 }
 /**
 * Immutably remove from a frame-plane dispatch tree every enabled DETAIL
-* subtree (at any depth below the top level) whose top step `isUnrunnable`.
+* subtree (at any depth below the top level) that `isUnrunnable` — judged
+* WHOLE at its top, because its children ride its call (D657).
 * Those steps can never load into the dispatch device's pool — leaving them
 * in the tree makes `ensureModelsForSteps` fail the whole call on a format
 * build that does not exist. They are NOT dropped work: each runs later as
@@ -1661,6 +2334,11 @@ function collectFramePlaneSteps(steps, isDetailStep) {
 */
 function pruneUnrunnableDetailSteps(steps, isDetailStep, isUnrunnable) {
 	const prunedAddonIds = [];
+	const removedAddonIds = [];
+	const collect = (step) => {
+		removedAddonIds.push(step.addonId);
+		for (const child of step.children ?? []) collect(child);
+	};
 	const walk = (nodes, topLevel) => {
 		const kept = [];
 		for (const step of nodes) {
@@ -1668,8 +2346,9 @@ function pruneUnrunnableDetailSteps(steps, isDetailStep, isUnrunnable) {
 				kept.push(step);
 				continue;
 			}
-			if (!topLevel && isDetailStep(step.addonId) && isUnrunnable(step.addonId)) {
+			if (!topLevel && isDetailStep(step.addonId) && isUnrunnable(step)) {
 				prunedAddonIds.push(step.addonId);
+				collect(step);
 				continue;
 			}
 			kept.push(step.children?.length ? {
@@ -1681,7 +2360,8 @@ function pruneUnrunnableDetailSteps(steps, isDetailStep, isUnrunnable) {
 	};
 	return {
 		steps: walk(steps, true),
-		prunedAddonIds
+		prunedAddonIds,
+		removedAddonIds
 	};
 }
 //#endregion
@@ -1865,6 +2545,28 @@ function poolDecodeSettingsEqual(a, b) {
 }
 //#endregion
 //#region src/detection-pipeline/engine/pipeline-model-manager.ts
+/**
+* A (step, model) load the pool rejected — carrying the pool's failure class
+* (`compile-timeout`, …) so the provider can count a timeout against THAT
+* model without parsing a message (D653).
+*/
+var StepVariantLoadError = class extends Error {
+	stepId;
+	modelId;
+	poolIndex;
+	reason;
+	/** The model's file stem as the pool names it (`camstack-yunet-2023mar`). */
+	poolModel;
+	constructor(stepId, modelId, poolIndex, cause) {
+		super(cause instanceof Error ? cause.message : String(cause));
+		this.name = "StepVariantLoadError";
+		this.stepId = stepId;
+		this.modelId = modelId;
+		this.poolIndex = poolIndex;
+		this.reason = cause instanceof PoolModelLoadError ? cause.reason : null;
+		this.poolModel = cause instanceof PoolModelLoadError ? cause.model : null;
+	}
+};
 var PipelineModelManager = class {
 	pool;
 	source;
@@ -1876,11 +2578,20 @@ var PipelineModelManager = class {
 	lruClock = 0;
 	log;
 	maxModelsPerStep;
+	deviceKey;
+	/**
+	* Loads issued and not yet answered, keyed `stepId::modelId`. A second
+	* caller for the same pair JOINS the pending load instead of issuing its own
+	* (D653): on 2026-09-26 43 identical YuNet loads queued behind one hung GPU
+	* compile, each allocating a pool index of its own.
+	*/
+	pendingLoads = /* @__PURE__ */ new Map();
 	constructor(pool, source, logger, options) {
 		this.pool = pool;
 		this.source = source;
 		this.log = logger;
 		this.maxModelsPerStep = options?.maxModelsPerStep ?? 4;
+		this.deviceKey = options?.deviceKey ?? null;
 	}
 	/**
 	* Apply a new pipeline configuration — driven by the runtime config
@@ -1923,16 +2634,26 @@ var PipelineModelManager = class {
 		for (const { entry, step } of diff.unchanged) await this.reconcileDecode(entry, step.settings);
 	}
 	/**
-	* Get an `IInferenceEngine` handle. Without `modelId`, returns the
-	* active model for the step (camera-runtime hot path). Pass
-	* `modelId` to grab a specific warm variant (benchmark path).
-	* Touches the LRU clock so the looked-up model survives eviction
-	* pressure from concurrent loads.
-	*/
-	getHandle(stepId, modelId) {
-		const entry = this.resolveOrThrow(stepId, modelId);
+	* The warm variant that runs `(stepId, modelId)` — its handle AND the model
+	* loaded at that pool index, which is the only honest answer to "which model
+	* produced this output" (D658).
+	*
+	* No active-variant fallback, deliberately. The step-only lookup this
+	* replaces (`getHandle(stepId)`, deleted) answered the step's ACTIVE
+	* variant — the first one loaded after boot — and a tree that paired that
+	* handle with the step's REQUESTED id stamped SigLIP2 on MobileCLIP vectors
+	* for twenty minutes on 2026-09-27. A pair that is not resident throws,
+	* naming both.
+	*/
+	getVariant(stepId, modelId) {
+		const entry = this.loaded.get(stepId)?.get(modelId);
+		if (entry === void 0) throw new Error(`Step "${stepId}" has no warm variant for model "${modelId}" in the inference pool — refusing rather than running another variant`);
 		this.touch(entry);
-		return this.pool.getHandle(entry.poolIndex);
+		return {
+			engine: this.pool.getHandle(entry.poolIndex),
+			modelId: entry.modelId,
+			poolIndex: entry.poolIndex
+		};
 	}
 	/** True iff the step has any model loaded. */
 	isLoaded(stepId) {
@@ -1963,13 +2684,14 @@ var PipelineModelManager = class {
 		return this.activeByStep.get(stepId);
 	}
 	/**
-	* Pool index for a given (stepId, modelId). Without `modelId`,
-	* returns the active model's index. Used by the inference fast
-	* paths that bypass `getHandle` and call `pool.inferBatch` directly.
+	* Pool index for exactly `(stepId, modelId)`, or `null` when that pair is not
+	* warm. Used by the inference fast paths that bypass `getVariant` and call
+	* `pool.inferBatch` directly. The model is REQUIRED: the active-variant
+	* fallback this had timed a benchmark on whichever variant was active (D658).
 	*/
 	getPoolIndex(stepId, modelId) {
-		const entry = this.resolve(stepId, modelId);
-		if (!entry) return null;
+		const entry = this.loaded.get(stepId)?.get(modelId);
+		if (entry === void 0) return null;
 		this.touch(entry);
 		return entry.poolIndex;
 	}
@@ -2030,6 +2752,23 @@ var PipelineModelManager = class {
 			await this.reconcileDecode(existing, settings);
 			return existing;
 		}
+		const key = `${stepId}::${modelId}`;
+		const pending = this.pendingLoads.get(key);
+		if (pending) {
+			const joined = await pending;
+			await this.reconcileDecode(joined, settings);
+			return joined;
+		}
+		const load = this.issueLoad(perStep, stepId, modelId, settings);
+		this.pendingLoads.set(key, load);
+		try {
+			return await load;
+		} finally {
+			this.pendingLoads.delete(key);
+		}
+	}
+	/** Evict if at capacity, then send ONE load command and record the result. */
+	async issueLoad(perStep, stepId, modelId, settings) {
 		while (perStep.size >= this.maxModelsPerStep) {
 			const evicted = this.pickEvictionTarget(stepId);
 			if (!evicted) break;
@@ -2041,16 +2780,31 @@ var PipelineModelManager = class {
 				cap: this.maxModelsPerStep
 			} });
 		}
-		const index = this.pool.allocateIndex();
 		const config = this.source.buildConfig(stepId, modelId, settings);
 		const decode = poolDecodeSettingsOf(config);
+		const index = this.pool.allocateIndex();
 		this.log.info("Loading step variant", { meta: {
 			step: stepId,
 			modelId,
 			poolIndex: index,
+			deviceKey: this.deviceKey,
 			...decode
 		} });
-		const { loadMs } = await this.pool.loadModel(index, config);
+		let loadMs;
+		try {
+			({loadMs} = await this.pool.loadModel(index, config));
+		} catch (err) {
+			this.pool.releaseIndex(index);
+			this.log.error("Step variant load failed", { meta: {
+				step: stepId,
+				modelId,
+				poolIndex: index,
+				deviceKey: this.deviceKey,
+				reason: err instanceof PoolModelLoadError ? err.reason : null,
+				error: err instanceof Error ? err.message : String(err)
+			} });
+			throw new StepVariantLoadError(stepId, modelId, index, err);
+		}
 		this.log.info("Step variant loaded", { meta: {
 			step: stepId,
 			modelId,
@@ -2157,18 +2911,6 @@ var PipelineModelManager = class {
 			await this.unloadEntry(evicted);
 		}
 	}
-	resolve(stepId, modelId) {
-		const perStep = this.loaded.get(stepId);
-		if (!perStep) return null;
-		const targetModelId = modelId ?? this.activeByStep.get(stepId);
-		if (!targetModelId) return null;
-		return perStep.get(targetModelId) ?? null;
-	}
-	resolveOrThrow(stepId, modelId) {
-		const entry = this.resolve(stepId, modelId);
-		if (!entry) throw new Error(`Step "${stepId}"${modelId ? ` (model "${modelId}")` : ""} is not loaded in the inference pool`);
-		return entry;
-	}
 	touch(entry) {
 		entry.lruTick = ++this.lruClock;
 	}
@@ -2364,16 +3106,15 @@ var EngineFactory = class {
 		await this.poolManager.applyConfig(steps);
 	}
 	/**
-	* Get an IInferenceEngine for a pipeline step. Without `modelId`,
-	* resolves to the active model (camera-runtime hot path); pass
-	* `modelId` to grab a specific warm variant (benchmark / override
-	* paths picking a non-runtime model).
+	* The resident variant for exactly `(stepId, modelId)`, with the model it
+	* runs — never the step's active variant in its place (D658). The executable
+	* tree is built from this, and a result's model stamp comes from it.
 	*
-	* @throws if step not loaded with the requested (or active) model.
+	* @throws if that exact pair is not warm.
 	*/
-	getEngine(stepId, modelId) {
+	getStepVariant(stepId, modelId) {
 		if (!this.poolManager) throw new Error("EngineFactory not initialized");
-		return this.poolManager.getHandle(stepId, modelId);
+		return this.poolManager.getVariant(stepId, modelId);
 	}
 	/** Check if a step is loaded (any variant). */
 	isLoaded(stepId) {
@@ -2420,6 +3161,15 @@ var EngineFactory = class {
 	isReady() {
 		return this.pool?.isReady() ?? false;
 	}
+	/**
+	* Why this factory's pool stopped being usable while its process was alive —
+	* a compile that never returned, or a worker that answered nothing (D653) —
+	* or `null`. A pool that merely crashed says nothing here; its `Worker
+	* process exited` line already names the signal.
+	*/
+	getDeathCause() {
+		return this.pool?.getDeathCause() ?? null;
+	}
 	/** Native pid of the underlying Python pool, if any. */
 	getPoolPid() {
 		return this.pool?.getPid() ?? null;
@@ -2468,7 +3218,7 @@ var EngineFactory = class {
 	async batchInferRaw(stepId, items, modelId, frameId = 0) {
 		if (!this.poolManager) throw new Error("EngineFactory.batchInferRaw: pool not initialized");
 		const idx = this.poolManager.getPoolIndex(stepId, modelId);
-		if (idx === null) throw new Error(`EngineFactory.batchInferRaw: step "${stepId}"${modelId ? ` (model "${modelId}")` : ""} is not loaded`);
+		if (idx === null) throw new Error(`EngineFactory.batchInferRaw: step "${stepId}" (model "${modelId}") is not loaded`);
 		return this.poolManager.getPool().inferBatch(idx, items, frameId);
 	}
 	async cacheFrameInPool(raw, width, height, format) {
@@ -2478,7 +3228,7 @@ var EngineFactory = class {
 	async inferCached(stepId, frameId, modelId) {
 		if (!this.poolManager) throw new Error("inferCached: pool not available");
 		const idx = this.poolManager.getPoolIndex(stepId, modelId);
-		if (idx === null) throw new Error(`inferCached: step "${stepId}"${modelId ? ` (model "${modelId}")` : ""} not loaded`);
+		if (idx === null) throw new Error(`inferCached: step "${stepId}" (model "${modelId}") not loaded`);
 		return this.poolManager.getPool().inferCached(idx, frameId);
 	}
 	async uncacheFrame(frameId) {
@@ -2546,12 +3296,13 @@ var EngineFactory = class {
 			concurrency,
 			tuning: resolvedTuning,
 			numWorkers,
-			...this.opts.engine.device ? { device: this.opts.engine.device } : {}
+			...this.opts.engine.device ? { device: this.opts.engine.device } : {},
+			...this.opts.onCompileFinishedLate ? { onCompileFinishedLate: this.opts.onCompileFinishedLate } : {}
 		});
 		this.poolManager = new PipelineModelManager(this.pool, {
 			buildConfig: (stepId, modelId, settings) => this.buildPoolModelConfig(stepId, modelId, poolRuntime, settings),
 			resolveDecode: (stepId, modelId, settings) => this.resolveDecode(stepId, modelId, settings)
-		}, this.log.child("model-mgr"));
+		}, this.log.child("model-mgr"), { deviceKey: this.deviceKey });
 		await this.pool.initialize([]);
 		await this.poolManager.applyConfig(steps);
 	}
@@ -2667,6 +3418,175 @@ function buildPoolModelConfigForStep(inputs) {
 	};
 }
 //#endregion
+//#region src/detection-pipeline/engine/model-load-governor.ts
+/**
+* What the dispatch path may ask of a pool's model loads, and what it must say
+* when it may not (D653, fix round 1).
+*
+* Three pieces of state, consulted ON DEMAND by the dispatch that needs a
+* model — there is no timer anywhere in here:
+*
+*  1. A NEGATIVE CACHE per (pool, model). `needsPoolUpdate` stays true after a
+*     failed load, so without it every frame of every camera re-issued the
+*     failing load and wrote two ERRORs. After a failure the model is not
+*     re-issued on that pool for an interval that doubles per consecutive
+*     failure up to a cap; a success clears it.
+*  2. COMPILE TIMEOUTS per (device, model). A load that outlived its soft
+*     bound is not a crash of the device. The SAME model timing out
+*     {@link COMPILE_TIMEOUTS_BEFORE_REFUSAL} times on one device — which,
+*     since a slow compile now runs on and writes its cache, only a compile
+*     that never finishes can do — refuses THAT MODEL there, by name. The
+*     device keeps serving every other model.
+*  3. Which camera has already been TOLD about the current state of a model,
+*     so each camera gets one line per state change, never one per frame.
+*/
+/** The first back-off after a failed load. */
+var MODEL_LOAD_BACKOFF_INITIAL_MS = 5e3;
+/** The back-off doubles per consecutive failure up to this. */
+var MODEL_LOAD_BACKOFF_MAX_MS = 5 * 6e4;
+/** Timeouts older than this no longer count toward a refusal. */
+var COMPILE_TIMEOUT_WINDOW_MS = 60 * 6e4;
+var ModelLoadGovernor = class {
+	/** Keyed by the pool OBJECT, so a respawned pool starts clean. */
+	backoff = /* @__PURE__ */ new WeakMap();
+	timeouts = /* @__PURE__ */ new Map();
+	reported = /* @__PURE__ */ new Map();
+	generation = 0;
+	/** May this dispatch load `models` on `pool` (a device `deviceKey`) now? */
+	check(pool, deviceKey, models, now = Date.now()) {
+		for (const model of models) {
+			const t = this.timeouts.get(timeoutKey(deviceKey, model));
+			if (t?.refused === true) return {
+				kind: "refused",
+				model,
+				timeouts: t.at.length,
+				error: t.lastError
+			};
+		}
+		const perPool = this.backoff.get(pool);
+		if (perPool === void 0) return { kind: "go" };
+		for (const model of models) {
+			const b = perPool.get(model);
+			if (b !== void 0 && now < b.retryAtMs) return {
+				kind: "backoff",
+				model,
+				retryInMs: b.retryAtMs - now,
+				failures: b.failures,
+				error: b.error,
+				generation: b.generation
+			};
+		}
+		return { kind: "go" };
+	}
+	/** One load of `model` on `pool` failed. */
+	recordFailure(pool, deviceKey, model, error, compileTimeout, now = Date.now(), poolModel = null, reason = null) {
+		let perPool = this.backoff.get(pool);
+		if (perPool === void 0) {
+			perPool = /* @__PURE__ */ new Map();
+			this.backoff.set(pool, perPool);
+		}
+		const previousEntry = perPool.get(model);
+		const failures = (previousEntry?.failures ?? 0) + 1;
+		const retryInMs = Math.min(MODEL_LOAD_BACKOFF_MAX_MS, MODEL_LOAD_BACKOFF_INITIAL_MS * 2 ** (failures - 1));
+		this.generation += 1;
+		const generation = this.generation;
+		perPool.set(model, {
+			failures,
+			retryAtMs: now + retryInMs,
+			error,
+			generation,
+			poolModel: poolModel ?? previousEntry?.poolModel ?? null,
+			reason
+		});
+		if (!compileTimeout) return {
+			generation,
+			retryInMs,
+			refusedNow: false,
+			compileTimeouts: 0
+		};
+		const key = timeoutKey(deviceKey, model);
+		const previous = this.timeouts.get(key);
+		const at = [...(previous?.at ?? []).filter((t) => t >= now - COMPILE_TIMEOUT_WINDOW_MS), now];
+		const refused = at.length >= 2;
+		this.timeouts.set(key, {
+			at,
+			refused,
+			lastError: error
+		});
+		return {
+			generation,
+			retryInMs,
+			refusedNow: refused && previous?.refused !== true,
+			compileTimeouts: at.length
+		};
+	}
+	/**
+	* A compile that outlived its bound came back on `pool` (D653 round 3).
+	*
+	* `ok`: THAT model's cache is written and the worker loads again, so its
+	* back-off is lifted — and only its: another model's failure on the same
+	* pool is not the compile that just finished. Not ok: it is one more
+	* failure of that model, and its back-off grows. Refusals by name are never
+	* lifted here; a model refused for timing out twice stays refused until the
+	* operator re-arms. Returns the governor keys it touched.
+	*/
+	settleLateCompile(pool, deviceKey, poolModel, ok, error, now = Date.now()) {
+		const perPool = this.backoff.get(pool);
+		if (perPool === void 0) return [];
+		const refusedMeanwhile = [...perPool].filter(([, b]) => b.reason === "worker-poisoned" && b.poolModel !== poolModel).map(([key]) => key);
+		for (const key of refusedMeanwhile) perPool.delete(key);
+		const own = poolModel === null ? [] : [...perPool].filter(([, b]) => b.poolModel === poolModel).map(([key]) => key);
+		for (const key of own) if (ok) perPool.delete(key);
+		else this.recordFailure(pool, deviceKey, key, error, false, now, poolModel);
+		return [...refusedMeanwhile, ...own];
+	}
+	/** `models` loaded on `pool`: forget their failures there. */
+	recordSuccess(pool, deviceKey, models) {
+		const perPool = this.backoff.get(pool);
+		for (const model of models) {
+			perPool?.delete(model);
+			this.timeouts.delete(timeoutKey(deviceKey, model));
+		}
+	}
+	/**
+	* Has `camera` been told about this `state` of `model` on `deviceKey` yet?
+	* Returns `true` exactly once per state change.
+	*/
+	shouldReport(camera, deviceKey, model, state) {
+		const key = `${camera ?? "-"}|${deviceKey}|${model}`;
+		if (this.reported.get(key) === state) return false;
+		this.reported.set(key, state);
+		return true;
+	}
+	/** Every model refused on some device. */
+	refused() {
+		const out = [];
+		for (const [key, t] of this.timeouts) {
+			if (!t.refused) continue;
+			const [deviceKey = "", model = ""] = key.split("\0");
+			out.push({
+				deviceKey,
+				model,
+				timeouts: t.at.length,
+				lastError: t.lastError
+			});
+		}
+		return out;
+	}
+	/** Operator re-arm: forget the refusals on `deviceKey`. Returns how many. */
+	rearm(deviceKey) {
+		let cleared = 0;
+		for (const key of [...this.timeouts.keys()]) if (key.startsWith(`${deviceKey}\u0000`)) {
+			this.timeouts.delete(key);
+			cleared += 1;
+		}
+		return cleared;
+	}
+};
+function timeoutKey(deviceKey, model) {
+	return `${deviceKey}\u0000${model}`;
+}
+//#endregion
 //#region src/detection-pipeline/engine/idle-pool-reaper.ts
 var IdlePoolReaper = class {
 	lastUsed = /* @__PURE__ */ new Map();
@@ -3203,13 +4123,16 @@ function checkReplayPin(input) {
 }
 var PRIMARY_ROOT_STEP = "object-detection";
 /**
-* The root model a frame result reports: `object-detection` when it runs (it
-* is the detector the tracks come from), else the first enabled video root.
-* `undefined` when no root runs — never a fabricated id.
+* The same rule over an EXECUTABLE tree's roots (D658): their model ids are the
+* pool variants that will run — after resolution, per-step loads and drops —
+* so this is what a live `runPipeline` stamps. The tree holds only enabled
+* video roots already.
 */
-function resolveRootModelId(roots) {
-	const enabled = roots.filter((s) => s.enabled && s.slot !== "audio-classifier");
-	return (enabled.find((s) => s.addonId === PRIMARY_ROOT_STEP) ?? enabled[0])?.modelId;
+function resolveTreeRootModelId(roots) {
+	return primaryRootModelId(roots);
+}
+function primaryRootModelId(roots) {
+	return (roots.find((s) => s.stepId === PRIMARY_ROOT_STEP) ?? roots[0])?.modelId;
 }
 //#endregion
 //#region src/detection-pipeline/engine-provisioner.ts
@@ -6002,26 +6925,47 @@ function resolveEffectiveClassMap(definition, modelId, resolveModel) {
 * Build an executable tree from user config.
 *
 * @param steps - User-configured pipeline steps (from PipelineDefaultStep[])
-* @param getEngine - Function that returns an IInferenceEngine for a step ID.
-*                    Throws if step not loaded.
+* @param getEngine - Returns the engine that runs exactly `(stepId, modelId)`
+*                    and the model it runs. Throws if that pair is not loaded.
 */
-function buildExecutableTree(steps, getEngine, resolveModel) {
-	return { roots: steps.filter((s) => s.enabled).filter((s) => s.slot !== "audio-classifier").map((s) => buildNode(s, getEngine, resolveModel)) };
+function buildExecutableTree(steps, getEngine, resolveModel, onMissingVariant) {
+	return { roots: steps.filter((s) => s.enabled).filter((s) => s.slot !== "audio-classifier").map((s) => buildNode(s, getEngine, resolveModel, onMissingVariant)) };
+}
+function subtreeIds(step) {
+	return [step.addonId, ...(step.children ?? []).filter((c) => c.enabled).flatMap((c) => subtreeIds(c))];
 }
-function buildNode(step, getEngine, resolveModel) {
+function buildChild(child, parentStepId, getEngine, resolveModel, onMissingVariant) {
+	if (onMissingVariant === void 0) return buildNode(child, getEngine, resolveModel, onMissingVariant);
+	let variant;
+	try {
+		variant = getEngine(child.addonId, child.modelId);
+	} catch (err) {
+		onMissingVariant({
+			step: child,
+			stepId: child.addonId,
+			modelId: child.modelId,
+			parentStepId,
+			droppedSubtree: subtreeIds(child),
+			error: err instanceof Error ? err.message : String(err)
+		});
+		return null;
+	}
+	return buildNode(child, getEngine, resolveModel, onMissingVariant, variant);
+}
+function buildNode(step, getEngine, resolveModel, onMissingVariant, resolved) {
 	const definition = require_default_detection_model.getStepDefinition(step.addonId);
-	const engine = getEngine(step.addonId);
-	const children = (step.children ?? []).filter((c) => c.enabled).map((c) => buildNode(c, getEngine, resolveModel));
+	const variant = resolved ?? getEngine(step.addonId, step.modelId);
+	const children = (step.children ?? []).filter((c) => c.enabled).map((c) => buildChild(c, step.addonId, getEngine, resolveModel, onMissingVariant)).filter((c) => c !== null);
 	const mergedSettings = {
 		...collectSchemaDefaults(step.addonId),
 		...step.settings
 	};
-	const effectiveClassMap = resolveEffectiveClassMap(definition, step.modelId, resolveModel);
+	const effectiveClassMap = resolveEffectiveClassMap(definition, variant.modelId, resolveModel);
 	return {
 		stepId: step.addonId,
 		definition,
-		engine,
-		modelId: step.modelId,
+		engine: variant.engine,
+		modelId: variant.modelId,
 		inputClasses: definition.inputClasses ?? [],
 		enabled: step.enabled,
 		children,
@@ -6051,43 +6995,6 @@ function walkFieldsForDefaults(fields, out) {
 	}
 }
 //#endregion
-//#region src/detection-pipeline/registry/custom-models.ts
-/**
-* Group a flat list of custom-model descriptors (as returned by the
-* `custom-model-registry` collection cap) into a `stepId → entries` map for
-* the picker / resolution union. Pure; order within a step preserved.
-*/
-function groupCustomModelsByStep(descriptors) {
-	const byStep = /* @__PURE__ */ new Map();
-	for (const d of descriptors) {
-		const arr = byStep.get(d.stepId) ?? [];
-		arr.push(d.entry);
-		byStep.set(d.stepId, arr);
-	}
-	return byStep;
-}
-/**
-* Union a step's static catalog models with operator-registered custom
-* models. On an `id` collision the static catalog entry wins — a custom
-* model can never shadow a built-in one.
-*
-* Pure + side-effect-free so it can be unit-tested in isolation and called
-* from the (free) `buildSchemaSlots` builder without any addon context.
-*/
-function mergeCustomModels(staticModels, customModels) {
-	const seen = new Set(staticModels.map((m) => m.id));
-	const merged = [...staticModels];
-	for (const m of customModels) {
-		if (seen.has(m.id)) continue;
-		seen.add(m.id);
-		merged.push({
-			...m,
-			provider: require_dist.inferModelProvider(m)
-		});
-	}
-	return merged;
-}
-//#endregion
 //#region src/detection-pipeline/cluster-model-resolution.ts
 /**
 * The cluster model rule, enforced on the node that has to obey it.
@@ -6192,8 +7099,10 @@ function collectSubstitutions(steps, format) {
 	return result;
 }
 /**
-* Return one {@link ZeroBuildIssue} per step in `steps` whose catalog entry
-* (via `getStepDef`) has ZERO models with a build for `format`. `steps` is
+* Return one {@link ZeroBuildIssue} per step in `steps` that a pool of `format`
+* cannot run: neither the model it names nor any NON-legacy model of its
+* catalog (via `getStepDef`) ships a build for `format`
+* (`stepModelRunsOnFormat`, D657). `steps` is
 * expected to already be flattened + filtered to the set the caller cares
 * about (e.g. enabled video steps) — this does not recurse into children or
 * filter by `enabled`, unlike {@link collectSubstitutions}.
@@ -6214,7 +7123,7 @@ function collectZeroBuildIssues(steps, format, getStepDef = require_default_dete
 		} catch {
 			continue;
 		}
-		if (!def.models.some((m) => m.formats[format])) result.push({
+		if (!require_dist.stepModelRunsOnFormat(def.models, step.modelId, format)) result.push({
 			addonId: step.addonId,
 			format
 		});
@@ -6264,11 +7173,20 @@ function collectUnknownAddonIssues(steps, getStepDef = require_default_detection
 * `diagnostics.substitutions` instead of happening silently. The caller
 * (the provider) owns logging + dedup for both.
 */
-function resolveInputSteps(steps, format, engine, clusterModels, clusterSettings) {
+function resolveInputSteps(steps, format, engine, clusterModels, clusterSettings, unknownClusterSteps) {
 	const resolvedSteps = [];
 	const unknownAddonIds = [];
 	const substitutions = [];
 	const clusterRefusals = [];
+	const clusterUnknown = [];
+	const recurse = (children) => {
+		const r = resolveInputSteps(children, format, engine, clusterModels, clusterSettings, unknownClusterSteps);
+		unknownAddonIds.push(...r.diagnostics.unknownAddonIds);
+		substitutions.push(...r.diagnostics.substitutions);
+		clusterRefusals.push(...r.diagnostics.clusterRefusals);
+		clusterUnknown.push(...r.diagnostics.clusterUnknown);
+		return r;
+	};
 	for (const s of steps) {
 		let def;
 		try {
@@ -6277,6 +7195,10 @@ function resolveInputSteps(steps, format, engine, clusterModels, clusterSettings
 			unknownAddonIds.push(s.addonId);
 			continue;
 		}
+		if (def.modelScope === "cluster" && unknownClusterSteps?.has(s.addonId) === true) {
+			clusterUnknown.push(s.addonId);
+			continue;
+		}
 		if (clusterModels !== void 0) {
 			const verdict = resolveClusterStepModel(s.addonId, clusterModels, format, () => def);
 			if (verdict.kind !== "node-scoped") {
@@ -6286,12 +7208,7 @@ function resolveInputSteps(steps, format, engine, clusterModels, clusterSettings
 					format: verdict.format,
 					formatsShipped: verdict.formatsShipped
 				});
-				const childResult = s.children ? resolveInputSteps(s.children, format, engine, clusterModels, clusterSettings) : null;
-				if (childResult) {
-					unknownAddonIds.push(...childResult.diagnostics.unknownAddonIds);
-					substitutions.push(...childResult.diagnostics.substitutions);
-					clusterRefusals.push(...childResult.diagnostics.clusterRefusals);
-				}
+				const childResult = s.children ? recurse(s.children) : null;
 				const settings = require_dist.overlayClusterStepSettings(s.addonId, s.settings, clusterSettings ?? {});
 				resolvedSteps.push({
 					addonId: s.addonId,
@@ -6315,12 +7232,7 @@ function resolveInputSteps(steps, format, engine, clusterModels, clusterSettings
 			running: runningModelId,
 			format
 		});
-		const childResult = s.children ? resolveInputSteps(s.children, format, engine, clusterModels, clusterSettings) : null;
-		if (childResult) {
-			unknownAddonIds.push(...childResult.diagnostics.unknownAddonIds);
-			substitutions.push(...childResult.diagnostics.substitutions);
-			clusterRefusals.push(...childResult.diagnostics.clusterRefusals);
-		}
+		const childResult = s.children ? recurse(s.children) : null;
 		const settings = require_dist.overlayClusterStepSettings(s.addonId, s.settings, clusterSettings ?? {});
 		resolvedSteps.push({
 			addonId: s.addonId,
@@ -6339,7 +7251,8 @@ function resolveInputSteps(steps, format, engine, clusterModels, clusterSettings
 		diagnostics: {
 			unknownAddonIds,
 			substitutions,
-			clusterRefusals
+			clusterRefusals,
+			clusterUnknown
 		}
 	};
 }
@@ -6563,6 +7476,90 @@ function parseWavToAudioChunk(filePath) {
 function enginesEqual(a, b) {
 	return a.runtime === b.runtime && a.backend === b.backend && a.format === b.format && (a.device ?? null) === (b.device ?? null);
 }
+/** A dispatch's load refused before it was issued: a back-off, or a refusal by name. */
+var ModelLoadRefusedError = class extends Error {
+	model;
+	state;
+	constructor(message, model, state) {
+		super(message);
+		this.name = "ModelLoadRefusedError";
+		this.model = model;
+		this.state = state;
+	}
+};
+/** `step/model` — how the governor and the log lines name a model. */
+function stepModelKey(step) {
+	return `${step.addonId}/${step.modelId}`;
+}
+/** The reporting state a refusal verdict stands for. */
+function stateOf(verdict) {
+	return verdict.kind === "refused" ? "refused" : `failed:${verdict.generation}`;
+}
+/** The load policy of a `runPipeline` call — see {@link DispatchLoadPolicy}. */
+function dispatchLoadPolicyOf(input) {
+	return input.deviceId !== void 0 && input.replay !== true ? "partial" : "all-or-nothing";
+}
+/**
+* An all-or-nothing call lost steps to failed loads (D657). Names every one,
+* so a benchmark never reports timings for a tree it did not run.
+*/
+var DispatchStepsNotLoadedError = class extends Error {
+	failedSteps;
+	constructor(failures) {
+		const named = failures.map((f) => `${f.step.addonId}/${f.step.modelId} (${f.error instanceof Error ? f.error.message : String(f.error)})`);
+		super(`pipeline steps failed to load — the call runs all of them or none: ${named.join("; ")}`, { cause: failures[0]?.error });
+		this.name = "DispatchStepsNotLoadedError";
+		this.failedSteps = failures.map((f) => `${f.step.addonId}/${f.step.modelId}`);
+	}
+};
+/**
+* A step's model artifact cannot be made present for the pool's format — no
+* build for that format, a custom model missing from this node, a download
+* that failed. Named by step and model so the failure is charged to THAT
+* model only (D657): a plain `Error` was charged to every model of the set,
+* and one face model with no tflite build backed the Coral's root detector
+* off for minutes.
+*/
+var StepModelArtifactError = class extends Error {
+	stepId;
+	modelId;
+	constructor(stepId, modelId, cause) {
+		super(cause instanceof Error ? cause.message : String(cause), { cause });
+		this.name = "StepModelArtifactError";
+		this.stepId = stepId;
+		this.modelId = modelId;
+	}
+};
+/** The one model a failed load is charged to, when the error names it. */
+function failedModelOf(err) {
+	if (err instanceof StepVariantLoadError || err instanceof StepModelArtifactError) return `${err.stepId}/${err.modelId}`;
+	return null;
+}
+/**
+* `steps` without the subtrees of the steps whose model did not load (D657).
+* A step's children consume its output, so they leave with it. Matched by
+* INSTANCE: two steps of one addon in a tree are two steps.
+*/
+function withoutFailedSteps(steps, failed) {
+	const kept = [];
+	for (const step of steps) {
+		if (failed.has(step)) continue;
+		kept.push(step.children && step.children.length > 0 ? {
+			...step,
+			children: withoutFailedSteps(step.children, failed)
+		} : step);
+	}
+	return kept;
+}
+/**
+* The reason a condemned pool is charged against its device's restart budget.
+* A pool recycled because it HUNG (D653) names the hang — `inference device
+* FAILED … reason: pool worker is not ready` could not tell a GPU compile that
+* never returned from a crash.
+*/
+function deathReasonOf(factory) {
+	return factory.getDeathCause()?.message ?? "pool worker is not ready";
+}
 /** Build a `RuntimeEnv` from the running process + probed hardware. */
 function runtimeEnvFromProcess(hardware) {
 	return {
@@ -6661,18 +7658,18 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* paths don't issue a cap round-trip on every call. Empty map = no provider
 	* (or a query failure) → behaviour identical to the static-catalog-only path.
 	*/
-	customModelsCache = null;
+	customModels = new require_default_detection_model.CustomModelCatalog({ warn: (message, extras) => this.log.warn(message, extras) });
 	/**
 	* SYNC custom-model lookup handed to every live {@link EngineFactory} so
 	* `buildPoolModelConfigForStep` can resolve an operator-registered model at
-	* pool-load time. Reads the {@link customModelsCache} snapshot — every load
+	* pool-load time. Reads the {@link customModels} snapshot — every load
 	* path (`ensureModelsForSteps`, `ensureModelsForCurrentSteps`) refreshes the
 	* cache via `resolveModelEntry` BEFORE `loadAdditional`/`applyConfig`, so
 	* the snapshot is fresh when the pool asks. Without this seam a selected
 	* custom model failed the WHOLE pool applyConfig with
 	* `Model "X" not found in step "Y" catalog`.
 	*/
-	customModelResolver = (stepId, modelId) => this.customModelsCache?.byStep.get(stepId)?.find((m) => m.id === modelId);
+	customModelResolver = (stepId, modelId) => this.customModels.current().get(stepId)?.find((m) => m.id === modelId);
 	/**
 	* Per-device {@link DeviceProxy} cache used for zone gating at the
 	* runtime path. Reads `state.zones.value` + `state.zoneRules.value`
@@ -6695,6 +7692,10 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		info: (message, extras) => this.log.info(message, extras),
 		warn: (message, extras) => this.log.warn(message, extras)
 	} });
+	/** One line per camera when a cluster-scoped step starts running another model (D658). */
+	clusterModelSwitchLog = new ClusterModelSwitchLog({ info: (message, extras) => this.log.info(message, extras) });
+	/** One line per (camera, step) while a cluster step is refused as unknown (D658). */
+	clusterUnknownLog = new ClusterUnknownRefusalLog({ warn: (message, extras) => this.log.warn(message, extras) });
 	/**
 	* Last logged "step configured" signature — used to dedupe the
 	* per-step debug trail so it only fires on actual config CHANGE
@@ -7161,7 +8162,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 			if (!steps) return;
 			const format = this.currentEngine.format;
 			const substitutionIssues = this.getActiveModelSubstitutions().map((s) => `${s.addonId}: chose ${s.chosen}, running ${s.running} (${s.format})`);
-			const zeroBuildIssues = collectZeroBuildIssues(flattenSteps(steps), format).map((i) => `${i.addonId}: no model has a ${i.format} build`);
+			const zeroBuildIssues = collectZeroBuildIssues(flattenSteps(steps), format, this.mergedStepDefOrThrow).map((i) => `${i.addonId}: no model has a ${i.format} build`);
 			this.configIssues = [...substitutionIssues, ...zeroBuildIssues];
 		} catch (err) {
 			this.configIssues = [];
@@ -7295,24 +8296,21 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* a warning — callers then behave exactly like the static-catalog path.
 	*/
 	async getCustomModels() {
-		const now = Date.now();
-		if (this.customModelsCache && now - this.customModelsCache.at < 5e3) return this.customModelsCache.byStep;
-		let byStep = /* @__PURE__ */ new Map();
-		try {
-			const api = this.addonCtx?.api;
-			if (api) {
-				if ((await api.addons.listCapabilityProviders.query({ capName: "custom-model-registry" })).some((p) => p.isActive)) byStep = groupCustomModelsByStep(await api.customModelRegistry.listModels.query());
-			}
-		} catch (err) {
-			this.log.warn("custom-model-registry query failed — using static catalog only", { meta: { error: require_dist.errMsg(err) } });
-		}
-		this.customModelsCache = {
-			at: now,
-			byStep
-		};
-		return byStep;
+		return this.customModels.refresh(this.addonCtx?.api);
 	}
 	/**
+	* A step's definition MERGED with its custom models, synchronously, from
+	* the last custom-model read (D657 round 2). Every placement gate here
+	* judges this catalog — the one the orchestrator's schema carries.
+	*/
+	mergedStepDef = (addonId) => this.customModels.stepDefinition(addonId);
+	/** {@link mergedStepDef} as the throwing lookup `collectZeroBuildIssues` takes. */
+	mergedStepDefOrThrow = (addonId) => {
+		const def = this.mergedStepDef(addonId);
+		if (def === null) throw new Error(`Unknown pipeline step: "${addonId}"`);
+		return def;
+	};
+	/**
 	* Resolve a model id within a step to a catalog entry — static catalog
 	* first, then the custom registry. Returns undefined if neither has it.
 	*/
@@ -7503,7 +8501,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 			const enabledSteps = this.pruneToEnabledSteps(steps);
 			const substitutions = collectSubstitutions(enabledSteps, format);
 			const unknownAddonIssues = collectUnknownAddonIssues(enabledSteps);
-			const zeroBuildIssues = collectZeroBuildIssues(this.flattenEnabledVideoStepInputs(steps), format);
+			const zeroBuildIssues = collectZeroBuildIssues(this.flattenEnabledVideoStepInputs(steps), format, this.mergedStepDefOrThrow);
 			const issues = [...unknownAddonIssues.map((u) => ({
 				addonId: u.addonId,
 				kind: "unknown-addon",
@@ -7820,6 +8818,12 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	}
 	async runPipeline(input, onProgress) {
 		await this.clusterModels.refresh(this.addonCtx?.api);
+		this.clusterUnknownLog.note(input.deviceId ?? 0, input.steps, this.clusterModels.unknownSteps());
+		if (dispatchLoadPolicyOf(input) === "all-or-nothing") {
+			const unknown = unknownStepsNamed(input.steps, this.clusterModels.unknownSteps());
+			if (unknown.length > 0) throw new Error(`cluster-model-unknown: the owner has not answered which model the cluster runs for ${unknown.join(", ")} — the call runs all of its steps or none`);
+		}
+		await this.getCustomModels();
 		const nodeId = this.addonCtx?.kernel?.localNodeId ?? "hub";
 		const sessionId = input.sessionId ?? `run-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
 		const isRuntime = Boolean(input.frame || input.frameRef);
@@ -7984,11 +8988,6 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		}
 		const enabledSteps = flattenSteps(benchmarkSteps);
 		emit(`Pipeline: ${enabledSteps.length} step(s) — ${enabledSteps.map((s) => s.addonId).join(" → ")}`);
-		const rootModelId = resolveRootModelId(benchmarkSteps);
-		const stampRoot = (r) => rootModelId === void 0 ? r : {
-			...r,
-			rootModelId
-		};
 		if (input.replay === true) {
 			const verdict = checkReplayPin({
 				requested: input.steps,
@@ -8030,21 +9029,51 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		const dispatchEngine = dispatchDeviceKey ? resolveDeviceEngine(dispatchDeviceKey) : runEngine;
 		const dispatchFactory = dispatchDeviceKey ? await this.resolveDeviceFactory(dispatchDeviceKey) : this.engineFactory;
 		const needed = enabledSteps.filter((s) => dispatchFactory.needsPoolUpdate(s));
+		let runSteps = benchmarkSteps;
+		const loadPolicy = dispatchLoadPolicyOf(input);
 		if (needed.length > 0) {
-			this.log.info("Benchmark: models to load", { meta: {
-				count: needed.length,
-				models: needed.map((s) => `${s.addonId}/${s.modelId}`)
+			const dispatchDeviceKeyLabel = deviceKeyOf(dispatchEngine);
+			const toIssue = needed.filter((s) => this.loadGovernor.check(dispatchFactory, dispatchDeviceKeyLabel, [stepModelKey(s)]).kind === "go");
+			if (toIssue.length > 0) this.log.info("Benchmark: models to load", { meta: {
+				count: toIssue.length,
+				models: toIssue.map((s) => `${s.addonId}/${s.modelId}`)
 			} });
-			for (const s of needed) emit(`Loading model: ${s.addonName} (${s.modelId})...`, {
+			for (const s of toIssue) emit(`Loading model: ${s.addonName} (${s.modelId})...`, {
 				step: s.addonId,
 				addonId: s.addonId,
 				modelId: s.modelId
 			});
-			await this.ensureModelsForSteps(benchmarkSteps, dispatchFactory, dispatchEngine.format);
-			emit(`All models loaded`);
+			const failures = await this.loadDispatchModels(benchmarkSteps, dispatchFactory, dispatchEngine.format, {
+				...input.deviceId !== void 0 ? { deviceId: input.deviceId } : {},
+				deviceKey: dispatchDeviceKeyLabel
+			}, loadPolicy);
+			runSteps = stepsSurvivingLoad(benchmarkSteps, failures, loadPolicy);
+			emit(failures.length === 0 ? `All models loaded` : `Models loaded (${failures.length} failed)`);
 		}
 		emit("Running inference...");
-		const tree = buildExecutableTree(benchmarkSteps, (stepId) => dispatchFactory.getEngine(stepId), this.customModelResolver);
+		const notWarm = [];
+		const tree = buildExecutableTree(runSteps, (stepId, modelId) => dispatchFactory.getStepVariant(stepId, modelId), this.customModelResolver, (drop) => notWarm.push({
+			step: drop.step,
+			error: new Error(drop.error)
+		}));
+		if (notWarm.length > 0) {
+			if (loadPolicy === "all-or-nothing") throw new DispatchStepsNotLoadedError(notWarm);
+			const dispatchDeviceKeyLabel = deviceKeyOf(dispatchEngine);
+			for (const failure of notWarm) {
+				const model = stepModelKey(failure.step);
+				this.reportDispatchLoadLoss({
+					...input.deviceId !== void 0 ? { deviceId: input.deviceId } : {},
+					deviceKey: dispatchDeviceKeyLabel,
+					outcome: "step-dropped"
+				}, dispatchDeviceKeyLabel, [model], model, "not-warm", require_dist.errMsg(failure.error));
+			}
+		}
+		const rootModelId = resolveTreeRootModelId(tree.roots);
+		const stampRoot = (r) => rootModelId === void 0 ? r : {
+			...r,
+			rootModelId
+		};
+		if (input.replay !== true) this.clusterModelSwitchLog.note(input.deviceId ?? 0, tree);
 		setupMs = performance.now() - wallT0 - decodeMs;
 		const effectiveDeviceId = input.deviceId ?? 0;
 		const deviceOverrides = effectiveDeviceId > 0 ? await this.deviceOverrides.resolve(effectiveDeviceId) : {};
@@ -8150,7 +9179,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		backend: this.currentEngine.backend,
 		device: this.currentEngine.device ?? null
 	}, clusterRefusalsOut) {
-		const { steps: resolvedSteps, diagnostics } = resolveInputSteps(steps, format, engine, this.clusterModels.current(), this.clusterModels.currentSettings());
+		const { steps: resolvedSteps, diagnostics } = resolveInputSteps(steps, format, engine, this.clusterModels.current(), this.clusterModels.currentSettings(), this.clusterModels.unknownSteps());
 		if (clusterRefusalsOut) clusterRefusalsOut.push(...diagnostics.clusterRefusals);
 		for (const addonId of diagnostics.unknownAddonIds) this.logLiveDispatchIssueOnce(`unknown:${addonId}`, () => this.log.warn("Live pipeline step references an unknown addon — dropping step", { meta: { addonId } }));
 		for (const sub of diagnostics.substitutions) this.logLiveDispatchIssueOnce(`${sub.addonId}|${sub.chosen}|${sub.running}|${sub.format}`, () => this.log.info("Live pipeline step model substituted for engine format (node-local)", { meta: {
@@ -8263,7 +9292,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		const deviceRefusals = [];
 		const steps = this.inputStepsToPipelineSteps(args.steps, deviceFormat, deviceStepEngine, deviceRefusals);
 		const framePlane = args.plane === "frame";
-		const zeroBuild = collectZeroBuildIssues(framePlane ? collectFramePlaneSteps(steps, isDetailPlaneStep) : flattenSteps(steps), deviceFormat);
+		const zeroBuild = collectZeroBuildIssues(framePlane ? collectFramePlaneSteps(steps, isDetailPlaneStep) : flattenSteps(steps), deviceFormat, this.mergedStepDefOrThrow);
 		if (zeroBuild.length === 0) {
 			if (!framePlane) {
 				this.logClusterRefusals(deviceRefusals, deviceId, `${deviceKey} (${deviceFormat})`);
@@ -8272,7 +9301,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 					steps
 				};
 			}
-			const pruneResult = pruneUnrunnableDetailSteps(steps, isDetailPlaneStep, (addonId) => stepHasZeroBuildsFor(addonId, deviceFormat));
+			const pruneResult = pruneUnrunnableDetailSteps(steps, isDetailPlaneStep, (subtree) => !require_default_detection_model.buildSubtreeCanRun(subtree, this.mergedStepDef)(deviceFormat));
 			if (pruneResult.prunedAddonIds.length > 0) {
 				const prunedList = pruneResult.prunedAddonIds.join(",");
 				this.logLiveDispatchIssueOnce(`detail-defer:${deviceKey}:${deviceFormat}:${prunedList}`, () => this.log.info("Detail steps have no model build for the dispatch device format — deferred to the detail-plane device jump", {
@@ -8284,7 +9313,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 					}
 				}));
 			}
-			this.logClusterRefusals(deviceRefusals.filter((r) => !pruneResult.prunedAddonIds.includes(r.stepId)), deviceId, `${deviceKey} (${deviceFormat})`);
+			this.logClusterRefusals(deviceRefusals.filter((r) => !pruneResult.removedAddonIds.includes(r.stepId)), deviceId, `${deviceKey} (${deviceFormat})`);
 			return {
 				deviceKey,
 				steps: pruneResult.steps
@@ -8318,7 +9347,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* microseconds instead of a 3-retry download backoff).
 	*/
 	assertStepsRunnable(steps, format, deviceId, enginesTried) {
-		const zeroBuild = collectZeroBuildIssues(flattenSteps(steps), format);
+		const zeroBuild = collectZeroBuildIssues(flattenSteps(steps), format, this.mergedStepDefOrThrow);
 		if (zeroBuild.length === 0) return;
 		const detail = zeroBuild.map((issue) => {
 			const step = flattenSteps(steps).find((s) => s.addonId === issue.addonId);
@@ -8337,53 +9366,235 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		throw new Error(message);
 	}
 	/**
-	* Single-flight gate around `ensureModelsForSteps`. Concurrent
-	* callers (every camera that fires motion in the same window calls
-	* `runPipeline` → `ensureModelsForSteps`) share the same in-flight
-	* promise instead of each racing into `loadAdditional`. Without this
-	* the pool gets the SAME model loaded N times under N distinct
-	* pool indices because every caller saw `isLoadedWithModel === false`
-	* before any of them committed via `stepToLoaded.set`.
-	*/
-	modelLoadInFlight = null;
+	* In-flight model loads, per POOL and per MODEL SET (D653). Two dispatches
+	* needing the same models on the same pool share ONE load and its outcome;
+	* a dispatch needing a different set is never handed another set's failure
+	* — it queues behind the pool's current load ({@link modelLoadTail}) and
+	* then decides for itself. Keyed by pool, not node-wide: one GPU compile
+	* that never returned held EVERY load on the node — the NPU's and the CPU's
+	* too — behind it.
+	*/
+	modelLoadInFlight = /* @__PURE__ */ new Map();
+	/** The last load issued on each pool — loads into one pool run one at a time. */
+	modelLoadTail = /* @__PURE__ */ new Map();
+	/** Stands in for "the node-default pool" before it exists. */
+	nullPoolKey = {};
+	/** Negative cache, per-model compile-timeout refusals, per-camera reporting (D653). */
+	loadGovernor = new ModelLoadGovernor();
+	/**
+	* Load a dispatch's models ONE STEP AT A TIME (D657), and answer which steps
+	* did not load. Never throws: each step is its own load, joined, backed off
+	* and charged on its own model, so a step whose model cannot load on this
+	* pool — no build for its format, a compile that failed — costs that step,
+	* never the root or a sibling. The caller decides what the dispatch runs
+	* without it ({@link stepsSurvivingLoad}).
+	*/
+	async loadDispatchModels(steps, factory, format, context, policy) {
+		const failures = [];
+		const walk = async (nodes, root) => {
+			for (const step of nodes) {
+				if (!step.enabled || step.slot === "audio-classifier") continue;
+				if (factory === null || factory.needsPoolUpdate(step)) try {
+					await this.ensureModelsForSteps([{
+						...step,
+						children: []
+					}], factory, format, {
+						...context,
+						outcome: policy === "all-or-nothing" ? "call-failed" : root ? "root-lost" : "step-dropped"
+					});
+				} catch (err) {
+					failures.push({
+						step,
+						error: err
+					});
+					continue;
+				}
+				await walk(step.children ?? [], false);
+			}
+		};
+		await walk(steps, true);
+		return failures;
+	}
 	/** Ensure all models needed by steps are downloaded and loaded in the engine pool.
 	* `factory`/`format` default to the node's engine; a per-device dispatch passes
-	* that device's factory + format (Phase 2 multi-device). */
-	async ensureModelsForSteps(steps, factory = this.engineFactory, format = this.currentEngine?.format ?? "onnx") {
-		while (this.modelLoadInFlight) await this.modelLoadInFlight;
-		const allEnabled = flattenSteps(steps);
+	* that device's factory + format (Phase 2 multi-device). `context` names the
+	* camera and the accelerator on the line a rejected load writes. */
+	async ensureModelsForSteps(steps, factory = this.engineFactory, format = this.currentEngine?.format ?? "onnx", context = {}) {
+		const needed = this.stepsNeedingLoad(steps, factory);
+		if (needed.length === 0) return;
+		const pool = factory ?? this.nullPoolKey;
+		const deviceKey = context.deviceKey ?? deviceKeyOf(this.currentEngine);
+		const models = needed.map(stepModelKey);
+		const refusal = this.loadRefusal(pool, deviceKey, models);
+		if (refusal !== null) {
+			this.reportDispatchLoadLoss(context, deviceKey, models, refusal.model, refusal.state, refusal.message);
+			throw refusal;
+		}
+		const setKey = [...models].sort().join(",");
+		let perPool = this.modelLoadInFlight.get(pool);
+		if (perPool === void 0) {
+			perPool = /* @__PURE__ */ new Map();
+			this.modelLoadInFlight.set(pool, perPool);
+		}
+		const flight = perPool.get(setKey) ?? this.issueModelLoad(pool, perPool, setKey, steps, factory, format, deviceKey);
+		try {
+			await flight.work;
+		} catch (err) {
+			if (err instanceof ModelLoadRefusedError) {
+				this.reportDispatchLoadLoss(context, deviceKey, models, err.model, err.state, err.message);
+				throw err;
+			}
+			const failed = failedModelOf(err) ?? models[0] ?? "";
+			this.reportDispatchLoadLoss(context, deviceKey, models, failed, `failed:${flight.generation ?? "unknown"}`, require_dist.errMsg(err));
+			throw err;
+		}
+	}
+	/**
+	* Why `models` may not be loaded on `pool` right now — a back-off after a
+	* failure, or a refusal by name after repeated compile timeouts — or `null`.
+	*/
+	loadRefusal(pool, deviceKey, models) {
+		const verdict = this.loadGovernor.check(pool, deviceKey, models);
+		if (verdict.kind === "go") return null;
+		return new ModelLoadRefusedError(verdict.kind === "refused" ? `model ${verdict.model} is refused on ${deviceKey}: its compile timed out ${verdict.timeouts} times (${verdict.error}) — re-arm with pipelineExecutor.rearmInferenceDevice` : `model ${verdict.model} failed to load on ${deviceKey} ${verdict.failures} time(s); not retried for ${verdict.retryInMs}ms (${verdict.error})`, verdict.model, stateOf(verdict));
+	}
+	/**
+	* A compile that outlived its soft bound came back on `factory`'s pool
+	* (D653 rounds 2-3). A SUCCESS wrote that model's cache and the worker loads
+	* again, so that model's back-off is lifted at once rather than waited out;
+	* no other model's is. A FAILURE is one more failure of that model, and its
+	* back-off grows.
+	*/
+	noteLateCompile(factory, deviceKey, event) {
+		const models = this.loadGovernor.settleLateCompile(factory, deviceKey, event.model, event.ok, event.error ?? "late compile failed");
+		if (event.ok) {
+			this.log.info("model loadable again after late compile", { meta: {
+				deviceKey,
+				model: event.model,
+				backoffLifted: models
+			} });
+			return;
+		}
+		this.log.warn("late compile failed — the model stays backed off", { meta: {
+			deviceKey,
+			model: event.model,
+			backoffExtended: models,
+			error: event.error ?? null
+		} });
+	}
+	/** The enabled steps the factory's pool does not reflect yet. */
+	stepsNeedingLoad(steps, factory) {
 		const needed = [];
-		for (const step of allEnabled) {
+		for (const step of flattenSteps(steps)) {
 			if (!step.enabled) continue;
 			if (factory !== null && !factory.needsPoolUpdate(step)) continue;
 			needed.push(step);
 		}
-		if (needed.length === 0) return;
-		const work = (async () => {
-			for (const step of needed) {
-				const modelEntry = await this.resolveModelEntry(step.addonId, step.modelId);
-				if (modelEntry && !(0, _camstack_system_addon_utils.isModelDownloaded)(this.modelsDir, modelEntry, format)) {
-					if (modelEntry.formats[format] === void 0) throw new Error(`Model "${modelEntry.id}" has no ${format} format build (available: ${Object.keys(modelEntry.formats).join(", ") || "none"}) — not retrying a permanent format mismatch`);
-					if (modelEntry.formats[format]?.url.startsWith("camstack-local://") === true) throw new Error(`Custom model "${modelEntry.id}" (${format}) is not present in this node's models directory — distribute it to this node from Model Studio before selecting it here`);
-					this.log.info("Downloading model for step", { meta: {
-						modelId: step.modelId,
-						format,
-						step: step.addonId
-					} });
-					await this.downloadWithRetry(modelEntry, format, 3);
-				}
+		return needed;
+	}
+	/**
+	* Issue ONE load for a model set on a pool, queued behind the pool's
+	* previous load, and record its outcome with the governor. The set is
+	* re-derived once the queue reaches it: the load ahead may have brought
+	* some of it in.
+	*/
+	issueModelLoad(pool, perPool, setKey, steps, factory, format, deviceKey) {
+		const previous = this.modelLoadTail.get(pool) ?? Promise.resolve();
+		const flight = {
+			work: Promise.resolve(),
+			generation: null
+		};
+		flight.work = (async () => {
+			await previous;
+			const needed = this.stepsNeedingLoad(steps, factory);
+			if (needed.length === 0) return;
+			const models = needed.map(stepModelKey);
+			const refusal = this.loadRefusal(pool, deviceKey, models);
+			if (refusal !== null) throw refusal;
+			try {
+				await this.downloadNeededModels(needed, format);
+				this.log.info("Loading additional models for benchmark", { meta: {
+					count: needed.length,
+					models,
+					deviceKey
+				} });
+				await factory.loadAdditional(needed);
+			} catch (err) {
+				flight.generation = this.recordModelLoadFailure(pool, deviceKey, models, err);
+				throw err;
 			}
-			this.log.info("Loading additional models for benchmark", { meta: {
-				count: needed.length,
-				models: needed.map((s) => `${s.addonId}/${s.modelId}`)
-			} });
-			await factory.loadAdditional(needed);
+			this.loadGovernor.recordSuccess(pool, deviceKey, models);
 		})();
-		this.modelLoadInFlight = work;
-		try {
-			await work;
-		} finally {
-			if (this.modelLoadInFlight === work) this.modelLoadInFlight = null;
+		perPool.set(setKey, flight);
+		const tail = flight.work.catch(() => void 0);
+		this.modelLoadTail.set(pool, tail);
+		tail.then(() => {
+			if (perPool.get(setKey) === flight) perPool.delete(setKey);
+			if (this.modelLoadTail.get(pool) === tail) this.modelLoadTail.delete(pool);
+		});
+		return flight;
+	}
+	/** Charge a failed load to the model that failed; say so once if it is now refused. */
+	recordModelLoadFailure(pool, deviceKey, models, err) {
+		const named = failedModelOf(err);
+		const failedModels = named !== null ? [named] : models;
+		const compileTimeout = err instanceof StepVariantLoadError && err.reason === "compile-timeout";
+		const poolModel = err instanceof StepVariantLoadError ? err.poolModel : null;
+		let generation = 0;
+		for (const model of failedModels) {
+			const outcome = this.loadGovernor.recordFailure(pool, deviceKey, model, require_dist.errMsg(err), compileTimeout, Date.now(), poolModel, err instanceof StepVariantLoadError ? err.reason : null);
+			generation = outcome.generation;
+			if (outcome.refusedNow) this.log.error("model REFUSED on this device — its compile timed out repeatedly", { meta: {
+				deviceKey,
+				model,
+				compileTimeouts: outcome.compileTimeouts,
+				error: require_dist.errMsg(err),
+				device: "keeps serving every other model",
+				rearm: "pipelineExecutor.rearmInferenceDevice"
+			} });
+		}
+		return generation;
+	}
+	/**
+	* One WARN per camera per state change of the model that cost it its frame.
+	* The ERROR for the failed load itself is written once, where the load
+	* failed (`Step variant load failed`); this line answers "which cameras
+	* did it cost?", tagged so it can be counted per camera.
+	*/
+	reportDispatchLoadLoss(context, deviceKey, models, model, state, error) {
+		if (!this.loadGovernor.shouldReport(context.deviceId, deviceKey, model, state)) return;
+		this.log.warn("model load for dispatch failed", {
+			...context.deviceId !== void 0 ? { tags: { deviceId: context.deviceId } } : {},
+			meta: {
+				deviceKey,
+				models,
+				model,
+				state,
+				error,
+				...context.outcome !== void 0 ? { outcome: context.outcome } : {}
+			}
+		});
+	}
+	/** Download any model of `needed` missing in `format` (the device's). */
+	async downloadNeededModels(needed, format) {
+		for (const step of needed) try {
+			await this.ensureStepModelArtifact(step, format);
+		} catch (err) {
+			throw new StepModelArtifactError(step.addonId, step.modelId, err);
+		}
+	}
+	/** Make one step's model artifact present on disk for `format`, or say why it cannot be. */
+	async ensureStepModelArtifact(step, format) {
+		const modelEntry = await this.resolveModelEntry(step.addonId, step.modelId);
+		if (modelEntry && !(0, _camstack_system_addon_utils.isModelDownloaded)(this.modelsDir, modelEntry, format)) {
+			if (modelEntry.formats[format] === void 0) throw new Error(`Model "${modelEntry.id}" has no ${format} format build (available: ${Object.keys(modelEntry.formats).join(", ") || "none"}) — not retrying a permanent format mismatch`);
+			if (modelEntry.formats[format]?.url.startsWith("camstack-local://") === true) throw new Error(`Custom model "${modelEntry.id}" (${format}) is not present in this node's models directory — distribute it to this node from Model Studio before selecting it here`);
+			this.log.info("Downloading model for step", { meta: {
+				modelId: step.modelId,
+				format,
+				step: step.addonId
+			} });
+			await this.downloadWithRetry(modelEntry, format, 3);
 		}
 	}
 	/** Download a model with retry + exponential backoff */
@@ -8481,6 +9692,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	}
 	async runPipelineBatchImpl(input) {
 		if (input.frames.length === 0) return { results: [] };
+		await this.getCustomModels();
 		const dispatchResolution = this.resolveStepsForDispatch({
 			steps: input.steps,
 			deviceKey: input.deviceKey,
@@ -8511,7 +9723,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		await this.ensureEngineFactory();
 		const factory = dispatchDeviceKey ? await this.resolveDeviceFactory(dispatchDeviceKey) : this.engineFactory;
 		if (!factory) throw new Error("runPipelineBatch: factory not initialised");
-		if (enabledSteps.filter((s) => factory.needsPoolUpdate(s)).length > 0) await this.ensureModelsForSteps(benchmarkSteps, factory, resolveFormat);
+		if (enabledSteps.filter((s) => factory.needsPoolUpdate(s)).length > 0) await this.ensureModelsForSteps(benchmarkSteps, factory, resolveFormat, { deviceKey: dispatchDeviceKey ?? deviceKeyOf(this.currentEngine) });
 		const canFastPath = singleRoot && allRaw && uniformDims && factory.supportsBatch();
 		this.log.info("runPipelineBatch path decision", { meta: {
 			phase: "batch",
@@ -8723,9 +9935,10 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		return this.engineFactory.cacheFrameInPool(buf, input.width, input.height, input.format);
 	}
 	async inferCached(input) {
+		if (typeof input.modelId !== "string" || input.modelId === "") throw new Error(`inferCached: no modelId for step "${input.stepId}" — refusing rather than running the step's active variant`);
 		await this.ensureEngineFactory();
 		if (!this.engineFactory) throw new Error("inferCached: factory not initialized");
-		return this.engineFactory.inferCached(input.stepId, input.frameId);
+		return this.engineFactory.inferCached(input.stepId, input.frameId, input.modelId);
 	}
 	async uncacheFrame(input) {
 		if (!this.engineFactory) return;
@@ -8766,7 +9979,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 			if (this.engineFactory.isReady()) return;
 			const dead = this.engineFactory;
 			this.engineFactory = null;
-			this.noteDeviceDeath(defaultDeviceKey, "pool worker is not ready");
+			this.noteFactoryDeath(defaultDeviceKey, dead);
 			await dead.dispose().catch(() => void 0);
 			this.refuseIfDeviceUnusable(defaultDeviceKey);
 		}
@@ -8784,7 +9997,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				logger: this.log.child("engine"),
 				pythonPath: this.executorOptions.pythonPath ?? "",
 				provisioning: this.executorOptions.provisioning,
-				resolveCustomModel: this.customModelResolver
+				resolveCustomModel: this.customModelResolver,
+				onCompileFinishedLate: (event) => this.noteLateCompile(factory, defaultDeviceKey, event)
 			});
 			try {
 				await factory.initialize([]);
@@ -8904,15 +10118,35 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* dispose — which rejects whatever was still in flight on it with a reason
 	* rather than letting those requests sit until their own deadlines.
 	*/
-	async condemnDeviceFactory(deviceKey, factory, reason) {
+	async condemnDeviceFactory(deviceKey, factory) {
 		if (this.factoriesByDevice.get(deviceKey) === factory) {
 			this.factoriesByDevice.delete(deviceKey);
 			this.deviceReaper.cancel(deviceKey);
-			this.noteDeviceDeath(deviceKey, reason);
+			this.noteFactoryDeath(deviceKey, factory);
 		}
 		await factory.dispose().catch(() => void 0);
 	}
 	/**
+	* Charge one dead pool to its device — unless it died of a compile that was
+	* still running at its hard bound (D653, fix round 1). That death is not a
+	* crash of the DEVICE: the model that hung was already counted when its
+	* load answered `compile-timeout`, and the same model timing out again on
+	* the respawn is refused by name (`ModelLoadGovernor`), which is what bounds
+	* the respawns. A crash stays a crash.
+	*/
+	noteFactoryDeath(deviceKey, factory) {
+		const cause = factory.getDeathCause();
+		if (cause !== null && !cause.chargesDeviceBudget) {
+			this.log.warn("inference pool recycled after a compile hung — not charged to the device budget", { meta: {
+				deviceKey,
+				reason: cause.reason,
+				cause: cause.message
+			} });
+			return;
+		}
+		this.noteDeviceDeath(deviceKey, cause?.message ?? "pool worker is not ready");
+	}
+	/**
 	* Every inference device this node currently refuses, and why.
 	*
 	* The CHANNEL the 2026-08-26 analysis found missing. Pool health lived
@@ -8947,7 +10181,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				state: "backoff",
 				since: Date.now(),
 				deaths: 0,
-				lastError: "pool worker is not ready"
+				lastError: deathReasonOf(factory)
 			});
 		}
 		return { unhealthy: [...byKey.values()].toSorted((a, b) => a.deviceKey.localeCompare(b.deviceKey)) };
@@ -8964,10 +10198,13 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* no-op, reported as such.
 	*/
 	async rearmInferenceDevice(input) {
-		const rearmed = this.deviceLiveness.rearm(input.deviceKey);
+		const rearmedDevice = this.deviceLiveness.rearm(input.deviceKey);
+		const rearmedModels = this.loadGovernor.rearm(input.deviceKey);
+		const rearmed = rearmedDevice || rearmedModels > 0;
 		this.log.info("inference device re-armed by operator", { meta: {
 			deviceKey: input.deviceKey,
-			rearmed
+			rearmed,
+			rearmedModels
 		} });
 		return { rearmed };
 	}
@@ -8984,7 +10221,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				this.deviceReaper.touch(deviceKey);
 				return existing;
 			}
-			await this.condemnDeviceFactory(deviceKey, existing, "pool worker is not ready");
+			await this.condemnDeviceFactory(deviceKey, existing);
 			this.refuseIfDeviceUnusable(deviceKey);
 		}
 		const inflight = this.deviceFactoryInflight.get(deviceKey);
@@ -8997,7 +10234,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 				logger: this.log.child(`engine:${deviceKey}`),
 				pythonPath: this.executorOptions.pythonPath ?? "",
 				provisioning: this.executorOptions.provisioning,
-				resolveCustomModel: this.customModelResolver
+				resolveCustomModel: this.customModelResolver,
+				onCompileFinishedLate: (event) => this.noteLateCompile(factory, deviceKey, event)
 			});
 			try {
 				await factory.initialize([]);
@@ -9340,7 +10578,7 @@ function buildSchemaSlots(formats, modelsDir, customByStep) {
 	const formatList = [...formats];
 	for (const pipelineStep of require_default_detection_model.ALL_PIPELINE_STEPS) {
 		const step = pipelineStep.definition;
-		const availableModels = mergeCustomModels(step.models, customByStep?.get(step.id) ?? []).filter((m) => m.legacy !== true && formatList.some((f) => m.formats[f]));
+		const availableModels = require_default_detection_model.mergeCustomModels(step.models, customByStep?.get(step.id) ?? []).filter((m) => m.legacy !== true && formatList.some((f) => m.formats[f]));
 		if (availableModels.length === 0) continue;
 		const slot = step.slot;
 		if (!slotMap.has(slot)) slotMap.set(slot, []);
@@ -9504,13 +10742,22 @@ function isDetailPlaneStep(addonId) {
 	}
 }
 /**
-* True when the step's FULL catalog ships no model with a build for
-* `format` — the single-step form of `collectZeroBuildIssues`. Unknown/custom
-* addonIds (no catalog to check) are never zero-build, mirroring the
-* collector's skip.
-*/
-function stepHasZeroBuildsFor(addonId, format) {
-	return collectZeroBuildIssues([{ addonId }], format).length > 0;
+* What a dispatch runs after its per-step loads (D657).
+*
+* An ALL-OR-NOTHING call (a benchmark, a replay — D56 — a capacity ramp) fails
+* on any failure, naming every step it lost ({@link DispatchStepsNotLoadedError}):
+* it measures or reproduces exactly what it names. A camera's LIVE dispatch
+* runs the tree without the steps whose model did not load, and fails — with
+* the first root's own named error — only when no root survives: a dispatch
+* with no root has nothing to produce.
+*/
+function stepsSurvivingLoad(steps, failures, policy) {
+	if (failures.length === 0) return steps;
+	if (policy === "all-or-nothing") throw new DispatchStepsNotLoadedError(failures);
+	const kept = withoutFailedSteps(steps, new Set(failures.map((f) => f.step)));
+	if (kept.some((s) => s.enabled)) return kept;
+	const roots = new Set(steps);
+	throw (failures.find((f) => roots.has(f.step)) ?? failures[0]).error;
 }
 /**
 * The union of model formats a step's catalog ships ANY build for — the