@camstack/addon-pipeline 1.2.130 → 1.2.132

package/dist/detection-pipeline/index.mjs

@@ -1,8 +1,8 @@
 import { n as __require } from "../chunk-DnnnRqeS.mjs";
-import { $ as pickClusterStepModels, D as YAMNET_TO_MACRO, Dt as hydrateSchema, Et as createEvent, Ft as sleep, G as evaluateZoneRules, Gt as string, I as defaultDeviceFor$1, Kt as union, Lt as array, Mt as parseJsonUnknown, St as BaseAddon, Ut as object, V as detectionPipelineCapability, W as enumerateInferenceDevices, Z as overlayClusterStepSettings, c as DEFAULT_CLUSTER_STEP_MODELS, dt as runtimeDevices$1, et as pickClusterStepSettings, gt as errMsg, jt as nodePin, l as DEFAULT_CLUSTER_STEP_SETTINGS, lt as resolvePoolMemoryPolicy, mt as supportedRuntimes$1, p as DEVICE_BACKEND_TO_FORMAT, q as inferModelProvider, qt as EventCategory, rt as pipelineExecutorCapability, st as resolveClusterStepModelId, t as APPLE_SA_TO_MACRO, x as PoolMemoryWatchdog } from "../dist-Wt81pN5Y.mjs";
-import { a as ALL_PIPELINE_STEPS, c as getDefaultModelForFormatFromDef, d as resolveModelForFormat, f as landmarkPrecisionVerdict, l as getStep, n as startEventLoopStallMonitor, o as ALL_STEPS, r as localFrameRegistry, s as getDefaultModelForFormat, u as getStepDefinition } from "../event-loop-stall-monitor-CtQCOZqm.mjs";
+import { $ as pickClusterStepModels, D as YAMNET_TO_MACRO, Dt as hydrateSchema, Et as createEvent, Ft as sleep, G as evaluateZoneRules, Gt as string, I as defaultDeviceFor$1, Kt as union, Lt as array, Mt as parseJsonUnknown, St as BaseAddon, Ut as object, V as detectionPipelineCapability, W as enumerateInferenceDevices, Z as overlayClusterStepSettings, c as DEFAULT_CLUSTER_STEP_MODELS, dt as runtimeDevices$1, et as pickClusterStepSettings, gt as errMsg, jt as nodePin, l as DEFAULT_CLUSTER_STEP_SETTINGS, lt as resolvePoolMemoryPolicy, mt as supportedRuntimes$1, p as DEVICE_BACKEND_TO_FORMAT, q as inferModelProvider, qt as EventCategory, rt as pipelineExecutorCapability, st as resolveClusterStepModelId, t as APPLE_SA_TO_MACRO, x as PoolMemoryWatchdog } from "../dist-BZYUUBUx.mjs";
+import { a as ALL_PIPELINE_STEPS, c as getDefaultModelForFormatFromDef, d as resolveModelForFormat, f as landmarkPrecisionVerdict, l as getStep, n as startEventLoopStallMonitor, o as ALL_STEPS, r as localFrameRegistry, s as getDefaultModelForFormat, u as getStepDefinition } from "../event-loop-stall-monitor-IACE22fK.mjs";
 import { t as getSharp } from "../lazy-sharp-BtT5x-qP.mjs";
-import { n as pickNodePlatformArch, t as readProcessMemory } from "../process-memory-1_D3EZT9.mjs";
+import { n as pickNodePlatformArch, t as readProcessMemory } from "../process-memory-C7rsraCR.mjs";
 import { a as ensureModel, o as isModelDownloaded, r as deleteModelFromDisk } from "../addon-utils-CgBCF-mE.mjs";
 import sharp from "sharp";
 import { spawn } from "node:child_process";
@@ -901,15 +901,20 @@ var PoolWorker = class {
 			} });
 			this.rejectAll(err);
 		});
-		this.process.on("exit", (code) => {
-			if (code !== 0 && code !== null) {
-				this.log.error("Worker process exited", { meta: {
-					worker: this.opts.workerLabel,
-					code
-				} });
-				this.rejectAll(/* @__PURE__ */ new Error(`Worker process exited with code ${code}`));
-			}
+		const spawnedProcess = this.process;
+		spawnedProcess.on("exit", (code, signal) => {
 			this.ready = false;
+			if (this.process !== spawnedProcess) return;
+			this.log.error("Worker process exited", { meta: {
+				worker: this.opts.workerLabel,
+				pid: spawnedProcess.pid ?? null,
+				runtime: this.opts.poolRuntime,
+				device: this.opts.device ?? "default",
+				code,
+				signal,
+				inFlight: this.pending.size
+			} });
+			this.rejectAll(/* @__PURE__ */ new Error(`PoolWorker[${this.opts.workerLabel}]: worker process exited (code=${code ?? "null"}, signal=${signal ?? "none"})`));
 		});
 		this.process.stdout.on("data", (chunk) => {
 			const t0 = Date.now();
@@ -1037,6 +1042,7 @@ var PoolWorker = class {
 		this.process = null;
 		this.ready = false;
 		await terminateChild(proc, POOL_WORKER_TERM_GRACE_MS);
+		this.rejectAll(/* @__PURE__ */ new Error(`PoolWorker[${this.opts.workerLabel}]: pool disposed while the request was in flight`));
 	}
 	/**
 	* Shed an inference request when the worker already has {@link
@@ -2349,6 +2355,23 @@ var EngineFactory = class {
 			active: l.active
 		}));
 	}
+	/**
+	* Is this factory's pool alive and able to answer?
+	*
+	* `SharedInferencePool.isReady()` and `PoolWorker.isReady()` have always
+	* existed; this factory surfaced `getPoolPids()` and `getPoolBacklog()` and
+	* **not** readiness, so the one signal every caller needed sat one layer
+	* below all of them and was never lifted. That is why a pool whose Python
+	* worker was SIGABRT'd 1.3 s after `Pool ready` was handed back to 370 000
+	* consecutive dispatches over 31 hours: `resolveDeviceFactory` had nothing
+	* to ask.
+	*
+	* A factory that never initialised answers `false` — a pool that is not
+	* there cannot serve a frame, and the caller's job is the same either way.
+	*/
+	isReady() {
+		return this.pool?.isReady() ?? false;
+	}
 	/** Native pid of the underlying Python pool, if any. */
 	getPoolPid() {
 		return this.pool?.getPid() ?? null;
@@ -2655,24 +2678,160 @@ var IdlePoolReaper = class {
 	}
 };
 //#endregion
+//#region src/detection-pipeline/engine/inference-device-liveness.ts
+/**
+* A dispatch refused because its inference device is not usable.
+*
+* Distinct from `PoolWorker[w0]: not initialized`, which said the same thing
+* for three different states and so distinguished none of them. This one names
+* the device, the state and the underlying death, so a single rejected frame
+* is decidable without correlating logs.
+*/
+var InferenceDeviceUnusableError = class extends Error {
+	deviceKey;
+	state;
+	constructor(unhealthy) {
+		super(`inference device "${unhealthy.deviceKey}" is ${unhealthy.state} after ${unhealthy.deaths} pool death(s): ${unhealthy.lastError}`);
+		this.name = "InferenceDeviceUnusableError";
+		this.deviceKey = unhealthy.deviceKey;
+		this.state = unhealthy.state;
+	}
+};
+/**
+* Defaults. Three deaths inside ten minutes trips the breaker, so a
+* deterministic crash costs **three** Python spawns instead of one per frame,
+* and a flaky-once-an-hour pool still self-heals forever.
+*/
+var DEFAULT_INFERENCE_DEVICE_LIVENESS = {
+	windowMs: 600 * 1e3,
+	maxDeaths: 3,
+	maxBackoffMs: 3e4
+};
+var InferenceDeviceLivenessSupervisor = class {
+	opts;
+	state = /* @__PURE__ */ new Map();
+	constructor(opts = DEFAULT_INFERENCE_DEVICE_LIVENESS) {
+		this.opts = opts;
+	}
+	/**
+	* Record one pool death for `deviceKey` and decide what happens next.
+	* `now` is injectable for deterministic tests. Never throws.
+	*/
+	recordDeath(deviceKey, error, now = Date.now()) {
+		const cutoff = now - this.opts.windowMs;
+		const recent = [...(this.state.get(deviceKey)?.deaths ?? []).filter((t) => t >= cutoff), now];
+		if (recent.length >= this.opts.maxDeaths) {
+			this.state.set(deviceKey, {
+				deaths: recent,
+				failed: true,
+				retryAtMs: Number.POSITIVE_INFINITY,
+				lastError: error,
+				since: now
+			});
+			return {
+				action: "failed",
+				deathsInWindow: recent.length
+			};
+		}
+		const backoffMs = Math.min(this.opts.maxBackoffMs, 500 * 2 ** Math.min(6, Math.max(0, recent.length - 1)));
+		this.state.set(deviceKey, {
+			deaths: recent,
+			failed: false,
+			retryAtMs: now + backoffMs,
+			lastError: error,
+			since: now
+		});
+		return {
+			action: "rebuild",
+			backoffMs,
+			deathsInWindow: recent.length
+		};
+	}
+	/** Has the breaker tripped for this device? */
+	isFailed(deviceKey) {
+		return this.state.get(deviceKey)?.failed ?? false;
+	}
+	/**
+	* Why this device cannot be used right now, or `null` when it may be
+	* (re)built. A device inside its backoff answers `'backoff'`; a device whose
+	* budget is spent answers `'failed'` forever.
+	*/
+	unusableReason(deviceKey, now = Date.now()) {
+		const s = this.state.get(deviceKey);
+		if (!s) return null;
+		if (s.failed) return {
+			deviceKey,
+			state: "failed",
+			since: s.since,
+			deaths: s.deaths.length,
+			lastError: s.lastError
+		};
+		if (now < s.retryAtMs) return {
+			deviceKey,
+			state: "backoff",
+			since: s.since,
+			deaths: s.deaths.length,
+			lastError: s.lastError
+		};
+		return null;
+	}
+	/**
+	* Every device the breaker currently refuses, sorted by key.
+	*
+	* This is what the health channel publishes. `'backoff'` entries are
+	* included deliberately: a device mid-backoff cannot serve a frame either,
+	* and the reader's own arm/apply reluctance (D49) is what keeps a single
+	* transient observation from excluding it.
+	*/
+	unhealthy(now = Date.now()) {
+		const out = [];
+		for (const deviceKey of [...this.state.keys()].toSorted()) {
+			const reason = this.unusableReason(deviceKey, now);
+			if (reason) out.push(reason);
+		}
+		return out;
+	}
+	/**
+	* Operator re-arm: forget everything about `deviceKey` so the next dispatch
+	* builds a fresh pool with a full budget. Returns whether there was anything
+	* to forget — a terminal state nobody can clear is a silent fault, and this
+	* is the clearing.
+	*/
+	rearm(deviceKey) {
+		return this.state.delete(deviceKey);
+	}
+	/** Forget every device (shutdown / full engine re-spin). */
+	reset() {
+		this.state.clear();
+	}
+};
+//#endregion
 //#region src/detection-pipeline/engine/pool-memory-guard.ts
 function createDetectionPoolMemoryGuard(source, logger) {
 	const log = logger.child("pool-memory");
+	/** Pools already reported dead — the `pool is DEAD` line fires on the EDGE.
+	*  A pool nobody dispatches to is never condemned, so without this gate one
+	*  idle corpse writes an ERROR every sweep forever, which is how a real
+	*  signal becomes noise. Cleared when the pool comes back or goes away. */
+	const reportedDead = /* @__PURE__ */ new Set();
 	return new PoolMemoryWatchdog({
 		policy: resolvePoolMemoryPolicy(process.env),
 		log,
 		restart: (key) => source.restartPool(key),
 		sample: async () => {
 			const out = [];
+			const seen = /* @__PURE__ */ new Set();
 			for (const pool of source.listPools()) {
-				const telemetry = await samplePool(pool, log);
+				seen.add(pool.key);
+				const telemetry = await samplePool(pool, log, reportedDead);
 				if (telemetry) out.push(telemetry);
 			}
+			for (const key of [...reportedDead]) if (!seen.has(key)) reportedDead.delete(key);
 			return out;
 		}
 	});
 }
-async function samplePool(pool, log) {
+async function samplePool(pool, log, reportedDead) {
 	const pids = pool.factory.getPoolPids();
 	if (pids.length === 0) return null;
 	let maxRssBytes = 0;
@@ -2696,7 +2855,18 @@ async function samplePool(pool, log) {
 			threads: mem.threads
 		});
 	}
-	if (workers.length === 0) return null;
+	if (workers.length === 0) {
+		if (!pool.factory.isReady()) {
+			if (!reportedDead.has(pool.key)) {
+				reportedDead.add(pool.key);
+				log.error("inference pool is DEAD — every worker pid is gone and the pool is not ready", { meta: {
+					poolKey: pool.key,
+					pids
+				} });
+			}
+		} else reportedDead.delete(pool.key);
+		return null;
+	}
 	const backlog = pool.factory.getPoolBacklog();
 	const memStats = await pool.factory.getPoolMemStats().catch((err) => err instanceof Error ? err.message : String(err));
 	return {
@@ -8401,7 +8571,16 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* If global steps exist, initializes with them. Otherwise, creates an empty pool.
 	*/
 	async ensureEngineFactory() {
-		if (this.engineFactory) return;
+		const defaultDeviceKey = deviceKeyOf(this.currentEngine);
+		this.refuseIfDeviceUnusable(defaultDeviceKey);
+		if (this.engineFactory) {
+			if (this.engineFactory.isReady()) return;
+			const dead = this.engineFactory;
+			this.engineFactory = null;
+			this.noteDeviceDeath(defaultDeviceKey, "pool worker is not ready");
+			await dead.dispose().catch(() => void 0);
+			this.refuseIfDeviceUnusable(defaultDeviceKey);
+		}
 		const inflight = this.engineFactoryInflight;
 		if (inflight) {
 			await inflight;
@@ -8423,11 +8602,12 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 			} catch (err) {
 				await factory.dispose().catch(() => void 0);
 				this.log.error("Node-default pool failed to initialize", { meta: {
-					deviceKey: deviceKeyOf(this.currentEngine),
+					deviceKey: defaultDeviceKey,
 					backend: this.currentEngine.backend,
 					device: this.currentEngine.device ?? null,
 					error: errMsg(err)
 				} });
+				this.noteDeviceDeath(defaultDeviceKey, errMsg(err));
 				throw err;
 			}
 			this.engineFactory = factory;
@@ -8476,16 +8656,147 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* fail (the loser can't claim the already-held device).
 	*/
 	deviceFactoryInflight = /* @__PURE__ */ new Map();
+	/**
+	* Per-`deviceKey` restart budget for the Python pools (D6 discipline).
+	*
+	* In-memory and per-runner ON PURPOSE. The terminal state means "this
+	* accelerator aborted N times in a row a few minutes ago"; a runner respawn
+	* re-derives it in three dispatches if it is still true, and persisting it
+	* would outlive the driver update that fixes it. The operator's explicit
+	* re-arm is {@link rearmInferenceDevice}, and the state is never invisible:
+	* it is published on {@link getInferenceDeviceHealth}.
+	*/
+	deviceLiveness = new InferenceDeviceLivenessSupervisor();
+	/**
+	* Throw if `deviceKey` is currently refused — before a single Python process
+	* is spawned, a model index is allocated, or a frame is copied.
+	*
+	* The cost of NOT asking, measured: each frame of the 2026-08-25 storm ran
+	* the full `models to load` → `allocateIndex` → `loadModel` → throw cycle at
+	* 13 Hz for 31 hours, leaking one model index per frame — and past 254
+	* (`encodeModelByte`) that pool could never have served inference again even
+	* if its worker had come back.
+	*/
+	refuseIfDeviceUnusable(deviceKey) {
+		const unhealthy = this.deviceLiveness.unusableReason(deviceKey);
+		if (unhealthy) throw new InferenceDeviceUnusableError(unhealthy);
+	}
+	/**
+	* Record one pool death against the budget and SAY what was decided.
+	*
+	* One line per state change, not one per dispatch: the refusals that follow
+	* are already carried by the rejected frame's own error. The `failed`
+	* transition is an ERROR because it is the terminal state — an operator who
+	* never sees this line has a silently half-dead node, which is the failure
+	* this whole path exists to end.
+	*/
+	noteDeviceDeath(deviceKey, reason) {
+		const decision = this.deviceLiveness.recordDeath(deviceKey, reason);
+		if (decision.action === "failed") {
+			this.log.error("inference device FAILED — restart budget exhausted, no pool will be spawned", { meta: {
+				deviceKey,
+				deathsInWindow: decision.deathsInWindow,
+				reason,
+				rearm: "pipelineExecutor.rearmInferenceDevice"
+			} });
+			return;
+		}
+		this.log.warn("inference pool died — rebuilding under the restart budget", { meta: {
+			deviceKey,
+			deathsInWindow: decision.deathsInWindow,
+			backoffMs: decision.backoffMs,
+			reason
+		} });
+	}
+	/**
+	* Retire a dead per-device pool: drop it from the map FIRST (so a concurrent
+	* dispatch rebuilds fresh through the single-flight guard rather than
+	* grabbing the corpse), cancel its idle timer, charge the budget, then
+	* dispose — which rejects whatever was still in flight on it with a reason
+	* rather than letting those requests sit until their own deadlines.
+	*/
+	async condemnDeviceFactory(deviceKey, factory, reason) {
+		if (this.factoriesByDevice.get(deviceKey) === factory) {
+			this.factoriesByDevice.delete(deviceKey);
+			this.deviceReaper.cancel(deviceKey);
+			this.noteDeviceDeath(deviceKey, reason);
+		}
+		await factory.dispose().catch(() => void 0);
+	}
+	/**
+	* Every inference device this node currently refuses, and why.
+	*
+	* The CHANNEL the 2026-08-26 analysis found missing. Pool health lived
+	* entirely inside this addon and reached no routing decision at any level:
+	* the capability gate is keyed on model FORMAT and so is permanently blind
+	* between `openvino:gpu` and `openvino:npu`, and the orchestrator's live
+	* probe answers about HARDWARE, which was present the whole time. So the
+	* balancer kept handing this node cameras by rotation — 20 sessions onto a
+	* dead pool in 20 minutes — and every one of them lost its frames.
+	*
+	* Two sources, deliberately merged into one answer:
+	*  - the breaker's own `failed` / `backoff` states, and
+	*  - any factory currently cached but NOT ready. A pool nobody has dispatched
+	*    to since it died has not yet been condemned by the dispatch path, and a
+	*    device with no traffic is exactly the one that must stop being chosen
+	*    BEFORE the traffic arrives. Charging the budget is the dispatch path's
+	*    job; reporting is free and costs no spawn.
+	*
+	* Read-only and synchronous over in-memory state: it never probes, never
+	* spawns, and never throws — the reader's whole safety argument rests on a
+	* failed read changing nothing, and a read that can fail for its own reasons
+	* would poison that.
+	*/
+	async getInferenceDeviceHealth() {
+		const byKey = /* @__PURE__ */ new Map();
+		for (const entry of this.deviceLiveness.unhealthy()) byKey.set(entry.deviceKey, entry);
+		const live = [...this.factoriesByDevice, ...this.engineFactory ? [[deviceKeyOf(this.currentEngine), this.engineFactory]] : []];
+		for (const [deviceKey, factory] of live) {
+			if (factory.isReady() || byKey.has(deviceKey)) continue;
+			byKey.set(deviceKey, {
+				deviceKey,
+				state: "backoff",
+				since: Date.now(),
+				deaths: 0,
+				lastError: "pool worker is not ready"
+			});
+		}
+		return { unhealthy: [...byKey.values()].toSorted((a, b) => a.deviceKey.localeCompare(b.deviceKey)) };
+	}
+	/**
+	* Operator re-arm of a terminally failed inference device: forget the budget
+	* so the next dispatch builds a fresh pool.
+	*
+	* A terminal state with no way out is a silent fault dressed as a safety
+	* feature. This is the way out, and it is the ONLY automatic-looking one on
+	* offer: there is deliberately no timed probation, because the crash this
+	* bounds is deterministic and a probation would simply respawn Python
+	* forever at a slower rate. Re-arming a device that was never failed is a
+	* no-op, reported as such.
+	*/
+	async rearmInferenceDevice(input) {
+		const rearmed = this.deviceLiveness.rearm(input.deviceKey);
+		this.log.info("inference device re-armed by operator", { meta: {
+			deviceKey: input.deviceKey,
+			rearmed
+		} });
+		return { rearmed };
+	}
 	async resolveDeviceFactory(deviceKey) {
 		const engine = resolveDeviceEngine(deviceKey);
 		if (enginesEqual(engine, this.currentEngine)) {
 			await this.ensureEngineFactory();
 			return this.engineFactory;
 		}
+		this.refuseIfDeviceUnusable(deviceKey);
 		const existing = this.factoriesByDevice.get(deviceKey);
 		if (existing) {
-			this.deviceReaper.touch(deviceKey);
-			return existing;
+			if (existing.isReady()) {
+				this.deviceReaper.touch(deviceKey);
+				return existing;
+			}
+			await this.condemnDeviceFactory(deviceKey, existing, "pool worker is not ready");
+			this.refuseIfDeviceUnusable(deviceKey);
 		}
 		const inflight = this.deviceFactoryInflight.get(deviceKey);
 		if (inflight) return inflight;
@@ -8510,6 +8821,7 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 					error: err instanceof Error ? err.message : String(err)
 				} });
 				await factory.dispose().catch(() => void 0);
+				this.noteDeviceDeath(deviceKey, errMsg(err));
 				throw err;
 			}
 			this.factoriesByDevice.set(deviceKey, factory);