@camstack/addon-pipeline 1.2.180 → 1.2.182

package/dist/detection-pipeline/index.js

@@ -869,6 +869,8 @@ var PoolWorker = class {
 	*  Surfaced so "the camera is quiet" and "the camera is saturated" can be
 	*  told apart without reading the logs. */
 	shedCount = 0;
+	/** Requests that hit their client-side deadline (`inference request timed out`). */
+	timedOutCount = 0;
 	/**
 	* Live requests whose deadline fired IN FLIGHT and were answered dropped by
 	* the local watchdog (the worker never replied in time). With a v2 worker
@@ -903,6 +905,9 @@ var PoolWorker = class {
 	getShedCount() {
 		return this.shedCount;
 	}
+	getTimedOutCount() {
+		return this.timedOutCount;
+	}
 	isReady() {
 		return this.ready;
 	}
@@ -1197,6 +1202,7 @@ var PoolWorker = class {
 					});
 					return;
 				}
+				this.timedOutCount++;
 				this.log.error("inference request timed out", {
 					...deviceId !== void 0 ? { tags: { deviceId } } : {},
 					meta: {
@@ -1468,6 +1474,15 @@ var SharedInferencePool = class {
 	getDeadlineExpiredShedCount() {
 		return this.deadlineExpiredShedCount;
 	}
+	/**
+	* Requests that timed out client-side, summed over the live workers. A
+	* worker restart resets its share; the inference-timeout guard treats a
+	* drop as "the new count is the delta". This is the metric 1.2.218's
+	* regression was invisible on for 4 h 19 min (2026-09-04).
+	*/
+	getTimedOutCount() {
+		return this.workers.reduce((sum, w) => sum + w.getTimedOutCount(), 0);
+	}
 	getHandle(modelIndex) {
 		return new PoolHandle(this, modelIndex);
 	}
@@ -2502,12 +2517,14 @@ var EngineFactory = class {
 			inFlight: 0,
 			shed: 0,
 			dropped: 0,
-			deadlineExpired: 0
+			deadlineExpired: 0,
+			timedOut: 0
 		};
 		return {
 			...this.pool.getBacklog(),
 			dropped: this.pool.getDroppedResponseCount(),
-			deadlineExpired: this.pool.getDeadlineExpiredShedCount()
+			deadlineExpired: this.pool.getDeadlineExpiredShedCount(),
+			timedOut: this.pool.getTimedOutCount()
 		};
 	}
 	/** Python-side per-worker memory diagnostics (see SharedInferencePool). */
@@ -3001,6 +3018,235 @@ async function samplePool(pool, log, reportedDead) {
 	};
 }
 //#endregion
+//#region src/detection-pipeline/engine/inference-timeout-guard.ts
+/**
+* Inference-timeout regression guard — "did this update make the GPU time
+* out more than the last one did?", asked by the node itself.
+*
+* ## Why
+*
+* `@camstack/server` 1.2.218 reintroduced the inference timeouts 1.2.216 had
+* fixed: ~3 760 `inference request timed out` an hour for 4 h 19 min (16 215
+* lines, ~19 000 `runInference failed`), on a cluster whose previous hour had
+* seen ~5. Nothing said so. 1.2.219 fixed it again as a side effect of the
+* next deploy, and the regression was found a day later by an audit that
+* counted log lines per version by hand (2026-09-05). A green typecheck, a
+* green suite and a 200 on `/health` all held throughout.
+*
+* ## What it does
+*
+* Every five minutes the guard records how many inference requests timed out
+* on this node under the current FINGERPRINT (the closure version). The last
+* hour of buckets is persisted in the addon store, so a restart hands the
+* previous fingerprint's rate to the next one. Once the new fingerprint has a
+* quarter hour of its own, the two hourly rates are compared:
+*
+*  - ≥ `TIMEOUT_GUARD_RATIO`× worse AND above `TIMEOUT_GUARD_FLOOR_PER_HOUR`
+*    → `system.liveness-failed` (AlertCenter raises a persistent alert keyed
+*    on the finding id) and an ERROR line naming both versions and rates;
+*  - ≥ `TIMEOUT_GUARD_RATIO`× better → an INFO line, no alert;
+*  - the same fingerprint (a plain restart) → nothing to compare.
+*
+* The rate is per hour, not per frame: it is the unit the audit measured and
+* the unit an operator reads. A traffic doubling doubles the rate, which is
+* why the ratio is five and not two.
+*
+* Pure core (`recordTimeoutBucket`, `evaluateTimeoutRegression`) tested in
+* `__tests__/inference-timeout-guard.spec.ts`; the runtime wrapper owns the
+* timer, the store and the emit.
+*/
+var TIMEOUT_GUARD_BUCKET_MS = 5 * 6e4;
+/** Addon-store key the ledger persists under (the addon's own settings blob). */
+var INFERENCE_TIMEOUT_LEDGER_KEY = "_inferenceTimeoutLedger";
+var HOUR_MS = 60 * 6e4;
+function recordTimeoutBucket(ledger, bucket) {
+	const buckets = [...ledger.buckets, bucket];
+	return {
+		version: ledger.version,
+		buckets: buckets.length > 12 ? buckets.slice(buckets.length - 12) : buckets
+	};
+}
+/** Timeouts per hour over the ledger's buckets; `null` under the minimum. */
+function timeoutsPerHour(ledger) {
+	if (ledger === null || ledger.buckets.length < 3) return null;
+	return ledger.buckets.reduce((sum, b) => sum + b.timeouts, 0) / (ledger.buckets.length * TIMEOUT_GUARD_BUCKET_MS) * HOUR_MS;
+}
+function evaluateTimeoutRegression(input) {
+	const { previous, current } = input;
+	const previousPerHour = timeoutsPerHour(previous);
+	const currentPerHour = timeoutsPerHour(current);
+	const base = {
+		previousVersion: previous?.version ?? null,
+		currentVersion: current.version,
+		previousPerHour,
+		currentPerHour
+	};
+	if (previous === null || previous.version === current.version || previousPerHour === null || currentPerHour === null) return {
+		kind: "not-comparable",
+		...base
+	};
+	if (currentPerHour >= 20 && currentPerHour >= Math.max(previousPerHour, 20 / 5) * 5) return {
+		kind: "regressed",
+		...base
+	};
+	if (previousPerHour >= 20 && currentPerHour * 5 <= previousPerHour) return {
+		kind: "improved",
+		...base
+	};
+	return {
+		kind: "quiet",
+		...base
+	};
+}
+/**
+* The closure version, read from the closure the runner was started from.
+*
+* The forked runner script is `<closure>/node_modules/@camstack/system/dist/
+* addon-runner.js` (hub: `/data/server-root/current/…`); the closure's own
+* `@camstack/server/package.json` sits two directories up. This is read from
+* disk rather than asked of `server-management` because that cap is
+* server-provided and the parent refuses to route it for a forked addon —
+* "no provider registered for cap server-management", measured on all
+* three nodes on 2026-09-05. A file the process was started from cannot
+* be refused.
+*/
+function closureVersionFromRunnerPath(runnerScript, fs) {
+	if (runnerScript === void 0 || runnerScript.length === 0) return null;
+	let dir = (0, node_path.dirname)(runnerScript);
+	for (let depth = 0; depth < 12; depth += 1) {
+		const candidate = (0, node_path.join)(dir, "node_modules", "@camstack", "server", "package.json");
+		const raw = fs.readFile(candidate);
+		if (raw !== null) {
+			try {
+				const parsed = JSON.parse(raw);
+				const version = typeof parsed === "object" && parsed !== null ? parsed.version : void 0;
+				if (typeof version === "string" && version.length > 0) return `server@${version}`;
+			} catch {
+				return null;
+			}
+			return null;
+		}
+		const parent = (0, node_path.dirname)(dir);
+		if (parent === dir) break;
+		dir = parent;
+	}
+	return null;
+}
+function readStoredLedgers(blob) {
+	const raw = blob[INFERENCE_TIMEOUT_LEDGER_KEY];
+	if (typeof raw !== "object" || raw === null) return {};
+	const r = raw;
+	return {
+		...isLedger(r.current) ? { current: r.current } : {},
+		...isLedger(r.previous) ? { previous: r.previous } : {}
+	};
+}
+function isLedger(value) {
+	if (typeof value !== "object" || value === null) return false;
+	const v = value;
+	return typeof v.version === "string" && Array.isArray(v.buckets) && v.buckets.every((b) => typeof b === "object" && b !== null && typeof b.at === "number" && typeof b.timeouts === "number");
+}
+function createInferenceTimeoutGuard(deps) {
+	const now = deps.now ?? (() => Date.now());
+	const findingId = `inference-timeout-regression:${deps.nodeId}`;
+	let timer = null;
+	let current = null;
+	let previous = null;
+	let lastTotal = 0;
+	let raised = false;
+	let improvementLogged = false;
+	const ensureLedger = async () => {
+		if (current !== null) return true;
+		const fingerprint = await deps.resolveFingerprint();
+		if (fingerprint === null) return false;
+		const stored = readStoredLedgers(await deps.readStore());
+		if (stored.current !== void 0 && stored.current.version === fingerprint) {
+			current = stored.current;
+			previous = stored.previous ?? null;
+		} else {
+			current = {
+				version: fingerprint,
+				buckets: []
+			};
+			previous = stored.current ?? stored.previous ?? null;
+		}
+		lastTotal = deps.readTotalTimeouts();
+		deps.log.info("inference timeout guard armed", { meta: {
+			fingerprint,
+			previousVersion: previous?.version ?? null,
+			previousPerHour: timeoutsPerHour(previous),
+			bucketMs: TIMEOUT_GUARD_BUCKET_MS
+		} });
+		return true;
+	};
+	const tick = async () => {
+		try {
+			if (!await ensureLedger() || current === null) return;
+			const total = deps.readTotalTimeouts();
+			const delta = total >= lastTotal ? total - lastTotal : total;
+			lastTotal = total;
+			current = recordTimeoutBucket(current, {
+				at: now(),
+				timeouts: delta
+			});
+			await deps.writeStore({ [INFERENCE_TIMEOUT_LEDGER_KEY]: {
+				current,
+				...previous !== null ? { previous } : {}
+			} });
+			const verdict = evaluateTimeoutRegression({
+				previous,
+				current
+			});
+			const meta = {
+				previousVersion: verdict.previousVersion,
+				currentVersion: verdict.currentVersion,
+				previousPerHour: verdict.previousPerHour,
+				currentPerHour: verdict.currentPerHour
+			};
+			if (verdict.kind === "regressed") {
+				if (!raised) {
+					raised = true;
+					const message = `inference requests time out ${Math.round(verdict.currentPerHour ?? 0)}/h under ${verdict.currentVersion}, against ${Math.round(verdict.previousPerHour ?? 0)}/h under ${verdict.previousVersion ?? "unknown"} — the update regressed inference on ${deps.nodeId}`;
+					deps.log.error("inference timeout REGRESSION after update", { meta });
+					deps.emitFailed({
+						findingId,
+						severity: "error",
+						title: "Inference timeouts regressed after update",
+						message
+					});
+				}
+				return;
+			}
+			if (raised && verdict.kind !== "not-comparable") {
+				raised = false;
+				deps.log.info("inference timeout regression cleared", { meta });
+				deps.emitRecovered(findingId);
+			}
+			if (verdict.kind === "improved" && !improvementLogged) {
+				improvementLogged = true;
+				deps.log.info("inference timeouts improved after update", { meta });
+			}
+		} catch (err) {
+			deps.log.warn("inference timeout guard tick failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
+		}
+	};
+	return {
+		start: () => {
+			if (timer !== null) return;
+			timer = setInterval(() => {
+				tick();
+			}, TIMEOUT_GUARD_BUCKET_MS);
+			timer.unref?.();
+			ensureLedger();
+		},
+		tick,
+		stop: () => {
+			if (timer !== null) clearInterval(timer);
+			timer = null;
+		}
+	};
+}
+//#endregion
 //#region src/detection-pipeline/engine-provisioner.ts
 /** Incremental backoff growing to a ~5 min cap; retries indefinitely at cap. */
 var BACKOFF_SCHEDULE_MS = [
@@ -6179,6 +6425,11 @@ function applyZoneRuleGate(result, zones, rules) {
 * This is the main provider that consumers (DetectionWiring, Benchmark, tRPC)
 * interact with. It manages the engine factory, pipeline executor, and config persistence.
 */
+/** Event source of the inference-timeout guard's liveness findings. */
+var TIMEOUT_GUARD_SOURCE = {
+	type: "addon",
+	id: "detection-pipeline"
+};
 var KEY_TEMPLATES = "pipelineTemplates";
 function pythonModuleForBackend(backend) {
 	switch (backend) {
@@ -6606,6 +6857,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* restart action. Started in `init()`, stopped in `shutdown()`.
 	*/
 	poolMemoryGuard = null;
+	/** Compares this node's inference-timeout rate across closure updates. */
+	inferenceTimeoutGuard = null;
 	/** Watchdog view of the live pools: the node-default pool plus every
 	*  per-device pool. Keys are stable per pool identity so the watchdog's
 	*  baseline tracks one pool across sweeps. */
@@ -6702,6 +6955,30 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 			}, this.log);
 			this.poolMemoryGuard.start();
 		}
+		if (!this.inferenceTimeoutGuard) {
+			const nodeId = this.localProbeNodeId();
+			this.inferenceTimeoutGuard = createInferenceTimeoutGuard({
+				readTotalTimeouts: () => this.listGuardedPools().reduce((sum, pool) => sum + pool.factory.getPoolBacklog().timedOut, 0),
+				readStore: () => this.readStore(),
+				writeStore: (patch) => this.writeStore(patch),
+				resolveFingerprint: async () => closureVersionFromRunnerPath(process.argv[1], { readFile: (path) => {
+					try {
+						return node_fs.readFileSync(path, "utf8");
+					} catch {
+						return null;
+					}
+				} }),
+				emitFailed: (finding) => {
+					this.eventBus?.emit(require_dist.createEvent(require_dist.EventCategory.SystemLivenessFailed, TIMEOUT_GUARD_SOURCE, finding));
+				},
+				emitRecovered: (findingId) => {
+					this.eventBus?.emit(require_dist.createEvent(require_dist.EventCategory.SystemLivenessRecovered, TIMEOUT_GUARD_SOURCE, { findingId }));
+				},
+				nodeId,
+				log: this.log.child("inference-timeout-guard")
+			});
+			this.inferenceTimeoutGuard.start();
+		}
 	}
 	/**
 	* Lazy-install the pip requirements file matching `engine.backend` into
@@ -8623,6 +8900,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	async shutdown() {
 		this.poolMemoryGuard?.stop();
 		this.poolMemoryGuard = null;
+		this.inferenceTimeoutGuard?.stop();
+		this.inferenceTimeoutGuard = null;
 		await this.evictOverrideCache("shutdown");
 		if (this.engineFactory) {
 			await this.engineFactory.dispose();

package/dist/detection-pipeline/index.mjs

@@ -7,6 +7,7 @@ import { n as startEventLoopStallMonitor } from "../event-loop-stall-monitor-DXL
 import { n as pickNodePlatformArch, t as readProcessMemory } from "../process-memory-zLBrzuFc.mjs";
 import * as fs from "node:fs";
 import * as path$1 from "node:path";
+import { dirname, join } from "node:path";
 import { spawn } from "node:child_process";
 import sharp from "sharp";
 import * as os from "node:os";
@@ -862,6 +863,8 @@ var PoolWorker = class {
 	*  Surfaced so "the camera is quiet" and "the camera is saturated" can be
 	*  told apart without reading the logs. */
 	shedCount = 0;
+	/** Requests that hit their client-side deadline (`inference request timed out`). */
+	timedOutCount = 0;
 	/**
 	* Live requests whose deadline fired IN FLIGHT and were answered dropped by
 	* the local watchdog (the worker never replied in time). With a v2 worker
@@ -896,6 +899,9 @@ var PoolWorker = class {
 	getShedCount() {
 		return this.shedCount;
 	}
+	getTimedOutCount() {
+		return this.timedOutCount;
+	}
 	isReady() {
 		return this.ready;
 	}
@@ -1190,6 +1196,7 @@ var PoolWorker = class {
 					});
 					return;
 				}
+				this.timedOutCount++;
 				this.log.error("inference request timed out", {
 					...deviceId !== void 0 ? { tags: { deviceId } } : {},
 					meta: {
@@ -1461,6 +1468,15 @@ var SharedInferencePool = class {
 	getDeadlineExpiredShedCount() {
 		return this.deadlineExpiredShedCount;
 	}
+	/**
+	* Requests that timed out client-side, summed over the live workers. A
+	* worker restart resets its share; the inference-timeout guard treats a
+	* drop as "the new count is the delta". This is the metric 1.2.218's
+	* regression was invisible on for 4 h 19 min (2026-09-04).
+	*/
+	getTimedOutCount() {
+		return this.workers.reduce((sum, w) => sum + w.getTimedOutCount(), 0);
+	}
 	getHandle(modelIndex) {
 		return new PoolHandle(this, modelIndex);
 	}
@@ -2495,12 +2511,14 @@ var EngineFactory = class {
 			inFlight: 0,
 			shed: 0,
 			dropped: 0,
-			deadlineExpired: 0
+			deadlineExpired: 0,
+			timedOut: 0
 		};
 		return {
 			...this.pool.getBacklog(),
 			dropped: this.pool.getDroppedResponseCount(),
-			deadlineExpired: this.pool.getDeadlineExpiredShedCount()
+			deadlineExpired: this.pool.getDeadlineExpiredShedCount(),
+			timedOut: this.pool.getTimedOutCount()
 		};
 	}
 	/** Python-side per-worker memory diagnostics (see SharedInferencePool). */
@@ -2994,6 +3012,235 @@ async function samplePool(pool, log, reportedDead) {
 	};
 }
 //#endregion
+//#region src/detection-pipeline/engine/inference-timeout-guard.ts
+/**
+* Inference-timeout regression guard — "did this update make the GPU time
+* out more than the last one did?", asked by the node itself.
+*
+* ## Why
+*
+* `@camstack/server` 1.2.218 reintroduced the inference timeouts 1.2.216 had
+* fixed: ~3 760 `inference request timed out` an hour for 4 h 19 min (16 215
+* lines, ~19 000 `runInference failed`), on a cluster whose previous hour had
+* seen ~5. Nothing said so. 1.2.219 fixed it again as a side effect of the
+* next deploy, and the regression was found a day later by an audit that
+* counted log lines per version by hand (2026-09-05). A green typecheck, a
+* green suite and a 200 on `/health` all held throughout.
+*
+* ## What it does
+*
+* Every five minutes the guard records how many inference requests timed out
+* on this node under the current FINGERPRINT (the closure version). The last
+* hour of buckets is persisted in the addon store, so a restart hands the
+* previous fingerprint's rate to the next one. Once the new fingerprint has a
+* quarter hour of its own, the two hourly rates are compared:
+*
+*  - ≥ `TIMEOUT_GUARD_RATIO`× worse AND above `TIMEOUT_GUARD_FLOOR_PER_HOUR`
+*    → `system.liveness-failed` (AlertCenter raises a persistent alert keyed
+*    on the finding id) and an ERROR line naming both versions and rates;
+*  - ≥ `TIMEOUT_GUARD_RATIO`× better → an INFO line, no alert;
+*  - the same fingerprint (a plain restart) → nothing to compare.
+*
+* The rate is per hour, not per frame: it is the unit the audit measured and
+* the unit an operator reads. A traffic doubling doubles the rate, which is
+* why the ratio is five and not two.
+*
+* Pure core (`recordTimeoutBucket`, `evaluateTimeoutRegression`) tested in
+* `__tests__/inference-timeout-guard.spec.ts`; the runtime wrapper owns the
+* timer, the store and the emit.
+*/
+var TIMEOUT_GUARD_BUCKET_MS = 5 * 6e4;
+/** Addon-store key the ledger persists under (the addon's own settings blob). */
+var INFERENCE_TIMEOUT_LEDGER_KEY = "_inferenceTimeoutLedger";
+var HOUR_MS = 60 * 6e4;
+function recordTimeoutBucket(ledger, bucket) {
+	const buckets = [...ledger.buckets, bucket];
+	return {
+		version: ledger.version,
+		buckets: buckets.length > 12 ? buckets.slice(buckets.length - 12) : buckets
+	};
+}
+/** Timeouts per hour over the ledger's buckets; `null` under the minimum. */
+function timeoutsPerHour(ledger) {
+	if (ledger === null || ledger.buckets.length < 3) return null;
+	return ledger.buckets.reduce((sum, b) => sum + b.timeouts, 0) / (ledger.buckets.length * TIMEOUT_GUARD_BUCKET_MS) * HOUR_MS;
+}
+function evaluateTimeoutRegression(input) {
+	const { previous, current } = input;
+	const previousPerHour = timeoutsPerHour(previous);
+	const currentPerHour = timeoutsPerHour(current);
+	const base = {
+		previousVersion: previous?.version ?? null,
+		currentVersion: current.version,
+		previousPerHour,
+		currentPerHour
+	};
+	if (previous === null || previous.version === current.version || previousPerHour === null || currentPerHour === null) return {
+		kind: "not-comparable",
+		...base
+	};
+	if (currentPerHour >= 20 && currentPerHour >= Math.max(previousPerHour, 20 / 5) * 5) return {
+		kind: "regressed",
+		...base
+	};
+	if (previousPerHour >= 20 && currentPerHour * 5 <= previousPerHour) return {
+		kind: "improved",
+		...base
+	};
+	return {
+		kind: "quiet",
+		...base
+	};
+}
+/**
+* The closure version, read from the closure the runner was started from.
+*
+* The forked runner script is `<closure>/node_modules/@camstack/system/dist/
+* addon-runner.js` (hub: `/data/server-root/current/…`); the closure's own
+* `@camstack/server/package.json` sits two directories up. This is read from
+* disk rather than asked of `server-management` because that cap is
+* server-provided and the parent refuses to route it for a forked addon —
+* "no provider registered for cap server-management", measured on all
+* three nodes on 2026-09-05. A file the process was started from cannot
+* be refused.
+*/
+function closureVersionFromRunnerPath(runnerScript, fs) {
+	if (runnerScript === void 0 || runnerScript.length === 0) return null;
+	let dir = dirname(runnerScript);
+	for (let depth = 0; depth < 12; depth += 1) {
+		const candidate = join(dir, "node_modules", "@camstack", "server", "package.json");
+		const raw = fs.readFile(candidate);
+		if (raw !== null) {
+			try {
+				const parsed = JSON.parse(raw);
+				const version = typeof parsed === "object" && parsed !== null ? parsed.version : void 0;
+				if (typeof version === "string" && version.length > 0) return `server@${version}`;
+			} catch {
+				return null;
+			}
+			return null;
+		}
+		const parent = dirname(dir);
+		if (parent === dir) break;
+		dir = parent;
+	}
+	return null;
+}
+function readStoredLedgers(blob) {
+	const raw = blob[INFERENCE_TIMEOUT_LEDGER_KEY];
+	if (typeof raw !== "object" || raw === null) return {};
+	const r = raw;
+	return {
+		...isLedger(r.current) ? { current: r.current } : {},
+		...isLedger(r.previous) ? { previous: r.previous } : {}
+	};
+}
+function isLedger(value) {
+	if (typeof value !== "object" || value === null) return false;
+	const v = value;
+	return typeof v.version === "string" && Array.isArray(v.buckets) && v.buckets.every((b) => typeof b === "object" && b !== null && typeof b.at === "number" && typeof b.timeouts === "number");
+}
+function createInferenceTimeoutGuard(deps) {
+	const now = deps.now ?? (() => Date.now());
+	const findingId = `inference-timeout-regression:${deps.nodeId}`;
+	let timer = null;
+	let current = null;
+	let previous = null;
+	let lastTotal = 0;
+	let raised = false;
+	let improvementLogged = false;
+	const ensureLedger = async () => {
+		if (current !== null) return true;
+		const fingerprint = await deps.resolveFingerprint();
+		if (fingerprint === null) return false;
+		const stored = readStoredLedgers(await deps.readStore());
+		if (stored.current !== void 0 && stored.current.version === fingerprint) {
+			current = stored.current;
+			previous = stored.previous ?? null;
+		} else {
+			current = {
+				version: fingerprint,
+				buckets: []
+			};
+			previous = stored.current ?? stored.previous ?? null;
+		}
+		lastTotal = deps.readTotalTimeouts();
+		deps.log.info("inference timeout guard armed", { meta: {
+			fingerprint,
+			previousVersion: previous?.version ?? null,
+			previousPerHour: timeoutsPerHour(previous),
+			bucketMs: TIMEOUT_GUARD_BUCKET_MS
+		} });
+		return true;
+	};
+	const tick = async () => {
+		try {
+			if (!await ensureLedger() || current === null) return;
+			const total = deps.readTotalTimeouts();
+			const delta = total >= lastTotal ? total - lastTotal : total;
+			lastTotal = total;
+			current = recordTimeoutBucket(current, {
+				at: now(),
+				timeouts: delta
+			});
+			await deps.writeStore({ [INFERENCE_TIMEOUT_LEDGER_KEY]: {
+				current,
+				...previous !== null ? { previous } : {}
+			} });
+			const verdict = evaluateTimeoutRegression({
+				previous,
+				current
+			});
+			const meta = {
+				previousVersion: verdict.previousVersion,
+				currentVersion: verdict.currentVersion,
+				previousPerHour: verdict.previousPerHour,
+				currentPerHour: verdict.currentPerHour
+			};
+			if (verdict.kind === "regressed") {
+				if (!raised) {
+					raised = true;
+					const message = `inference requests time out ${Math.round(verdict.currentPerHour ?? 0)}/h under ${verdict.currentVersion}, against ${Math.round(verdict.previousPerHour ?? 0)}/h under ${verdict.previousVersion ?? "unknown"} — the update regressed inference on ${deps.nodeId}`;
+					deps.log.error("inference timeout REGRESSION after update", { meta });
+					deps.emitFailed({
+						findingId,
+						severity: "error",
+						title: "Inference timeouts regressed after update",
+						message
+					});
+				}
+				return;
+			}
+			if (raised && verdict.kind !== "not-comparable") {
+				raised = false;
+				deps.log.info("inference timeout regression cleared", { meta });
+				deps.emitRecovered(findingId);
+			}
+			if (verdict.kind === "improved" && !improvementLogged) {
+				improvementLogged = true;
+				deps.log.info("inference timeouts improved after update", { meta });
+			}
+		} catch (err) {
+			deps.log.warn("inference timeout guard tick failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
+		}
+	};
+	return {
+		start: () => {
+			if (timer !== null) return;
+			timer = setInterval(() => {
+				tick();
+			}, TIMEOUT_GUARD_BUCKET_MS);
+			timer.unref?.();
+			ensureLedger();
+		},
+		tick,
+		stop: () => {
+			if (timer !== null) clearInterval(timer);
+			timer = null;
+		}
+	};
+}
+//#endregion
 //#region src/detection-pipeline/engine-provisioner.ts
 /** Incremental backoff growing to a ~5 min cap; retries indefinitely at cap. */
 var BACKOFF_SCHEDULE_MS = [
@@ -6172,6 +6419,11 @@ function applyZoneRuleGate(result, zones, rules) {
 * This is the main provider that consumers (DetectionWiring, Benchmark, tRPC)
 * interact with. It manages the engine factory, pipeline executor, and config persistence.
 */
+/** Event source of the inference-timeout guard's liveness findings. */
+var TIMEOUT_GUARD_SOURCE = {
+	type: "addon",
+	id: "detection-pipeline"
+};
 var KEY_TEMPLATES = "pipelineTemplates";
 function pythonModuleForBackend(backend) {
 	switch (backend) {
@@ -6599,6 +6851,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	* restart action. Started in `init()`, stopped in `shutdown()`.
 	*/
 	poolMemoryGuard = null;
+	/** Compares this node's inference-timeout rate across closure updates. */
+	inferenceTimeoutGuard = null;
 	/** Watchdog view of the live pools: the node-default pool plus every
 	*  per-device pool. Keys are stable per pool identity so the watchdog's
 	*  baseline tracks one pool across sweeps. */
@@ -6695,6 +6949,30 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 			}, this.log);
 			this.poolMemoryGuard.start();
 		}
+		if (!this.inferenceTimeoutGuard) {
+			const nodeId = this.localProbeNodeId();
+			this.inferenceTimeoutGuard = createInferenceTimeoutGuard({
+				readTotalTimeouts: () => this.listGuardedPools().reduce((sum, pool) => sum + pool.factory.getPoolBacklog().timedOut, 0),
+				readStore: () => this.readStore(),
+				writeStore: (patch) => this.writeStore(patch),
+				resolveFingerprint: async () => closureVersionFromRunnerPath(process.argv[1], { readFile: (path) => {
+					try {
+						return fs.readFileSync(path, "utf8");
+					} catch {
+						return null;
+					}
+				} }),
+				emitFailed: (finding) => {
+					this.eventBus?.emit(createEvent(EventCategory.SystemLivenessFailed, TIMEOUT_GUARD_SOURCE, finding));
+				},
+				emitRecovered: (findingId) => {
+					this.eventBus?.emit(createEvent(EventCategory.SystemLivenessRecovered, TIMEOUT_GUARD_SOURCE, { findingId }));
+				},
+				nodeId,
+				log: this.log.child("inference-timeout-guard")
+			});
+			this.inferenceTimeoutGuard.start();
+		}
 	}
 	/**
 	* Lazy-install the pip requirements file matching `engine.backend` into
@@ -8616,6 +8894,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	async shutdown() {
 		this.poolMemoryGuard?.stop();
 		this.poolMemoryGuard = null;
+		this.inferenceTimeoutGuard?.stop();
+		this.inferenceTimeoutGuard = null;
 		await this.evictOverrideCache("shutdown");
 		if (this.engineFactory) {
 			await this.engineFactory.dispose();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@camstack/addon-pipeline",
-  "version": "1.2.180",
+  "version": "1.2.182",
   "description": "Pipeline bundle — runner, detection, motion, audio + stream broker. Multi-entry npm package shipping pipeline addons under a single bundle.",
   "keywords": [
     "camstack",