@camstack/addon-pipeline 1.2.94 → 1.2.95

package/dist/detection-pipeline/index.mjs

@@ -1,8 +1,8 @@
 import { n as __require } from "../chunk-DnnnRqeS.mjs";
-import { B as enumerateInferenceDevices, Et as array, Ft as union, It as EventCategory, J as pipelineExecutorCapability, L as detectionPipelineCapability, M as defaultDeviceFor$1, Mt as object, Pt as string, V as evaluateZoneRules, at as errMsg, bt as nodePin, d as DEVICE_BACKEND_TO_FORMAT, dt as BaseAddon, et as runtimeDevices$1, gt as hydrateSchema, ht as createEvent, rt as supportedRuntimes$1, t as APPLE_SA_TO_MACRO, w as YAMNET_TO_MACRO, wt as sleep, xt as parseJsonUnknown } from "../dist-XbrYiMV3.mjs";
-import { a as getDefaultModelForFormat, c as getStepDefinition, i as ALL_STEPS, l as resolveModelForFormat, n as startEventLoopStallMonitor, o as getDefaultModelForFormatFromDef, r as ALL_PIPELINE_STEPS, s as getStep } from "../event-loop-stall-monitor-CFWrOZ6G.mjs";
+import { Ct as nodePin, Dt as sleep, Ft as object, H as evaluateZoneRules, Lt as string, N as defaultDeviceFor$1, R as detectionPipelineCapability, Rt as union, T as YAMNET_TO_MACRO, V as enumerateInferenceDevices, X as pipelineExecutorCapability, ct as errMsg, d as DEVICE_BACKEND_TO_FORMAT, kt as array, mt as BaseAddon, ot as supportedRuntimes$1, rt as runtimeDevices$1, t as APPLE_SA_TO_MACRO, tt as resolvePoolMemoryPolicy, vt as createEvent, wt as parseJsonUnknown, y as PoolMemoryWatchdog, yt as hydrateSchema, zt as EventCategory } from "../dist-C49o1k7u.mjs";
+import { a as getDefaultModelForFormat, c as getStepDefinition, i as ALL_STEPS, l as resolveModelForFormat, n as startEventLoopStallMonitor, o as getDefaultModelForFormatFromDef, r as ALL_PIPELINE_STEPS, s as getStep } from "../event-loop-stall-monitor-CfxZBgNi.mjs";
 import { t as getSharp } from "../lazy-sharp-6oymT_yf.mjs";
-import { t as pickNodePlatformArch } from "../node-topology-platform-BkR_k6WT.mjs";
+import { n as pickNodePlatformArch, t as readProcessMemory } from "../process-memory-DXOB6eHX.mjs";
 import { a as ensureModel, o as isModelDownloaded, r as deleteModelFromDisk } from "../addon-utils-A2S9D7pu.mjs";
 import sharp from "sharp";
 import { spawn } from "node:child_process";
@@ -701,6 +701,14 @@ var PoolWorker = class {
 		const payload = Buffer.from(JSON.stringify(cmd), "utf8");
 		return await this.dispatch(MSG_COMMAND, payload);
 	}
+	/** Command whose reply shape is NOT the load/unload/replace/status envelope
+	*  (e.g. `mem_stats`) — returns the raw JSON record, so no cast is needed
+	*  at the call site. */
+	async sendRawCommand(cmd) {
+		this.ensureReady();
+		const payload = Buffer.from(JSON.stringify(cmd), "utf8");
+		return this.dispatch(MSG_COMMAND, payload);
+	}
 	async dispose() {
 		const proc = this.process;
 		if (!proc) return;
@@ -1065,6 +1073,46 @@ var SharedInferencePool = class {
 		for (const resp of responses) if (resp.status !== "ok") throw new Error(`Failed to replace model at index ${index}: ${resp.error ?? "unknown"}`);
 		return { loadMs: Math.max(...responses.map((r) => r.loadMs ?? 0)) };
 	}
+	/**
+	* Per-worker Python-side memory diagnostics (`mem_stats` command):
+	* RSS/VmSize/peak/threads from /proc/self, getrusage maxrss, glibc malloc
+	* arena count, gc counts, served-request counters, per-model queue depth,
+	* cache entry counts. Queried by the pool memory watchdog once per sweep —
+	* the internals that distinguish "leak" from "working set" and that
+	* /proc/<pid> alone cannot see. A worker that is not ready (or whose
+	* command fails) contributes an `error` record instead of vanishing.
+	*/
+	async getMemStats() {
+		const out = [];
+		for (const [i, w] of this.workers.entries()) {
+			if (!w.isReady()) {
+				out.push({
+					worker: `w${i}`,
+					pid: w.getPid(),
+					stats: null,
+					error: "worker not ready"
+				});
+				continue;
+			}
+			try {
+				const stats = await w.sendRawCommand({ cmd: "mem_stats" });
+				out.push({
+					worker: `w${i}`,
+					pid: w.getPid(),
+					stats,
+					error: null
+				});
+			} catch (err) {
+				out.push({
+					worker: `w${i}`,
+					pid: w.getPid(),
+					stats: null,
+					error: err instanceof Error ? err.message : String(err)
+				});
+			}
+		}
+		return out;
+	}
 	async getStatus() {
 		if (this.workers.length === 0) return [];
 		return (await this.workers[0].sendCommand({ cmd: "status" })).models ?? [];
@@ -1783,6 +1831,29 @@ var EngineFactory = class {
 	getPoolPid() {
 		return this.pool?.getPid() ?? null;
 	}
+	/** All live worker pids of the underlying Python pool (memory watchdog). */
+	getPoolPids() {
+		return (this.pool?.getPids() ?? []).filter((pid) => pid !== null);
+	}
+	/** Summed backlog / shed / dropped across the pool's workers — the cheap
+	*  internals the memory watchdog attaches to its periodic `pool memory`
+	*  line so a leak (RSS up, work flat) is separable from load. */
+	getPoolBacklog() {
+		if (!this.pool) return {
+			inFlight: 0,
+			shed: 0,
+			dropped: 0
+		};
+		return {
+			...this.pool.getBacklog(),
+			dropped: this.pool.getDroppedResponseCount()
+		};
+	}
+	/** Python-side per-worker memory diagnostics (see SharedInferencePool). */
+	async getPoolMemStats() {
+		if (!this.pool) return [];
+		return this.pool.getMemStats();
+	}
 	/**
 	* Whether this factory exposes the batched fast path (`batchInferRaw`).
 	*/
@@ -1992,6 +2063,65 @@ var IdlePoolReaper = class {
 	}
 };
 //#endregion
+//#region src/detection-pipeline/engine/pool-memory-guard.ts
+function createDetectionPoolMemoryGuard(source, logger) {
+	const log = logger.child("pool-memory");
+	return new PoolMemoryWatchdog({
+		policy: resolvePoolMemoryPolicy(process.env),
+		log,
+		restart: (key) => source.restartPool(key),
+		sample: async () => {
+			const out = [];
+			for (const pool of source.listPools()) {
+				const telemetry = await samplePool(pool, log);
+				if (telemetry) out.push(telemetry);
+			}
+			return out;
+		}
+	});
+}
+async function samplePool(pool, log) {
+	const pids = pool.factory.getPoolPids();
+	if (pids.length === 0) return null;
+	let maxRssBytes = 0;
+	const workers = [];
+	for (const pid of pids) {
+		const mem = await readProcessMemory(pid);
+		if (!mem) {
+			log.warn("pool memory sample dropped for worker pid", { meta: {
+				poolKey: pool.key,
+				pid
+			} });
+			continue;
+		}
+		if (mem.rssBytes > maxRssBytes) maxRssBytes = mem.rssBytes;
+		workers.push({
+			pid,
+			rssMb: Math.round(mem.rssBytes / (1024 * 1024)),
+			vmMb: Math.round(mem.vmBytes / (1024 * 1024)),
+			peakRssMb: Math.round(mem.hwmBytes / (1024 * 1024)),
+			swapMb: Math.round(mem.swapBytes / (1024 * 1024)),
+			threads: mem.threads
+		});
+	}
+	if (workers.length === 0) return null;
+	const backlog = pool.factory.getPoolBacklog();
+	const memStats = await pool.factory.getPoolMemStats().catch((err) => err instanceof Error ? err.message : String(err));
+	return {
+		key: pool.key,
+		pids,
+		rssBytes: maxRssBytes,
+		meta: {
+			workers,
+			inFlight: backlog.inFlight,
+			shed: backlog.shed,
+			dropped: backlog.dropped,
+			modelsLoaded: pool.factory.listLoaded().length,
+			memStats
+		}
+	};
+}
+//#endregion
 //#region src/detection-pipeline/engine-provisioner.ts
 /** Incremental backoff growing to a ~5 min cap; retries indefinitely at cap. */
 var BACKOFF_SCHEDULE_MS = [
@@ -4783,6 +4913,55 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		} })
 	});
 	static OVERRIDE_CACHE_TTL_MS = 6e4;
+	/**
+	* RSS bound + periodic memory telemetry for the Python inference pools
+	* (2026-08-18 OOM audit: the pools, not the recorder, were the host's OOM
+	* driver — grow for hours, kernel-killed at 3.8/8.8 GB, respawn, repeat).
+	* Policy, restart budget and the `pool memory` Loki line live in
+	* `@camstack/types` (`PoolMemoryWatchdog`); this provider only enumerates
+	* its pools and exposes its EXISTING dispose-and-recreate machinery as the
+	* restart action. Started in `init()`, stopped in `shutdown()`.
+	*/
+	poolMemoryGuard = null;
+	/** Watchdog view of the live pools: the node-default pool plus every
+	*  per-device pool. Keys are stable per pool identity so the watchdog's
+	*  baseline tracks one pool across sweeps. */
+	listGuardedPools() {
+		const pools = [];
+		if (this.engineFactory) pools.push({
+			key: `default:${deviceKeyOf(this.currentEngine)}`,
+			factory: this.engineFactory
+		});
+		for (const [deviceKey, factory] of this.factoriesByDevice) pools.push({
+			key: deviceKey,
+			factory
+		});
+		return pools;
+	}
+	/**
+	* Watchdog restart action — the same dispose-then-lazy-recreate the idle
+	* reaper and the tuning respawn already use, never a second lifecycle. The
+	* map/field is cleared FIRST so a concurrent dispatch rebuilds fresh via
+	* the single-flight guards instead of grabbing the disposing instance;
+	* models reload on the next dispatch via `ensureModelsForSteps`. Frames
+	* arriving during the seconds-long re-spin are dropped (and counted by the
+	* existing shed/timeout paths), not crashed — a bounded gap, unlike the
+	* kernel's alternative.
+	*/
+	async restartGuardedPool(key) {
+		if (key.startsWith("default:")) {
+			const factory = this.engineFactory;
+			if (!factory) return;
+			this.engineFactory = null;
+			await factory.dispose();
+			return;
+		}
+		const factory = this.factoriesByDevice.get(key);
+		if (!factory) return;
+		this.factoriesByDevice.delete(key);
+		this.deviceReaper.cancel(key);
+		await factory.dispose();
+	}
 	/** Read the addon settings store (all keys). */
 	readStore;
 	/** Write a patch to the addon settings store. */
@@ -4818,6 +4997,13 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	async init() {
 		this.needsAutoPick = true;
 		this.log.info("Node-default engine derive deferred to setApi()");
+		if (!this.poolMemoryGuard) {
+			this.poolMemoryGuard = createDetectionPoolMemoryGuard({
+				listPools: () => this.listGuardedPools(),
+				restartPool: (key) => this.restartGuardedPool(key)
+			}, this.log);
+			this.poolMemoryGuard.start();
+		}
 	}
 	/**
 	* Lazy-install the pip requirements file matching `engine.backend` into
@@ -6648,6 +6834,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		};
 	}
 	async shutdown() {
+		this.poolMemoryGuard?.stop();
+		this.poolMemoryGuard = null;
 		await this.evictOverrideCache("shutdown");
 		if (this.engineFactory) {
 			await this.engineFactory.dispose();

package/dist/{dist-XbrYiMV3.mjs → dist-C49o1k7u.mjs}

@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto";
-//#region ../types/dist/event-category-C0lyLd5U.mjs
+//#region ../types/dist/event-category-XfKNtfCc.mjs
 var EventCategory = /* @__PURE__ */ function(EventCategory) {
 	EventCategory["SystemBoot"] = "system.boot";
 	EventCategory["SystemAddonsReady"] = "system.addons-ready";
@@ -38,6 +38,22 @@ var EventCategory = /* @__PURE__ */ function(EventCategory) {
 	EventCategory["AddonInstalled"] = "addon.installed";
 	EventCategory["AddonUninstalled"] = "addon.uninstalled";
 	EventCategory["AddonCrashed"] = "addon.crashed";
+	/**
+	* A RUNNER the D6 crash circuit-breaker gave up on — terminal.
+	*
+	* `AddonCrashed` is the routine, self-healing fact ("it crashed; it is being
+	* respawned"); this is the one that never resolves itself. Emitted exactly
+	* once per trip, by `process-service.ts`, carrying the node, the runner, the
+	* addons it hosted and the crash count that tripped the breaker.
+	*
+	* It exists because a runner marked terminally `failed` used to be SILENT:
+	* on 2026-08-18 the `recorder` runner died of three unhandled ffmpeg spawn
+	* errors, the breaker stopped respawning it (correctly), `/health` kept
+	* returning 200, and fleet-wide recording was gone for 3h15 before the
+	* operator's phone told him. The Notification Center's system-event intake
+	* consumes this category and maps it to the `addon-crash-loop` kind.
+	*/
+	EventCategory["AddonRunnerFailed"] = "addon.runner-failed";
 	EventCategory["AddonError"] = "addon.error";
 	EventCategory["AddonPageReady"] = "addon.page-ready";
 	EventCategory["AddonWidgetReady"] = "addon.widget-ready";
@@ -16071,6 +16087,7 @@ var NcSystemEventKindSchema = _enum([
 	"node-offline",
 	"node-inference-unavailable",
 	"detection-blind",
+	"addon-crash-loop",
 	"addon-update-available",
 	"server-update-available",
 	"alarm-triggered",
@@ -36425,6 +36442,303 @@ function maskUrlCredentials(rawUrl) {
 		return rawUrl;
 	}
 }
+var MB = 1024 * 1024;
+var DEFAULT_POOL_MEMORY_POLICY = {
+	sampleIntervalMs: 6e4,
+	baselineSettleSamples: 5,
+	baselineSampleCount: 3,
+	restartMultiple: 4,
+	floorBytes: 1024 * MB,
+	ceilingBytes: 3072 * MB,
+	cooldownMs: 30 * 6e4,
+	maxRestartsPerWindow: 6,
+	restartWindowMs: 1440 * 6e4
+};
+/**
+* Resolve the policy from env overrides (`CAMSTACK_POOL_MEM_*`). Garbage or
+* absent values keep the default — an operator typo must never disable the
+* bound or set it to zero.
+*/
+function resolvePoolMemoryPolicy(env) {
+	const num = (key, fallback, min) => {
+		const raw = env[key];
+		if (raw === void 0) return fallback;
+		const parsed = Number(raw);
+		return Number.isFinite(parsed) && parsed >= min ? parsed : fallback;
+	};
+	const d = DEFAULT_POOL_MEMORY_POLICY;
+	return {
+		sampleIntervalMs: num("CAMSTACK_POOL_MEM_INTERVAL_MS", d.sampleIntervalMs, 5e3),
+		baselineSettleSamples: d.baselineSettleSamples,
+		baselineSampleCount: d.baselineSampleCount,
+		restartMultiple: num("CAMSTACK_POOL_MEM_MULTIPLE", d.restartMultiple, 1.5),
+		floorBytes: num("CAMSTACK_POOL_MEM_FLOOR_MB", d.floorBytes / MB, 128) * MB,
+		ceilingBytes: num("CAMSTACK_POOL_MEM_CEILING_MB", d.ceilingBytes / MB, 256) * MB,
+		cooldownMs: num("CAMSTACK_POOL_MEM_COOLDOWN_MS", d.cooldownMs, 6e4),
+		maxRestartsPerWindow: num("CAMSTACK_POOL_MEM_MAX_RESTARTS", d.maxRestartsPerWindow, 1),
+		restartWindowMs: num("CAMSTACK_POOL_MEM_RESTART_WINDOW_MS", d.restartWindowMs, 6e4)
+	};
+}
+/** Parse the fields this watchdog needs out of `/proc/<pid>/status` text.
+*  Returns null when VmRSS is missing (dead pid, kernel thread, bad read). */
+function parseProcStatus(text) {
+	const kb = (label) => {
+		const match = text.match(new RegExp(`^${label}:\\s+(\\d+)\\s*kB`, "m"));
+		return match ? Number(match[1]) * 1024 : null;
+	};
+	const rssBytes = kb("VmRSS");
+	if (rssBytes === null) return null;
+	const threadsMatch = text.match(/^Threads:\s+(\d+)/m);
+	return {
+		rssBytes,
+		vmBytes: kb("VmSize") ?? 0,
+		hwmBytes: kb("VmHWM") ?? 0,
+		swapBytes: kb("VmSwap") ?? 0,
+		threads: threadsMatch ? Number(threadsMatch[1]) : 0
+	};
+}
+function initialPoolMemoryState() {
+	return {
+		settleSeen: 0,
+		baselineWindow: [],
+		baselineBytes: null,
+		restartsAt: [],
+		lastRestartAt: null
+	};
+}
+function median(values) {
+	const sorted = [...values].sort((a, b) => a - b);
+	return sorted[Math.floor(sorted.length / 2)];
+}
+function advanceBaseline(state, rssBytes, policy) {
+	if (state.baselineBytes !== null) return state;
+	if (state.settleSeen < policy.baselineSettleSamples) return {
+		...state,
+		settleSeen: state.settleSeen + 1
+	};
+	const window = [...state.baselineWindow, rssBytes];
+	if (window.length < policy.baselineSampleCount) return {
+		...state,
+		baselineWindow: window
+	};
+	return {
+		...state,
+		baselineWindow: window,
+		baselineBytes: median(window)
+	};
+}
+/** The one place a pool's restart threshold is computed. */
+function poolMemoryThreshold(baselineBytes, policy) {
+	if (baselineBytes === null) return policy.ceilingBytes;
+	return Math.min(policy.ceilingBytes, Math.max(policy.floorBytes, policy.restartMultiple * baselineBytes));
+}
+function pruneRestarts(restartsAt, nowMs, policy) {
+	return restartsAt.filter((t) => nowMs - t < policy.restartWindowMs);
+}
+/**
+* Evaluate one RSS sample. Pure: returns the successor state plus the verdict.
+* The caller performs (and logs) the restart, then commits it via
+* {@link commitWatchdogRestart}.
+*/
+function evaluatePoolMemory(state, rssBytes, nowMs, policy) {
+	const next = advanceBaseline(state, rssBytes, policy);
+	const thresholdBytes = poolMemoryThreshold(next.baselineBytes, policy);
+	if (rssBytes <= thresholdBytes) return {
+		state: next,
+		action: next.baselineBytes === null ? "baseline-pending" : "ok",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+	const inWindow = pruneRestarts(next.restartsAt, nowMs, policy);
+	const pruned = {
+		...next,
+		restartsAt: inWindow
+	};
+	if (inWindow.length >= policy.maxRestartsPerWindow) return {
+		state: pruned,
+		action: "exhausted",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+	if (pruned.lastRestartAt !== null && nowMs - pruned.lastRestartAt < policy.cooldownMs) return {
+		state: pruned,
+		action: "cooldown",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+	return {
+		state: pruned,
+		action: "restart",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+}
+/** Record a performed watchdog restart: budget consumed, baseline reset —
+*  the recreated pool is a NEW process and gets a fresh settle window. */
+function commitWatchdogRestart(state, nowMs) {
+	return {
+		...initialPoolMemoryState(),
+		restartsAt: [...state.restartsAt, nowMs],
+		lastRestartAt: nowMs
+	};
+}
+/** Reset baseline tracking (pool process replaced OUTSIDE the watchdog —
+*  kernel OOM kill + respawn, idle-reaper eviction, tuning respawn). The
+*  restart budget is kept: it bounds the WATCHDOG, not the pool. */
+function resetPoolBaseline(state) {
+	return {
+		...initialPoolMemoryState(),
+		restartsAt: state.restartsAt,
+		lastRestartAt: state.lastRestartAt
+	};
+}
+/**
+* Pick ONE pool to restart this sweep — the worst overage ratio. Restarting
+* several pools at once (e.g. after a model rollout grew every baseline)
+* would take detection down on every camera simultaneously; one per sweep
+* means the survivors keep serving while the worst offender reloads, and the
+* next sweep (one interval later) takes the next one.
+*/
+function pickRestartCandidate(candidates) {
+	if (candidates.length === 0) return null;
+	let worst = candidates[0];
+	for (const c of candidates) if (c.rssBytes / c.thresholdBytes > worst.rssBytes / worst.thresholdBytes) worst = c;
+	return worst.key;
+}
+var PoolMemoryWatchdog = class {
+	pools = /* @__PURE__ */ new Map();
+	timer = null;
+	sweeping = false;
+	stopped = false;
+	opts;
+	now;
+	setTimer;
+	clearTimer;
+	constructor(opts) {
+		this.opts = opts;
+		this.now = opts.now ?? (() => Date.now());
+		this.setTimer = opts.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
+		this.clearTimer = opts.clearTimer ?? ((t) => clearTimeout(t));
+	}
+	start() {
+		if (this.timer || this.stopped) return;
+		this.arm();
+	}
+	stop() {
+		this.stopped = true;
+		if (this.timer) {
+			this.clearTimer(this.timer);
+			this.timer = null;
+		}
+	}
+	arm() {
+		const timer = this.setTimer(() => {
+			this.sweep().finally(() => {
+				if (!this.stopped) this.arm();
+			});
+		}, this.opts.policy.sampleIntervalMs);
+		if (typeof timer.unref === "function") timer.unref();
+		this.timer = timer;
+	}
+	/** One sweep: sample → log every pool → restart at most one. Public so the
+	*  loop is testable without fake global timers. */
+	async sweep() {
+		if (this.sweeping) return;
+		this.sweeping = true;
+		try {
+			await this.doSweep();
+		} catch (err) {
+			this.opts.log.warn("pool memory sweep failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
+		} finally {
+			this.sweeping = false;
+		}
+	}
+	async doSweep() {
+		const nowMs = this.now();
+		const samples = await this.opts.sample();
+		const candidates = [];
+		const verdictByKey = /* @__PURE__ */ new Map();
+		for (const t of samples) {
+			const tracked = this.pools.get(t.key);
+			let state = tracked?.state ?? initialPoolMemoryState();
+			if (tracked && tracked.pids.length > 0 && !samePids(tracked.pids, t.pids)) {
+				this.opts.log.warn("pool process changed outside the watchdog — re-baselining", { meta: {
+					poolKey: t.key,
+					previousPids: tracked.pids,
+					pids: t.pids
+				} });
+				state = resetPoolBaseline(state);
+			}
+			const verdict = evaluatePoolMemory(state, t.rssBytes, nowMs, this.opts.policy);
+			this.pools.set(t.key, {
+				state: verdict.state,
+				pids: t.pids
+			});
+			verdictByKey.set(t.key, verdict);
+			this.opts.log.info("pool memory", { meta: {
+				poolKey: t.key,
+				pids: t.pids,
+				rssMb: Math.round(t.rssBytes / MB),
+				baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
+				thresholdMb: Math.round(verdict.thresholdBytes / MB),
+				action: verdict.action,
+				...t.meta
+			} });
+			if (verdict.action === "restart") candidates.push({
+				key: t.key,
+				rssBytes: t.rssBytes,
+				thresholdBytes: verdict.thresholdBytes
+			});
+			else if (verdict.action === "cooldown") this.opts.log.warn("pool over memory threshold but inside restart cooldown", { meta: {
+				poolKey: t.key,
+				rssMb: Math.round(t.rssBytes / MB),
+				thresholdMb: Math.round(verdict.thresholdBytes / MB),
+				cooldownMs: this.opts.policy.cooldownMs
+			} });
+			else if (verdict.action === "exhausted") this.opts.log.error("pool memory watchdog gave up — restart budget exhausted, pool leaks faster than restarts can pay for; needs a heap profile (py-spy / tracemalloc)", { meta: {
+				poolKey: t.key,
+				rssMb: Math.round(t.rssBytes / MB),
+				thresholdMb: Math.round(verdict.thresholdBytes / MB),
+				maxRestartsPerWindow: this.opts.policy.maxRestartsPerWindow,
+				restartWindowMs: this.opts.policy.restartWindowMs
+			} });
+		}
+		for (const key of [...this.pools.keys()]) if (!samples.some((t) => t.key === key)) this.pools.delete(key);
+		const pickedKey = pickRestartCandidate(candidates);
+		if (pickedKey === null) return;
+		const picked = candidates.find((c) => c.key === pickedKey);
+		const verdict = verdictByKey.get(pickedKey);
+		const tracked = this.pools.get(pickedKey);
+		const restartsInWindow = tracked.state.restartsAt.length + 1;
+		this.opts.log.warn("pool memory watchdog restarting pool — RSS over threshold", { meta: {
+			poolKey: pickedKey,
+			pids: tracked.pids,
+			rssMb: Math.round(picked.rssBytes / MB),
+			thresholdMb: Math.round(picked.thresholdBytes / MB),
+			baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
+			restartsInWindow,
+			deferredCandidates: candidates.filter((c) => c.key !== pickedKey).map((c) => c.key)
+		} });
+		try {
+			await this.opts.restart(pickedKey);
+			this.pools.set(pickedKey, {
+				state: commitWatchdogRestart(tracked.state, this.now()),
+				pids: []
+			});
+		} catch (err) {
+			this.opts.log.error("pool memory watchdog restart failed", { meta: {
+				poolKey: pickedKey,
+				error: err instanceof Error ? err.message : String(err)
+			} });
+		}
+	}
+};
+function samePids(a, b) {
+	if (a.length !== b.length) return false;
+	const sortedA = [...a].sort((x, y) => x - y);
+	const sortedB = [...b].sort((x, y) => x - y);
+	return sortedA.every((v, i) => v === sortedB[i]);
+}
 /**
 * Fixed-capacity ring buffer. When full, push() overwrites the oldest entry.
 * drain() returns up to maxCount items in FIFO order and removes them.
@@ -36762,4 +37076,4 @@ function enumerateInferenceDevices(hw) {
 	return out;
 }
 //#endregion
-export { resolveRecordingProfiles as $, createHwAccelCache as A, literal as At, enumerateInferenceDevices as B, RingBuffer as C, selectAssignedProfileSlots as Ct, audioAnalyzerCapability as D, boolean as Dt, audioAnalysisCapability as E, array as Et, deriveDetailCropRect as F, union as Ft, motionDetectionCapability as G, hfModelUrl as H, deriveRecordingMode as I, EventCategory as It, pipelineExecutorCapability as J, pickDetailCropConvention as K, detectionPipelineCapability as L, defaultDeviceFor as M, object as Mt, defineCustomActions as N, record as Nt, batteryCapability as O, discriminatedUnion as Ot, deriveBatteryPresence as P, string as Pt, resolveEgressDecodeHwAccel as Q, egressTranscodeSharingKey as R, RecordingConfigSchema as S, parseProfileBrokerId as St, addonWidgetsSourceCapability as T, _enum as Tt, mapAudioLabelToMacro as U, evaluateZoneRules as V, maskUrlCredentials as W, recordingCapability as X, pipelineRunnerCapability as Y, recordingExportCapability as Z, NativeLeaseSettingsSchema as _, isEvent as _t, COCO_80_LABELS as a, errMsg as at, RATE_CONTROL_TIGHT as b, nodePin as bt, DEFAULT_DETAIL_CROP_CONVENTION as c, buildFfmpegArgs as ct, DEVICE_BACKEND_TO_FORMAT as d, BaseAddon as dt, runtimeDevices as et, EVENT_PAD_MS as f, CAM_PROFILE_ORDER as ft, NativeLeaseAdmissionSchema as g, hydrateSchema as gt, HF_BASE_URL as h, createEvent as ht, BatteryStatusSchema as i, webrtcSessionCapability as it, customAction as j, number as jt, cameraStreamsCapability as k, lazy as kt, DEFAULT_EVENTS_BAND_BUFFER_SEC as l, invocationFromEncodeProfile as lt, ExportRecordSchema as m, DeviceType as mt, AUDIO_BACKEND_CHOICES as n, streamBrokerCapability as nt, COCO_TO_MACRO as o, AUDIO_PRESETS as ot, EncodeProfileSchema as p, DeviceFeature as pt, pickNativeLeaseOverride as q, AUDIO_MACRO_LABELS as r, supportedRuntimes as rt, DEFAULT_AUDIO_ANALYZER_CONFIG as s, Fmp4BoxSplitter as st, APPLE_SA_TO_MACRO as t, storageEvictableCapability as tt, DEFAULT_NATIVE_LEASE_SETTINGS as u, isSoftwareDecode as ut, OpsLogEntrySchema as v, makeProfileBrokerId as vt, YAMNET_TO_MACRO as w, sleep as wt, RECORDING_EXPORT_MAX_READ_BYTES as x, parseJsonUnknown as xt, RATE_CONTROL_RELAXED as y, makeSourceBrokerId as yt, egressTransportFromRequest as z };
+export { recordingExportCapability as $, cameraStreamsCapability as A, boolean as At, egressTransportFromRequest as B, RecordingConfigSchema as C, nodePin as Ct, audioAnalysisCapability as D, sleep as Dt, addonWidgetsSourceCapability as E, selectAssignedProfileSlots as Et, deriveBatteryPresence as F, object as Ft, maskUrlCredentials as G, evaluateZoneRules as H, deriveDetailCropRect as I, record as It, pickDetailCropConvention as J, motionDetectionCapability as K, deriveRecordingMode as L, string as Lt, customAction as M, lazy as Mt, defaultDeviceFor as N, literal as Nt, audioAnalyzerCapability as O, _enum as Ot, defineCustomActions as P, number as Pt, recordingCapability as Q, detectionPipelineCapability as R, union as Rt, RECORDING_EXPORT_MAX_READ_BYTES as S, makeSourceBrokerId as St, YAMNET_TO_MACRO as T, parseProfileBrokerId as Tt, hfModelUrl as U, enumerateInferenceDevices as V, mapAudioLabelToMacro as W, pipelineExecutorCapability as X, pickNativeLeaseOverride as Y, pipelineRunnerCapability as Z, NativeLeaseSettingsSchema as _, DeviceType as _t, COCO_80_LABELS as a, streamBrokerCapability as at, RATE_CONTROL_RELAXED as b, isEvent as bt, DEFAULT_DETAIL_CROP_CONVENTION as c, errMsg as ct, DEVICE_BACKEND_TO_FORMAT as d, buildFfmpegArgs as dt, resolveEgressDecodeHwAccel as et, EVENT_PAD_MS as f, invocationFromEncodeProfile as ft, NativeLeaseAdmissionSchema as g, DeviceFeature as gt, HF_BASE_URL as h, CAM_PROFILE_ORDER as ht, BatteryStatusSchema as i, storageEvictableCapability as it, createHwAccelCache as j, discriminatedUnion as jt, batteryCapability as k, array as kt, DEFAULT_EVENTS_BAND_BUFFER_SEC as l, AUDIO_PRESETS as lt, ExportRecordSchema as m, BaseAddon as mt, AUDIO_BACKEND_CHOICES as n, resolveRecordingProfiles as nt, COCO_TO_MACRO as o, supportedRuntimes as ot, EncodeProfileSchema as p, isSoftwareDecode as pt, parseProcStatus as q, AUDIO_MACRO_LABELS as r, runtimeDevices as rt, DEFAULT_AUDIO_ANALYZER_CONFIG as s, webrtcSessionCapability as st, APPLE_SA_TO_MACRO as t, resolvePoolMemoryPolicy as tt, DEFAULT_NATIVE_LEASE_SETTINGS as u, Fmp4BoxSplitter as ut, OpsLogEntrySchema as v, createEvent as vt, RingBuffer as w, parseJsonUnknown as wt, RATE_CONTROL_TIGHT as x, makeProfileBrokerId as xt, PoolMemoryWatchdog as y, hydrateSchema as yt, egressTranscodeSharingKey as z, EventCategory as zt };