@camstack/addon-pipeline 1.2.93 → 1.2.95

package/dist/detection-pipeline/index.mjs

@@ -1,8 +1,8 @@
 import { n as __require } from "../chunk-DnnnRqeS.mjs";
-import { B as enumerateInferenceDevices, Et as array, Ft as union, It as EventCategory, J as pipelineExecutorCapability, L as detectionPipelineCapability, M as defaultDeviceFor$1, Mt as object, Pt as string, V as evaluateZoneRules, at as errMsg, bt as nodePin, d as DEVICE_BACKEND_TO_FORMAT, dt as BaseAddon, et as runtimeDevices$1, gt as hydrateSchema, ht as createEvent, rt as supportedRuntimes$1, t as APPLE_SA_TO_MACRO, w as YAMNET_TO_MACRO, wt as sleep, xt as parseJsonUnknown } from "../dist-6POVIYmC.mjs";
-import { a as getDefaultModelForFormat, c as getStepDefinition, i as ALL_STEPS, l as resolveModelForFormat, n as startEventLoopStallMonitor, o as getDefaultModelForFormatFromDef, r as ALL_PIPELINE_STEPS, s as getStep } from "../event-loop-stall-monitor-C-VvWOxS.mjs";
+import { Ct as nodePin, Dt as sleep, Ft as object, H as evaluateZoneRules, Lt as string, N as defaultDeviceFor$1, R as detectionPipelineCapability, Rt as union, T as YAMNET_TO_MACRO, V as enumerateInferenceDevices, X as pipelineExecutorCapability, ct as errMsg, d as DEVICE_BACKEND_TO_FORMAT, kt as array, mt as BaseAddon, ot as supportedRuntimes$1, rt as runtimeDevices$1, t as APPLE_SA_TO_MACRO, tt as resolvePoolMemoryPolicy, vt as createEvent, wt as parseJsonUnknown, y as PoolMemoryWatchdog, yt as hydrateSchema, zt as EventCategory } from "../dist-C49o1k7u.mjs";
+import { a as getDefaultModelForFormat, c as getStepDefinition, i as ALL_STEPS, l as resolveModelForFormat, n as startEventLoopStallMonitor, o as getDefaultModelForFormatFromDef, r as ALL_PIPELINE_STEPS, s as getStep } from "../event-loop-stall-monitor-CfxZBgNi.mjs";
 import { t as getSharp } from "../lazy-sharp-6oymT_yf.mjs";
-import { t as pickNodePlatformArch } from "../node-topology-platform-BkR_k6WT.mjs";
+import { n as pickNodePlatformArch, t as readProcessMemory } from "../process-memory-DXOB6eHX.mjs";
 import { a as ensureModel, o as isModelDownloaded, r as deleteModelFromDisk } from "../addon-utils-A2S9D7pu.mjs";
 import sharp from "sharp";
 import { spawn } from "node:child_process";
@@ -701,6 +701,14 @@ var PoolWorker = class {
 		const payload = Buffer.from(JSON.stringify(cmd), "utf8");
 		return await this.dispatch(MSG_COMMAND, payload);
 	}
+	/** Command whose reply shape is NOT the load/unload/replace/status envelope
+	*  (e.g. `mem_stats`) — returns the raw JSON record, so no cast is needed
+	*  at the call site. */
+	async sendRawCommand(cmd) {
+		this.ensureReady();
+		const payload = Buffer.from(JSON.stringify(cmd), "utf8");
+		return this.dispatch(MSG_COMMAND, payload);
+	}
 	async dispose() {
 		const proc = this.process;
 		if (!proc) return;
@@ -1065,6 +1073,46 @@ var SharedInferencePool = class {
 		for (const resp of responses) if (resp.status !== "ok") throw new Error(`Failed to replace model at index ${index}: ${resp.error ?? "unknown"}`);
 		return { loadMs: Math.max(...responses.map((r) => r.loadMs ?? 0)) };
 	}
+	/**
+	* Per-worker Python-side memory diagnostics (`mem_stats` command):
+	* RSS/VmSize/peak/threads from /proc/self, getrusage maxrss, glibc malloc
+	* arena count, gc counts, served-request counters, per-model queue depth,
+	* cache entry counts. Queried by the pool memory watchdog once per sweep —
+	* the internals that distinguish "leak" from "working set" and that
+	* /proc/<pid> alone cannot see. A worker that is not ready (or whose
+	* command fails) contributes an `error` record instead of vanishing.
+	*/
+	async getMemStats() {
+		const out = [];
+		for (const [i, w] of this.workers.entries()) {
+			if (!w.isReady()) {
+				out.push({
+					worker: `w${i}`,
+					pid: w.getPid(),
+					stats: null,
+					error: "worker not ready"
+				});
+				continue;
+			}
+			try {
+				const stats = await w.sendRawCommand({ cmd: "mem_stats" });
+				out.push({
+					worker: `w${i}`,
+					pid: w.getPid(),
+					stats,
+					error: null
+				});
+			} catch (err) {
+				out.push({
+					worker: `w${i}`,
+					pid: w.getPid(),
+					stats: null,
+					error: err instanceof Error ? err.message : String(err)
+				});
+			}
+		}
+		return out;
+	}
 	async getStatus() {
 		if (this.workers.length === 0) return [];
 		return (await this.workers[0].sendCommand({ cmd: "status" })).models ?? [];
@@ -1783,6 +1831,29 @@ var EngineFactory = class {
 	getPoolPid() {
 		return this.pool?.getPid() ?? null;
 	}
+	/** All live worker pids of the underlying Python pool (memory watchdog). */
+	getPoolPids() {
+		return (this.pool?.getPids() ?? []).filter((pid) => pid !== null);
+	}
+	/** Summed backlog / shed / dropped across the pool's workers — the cheap
+	*  internals the memory watchdog attaches to its periodic `pool memory`
+	*  line so a leak (RSS up, work flat) is separable from load. */
+	getPoolBacklog() {
+		if (!this.pool) return {
+			inFlight: 0,
+			shed: 0,
+			dropped: 0
+		};
+		return {
+			...this.pool.getBacklog(),
+			dropped: this.pool.getDroppedResponseCount()
+		};
+	}
+	/** Python-side per-worker memory diagnostics (see SharedInferencePool). */
+	async getPoolMemStats() {
+		if (!this.pool) return [];
+		return this.pool.getMemStats();
+	}
 	/**
 	* Whether this factory exposes the batched fast path (`batchInferRaw`).
 	*/
@@ -1992,6 +2063,65 @@ var IdlePoolReaper = class {
 	}
 };
 //#endregion
+//#region src/detection-pipeline/engine/pool-memory-guard.ts
+function createDetectionPoolMemoryGuard(source, logger) {
+	const log = logger.child("pool-memory");
+	return new PoolMemoryWatchdog({
+		policy: resolvePoolMemoryPolicy(process.env),
+		log,
+		restart: (key) => source.restartPool(key),
+		sample: async () => {
+			const out = [];
+			for (const pool of source.listPools()) {
+				const telemetry = await samplePool(pool, log);
+				if (telemetry) out.push(telemetry);
+			}
+			return out;
+		}
+	});
+}
+async function samplePool(pool, log) {
+	const pids = pool.factory.getPoolPids();
+	if (pids.length === 0) return null;
+	let maxRssBytes = 0;
+	const workers = [];
+	for (const pid of pids) {
+		const mem = await readProcessMemory(pid);
+		if (!mem) {
+			log.warn("pool memory sample dropped for worker pid", { meta: {
+				poolKey: pool.key,
+				pid
+			} });
+			continue;
+		}
+		if (mem.rssBytes > maxRssBytes) maxRssBytes = mem.rssBytes;
+		workers.push({
+			pid,
+			rssMb: Math.round(mem.rssBytes / (1024 * 1024)),
+			vmMb: Math.round(mem.vmBytes / (1024 * 1024)),
+			peakRssMb: Math.round(mem.hwmBytes / (1024 * 1024)),
+			swapMb: Math.round(mem.swapBytes / (1024 * 1024)),
+			threads: mem.threads
+		});
+	}
+	if (workers.length === 0) return null;
+	const backlog = pool.factory.getPoolBacklog();
+	const memStats = await pool.factory.getPoolMemStats().catch((err) => err instanceof Error ? err.message : String(err));
+	return {
+		key: pool.key,
+		pids,
+		rssBytes: maxRssBytes,
+		meta: {
+			workers,
+			inFlight: backlog.inFlight,
+			shed: backlog.shed,
+			dropped: backlog.dropped,
+			modelsLoaded: pool.factory.listLoaded().length,
+			memStats
+		}
+	};
+}
+//#endregion
 //#region src/detection-pipeline/engine-provisioner.ts
 /** Incremental backoff growing to a ~5 min cap; retries indefinitely at cap. */
 var BACKOFF_SCHEDULE_MS = [
@@ -4783,6 +4913,55 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		} })
 	});
 	static OVERRIDE_CACHE_TTL_MS = 6e4;
+	/**
+	* RSS bound + periodic memory telemetry for the Python inference pools
+	* (2026-08-18 OOM audit: the pools, not the recorder, were the host's OOM
+	* driver — grow for hours, kernel-killed at 3.8/8.8 GB, respawn, repeat).
+	* Policy, restart budget and the `pool memory` Loki line live in
+	* `@camstack/types` (`PoolMemoryWatchdog`); this provider only enumerates
+	* its pools and exposes its EXISTING dispose-and-recreate machinery as the
+	* restart action. Started in `init()`, stopped in `shutdown()`.
+	*/
+	poolMemoryGuard = null;
+	/** Watchdog view of the live pools: the node-default pool plus every
+	*  per-device pool. Keys are stable per pool identity so the watchdog's
+	*  baseline tracks one pool across sweeps. */
+	listGuardedPools() {
+		const pools = [];
+		if (this.engineFactory) pools.push({
+			key: `default:${deviceKeyOf(this.currentEngine)}`,
+			factory: this.engineFactory
+		});
+		for (const [deviceKey, factory] of this.factoriesByDevice) pools.push({
+			key: deviceKey,
+			factory
+		});
+		return pools;
+	}
+	/**
+	* Watchdog restart action — the same dispose-then-lazy-recreate the idle
+	* reaper and the tuning respawn already use, never a second lifecycle. The
+	* map/field is cleared FIRST so a concurrent dispatch rebuilds fresh via
+	* the single-flight guards instead of grabbing the disposing instance;
+	* models reload on the next dispatch via `ensureModelsForSteps`. Frames
+	* arriving during the seconds-long re-spin are dropped (and counted by the
+	* existing shed/timeout paths), not crashed — a bounded gap, unlike the
+	* kernel's alternative.
+	*/
+	async restartGuardedPool(key) {
+		if (key.startsWith("default:")) {
+			const factory = this.engineFactory;
+			if (!factory) return;
+			this.engineFactory = null;
+			await factory.dispose();
+			return;
+		}
+		const factory = this.factoriesByDevice.get(key);
+		if (!factory) return;
+		this.factoriesByDevice.delete(key);
+		this.deviceReaper.cancel(key);
+		await factory.dispose();
+	}
 	/** Read the addon settings store (all keys). */
 	readStore;
 	/** Write a patch to the addon settings store. */
@@ -4818,6 +4997,13 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 	async init() {
 		this.needsAutoPick = true;
 		this.log.info("Node-default engine derive deferred to setApi()");
+		if (!this.poolMemoryGuard) {
+			this.poolMemoryGuard = createDetectionPoolMemoryGuard({
+				listPools: () => this.listGuardedPools(),
+				restartPool: (key) => this.restartGuardedPool(key)
+			}, this.log);
+			this.poolMemoryGuard.start();
+		}
 	}
 	/**
 	* Lazy-install the pip requirements file matching `engine.backend` into
@@ -6648,6 +6834,8 @@ var DetectionPipelineProvider = class DetectionPipelineProvider {
 		};
 	}
 	async shutdown() {
+		this.poolMemoryGuard?.stop();
+		this.poolMemoryGuard = null;
 		await this.evictOverrideCache("shutdown");
 		if (this.engineFactory) {
 			await this.engineFactory.dispose();