@camstack/types 1.2.86 → 1.2.87

package/dist/index.js

@@ -1,6 +1,6 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_event_category = require("./event-category-DBHdQVIy.js");
-const require_sleep = require("./sleep-DZAjGv1f.js");
+const require_event_category = require("./event-category-CRPORAAz.js");
+const require_sleep = require("./sleep-CMRLJj2e.js");
 const require_canonical_hash = require("./canonical-hash-DNV8S5ET.js");
 const require_enums = require("./enums.js");
 const require_err_msg = require("./err-msg-COpsHMw2.js");
@@ -12252,6 +12252,7 @@ var NcSystemEventKindSchema = zod.z.enum([
 	"node-offline",
 	"node-inference-unavailable",
 	"detection-blind",
+	"addon-crash-loop",
 	"addon-update-available",
 	"server-update-available",
 	"alarm-triggered",
@@ -13403,6 +13404,10 @@ var NC_CONDITION_CATALOG = [
 				value: "detection-blind",
 				label: "Camera detecting nothing"
 			},
+			{
+				value: "addon-crash-loop",
+				label: "Addon stopped after repeated crashes"
+			},
 			{
 				value: "addon-update-available",
 				label: "Addon update available"
@@ -43625,7 +43630,7 @@ function systemEventFilterApplies(kind, filter) {
 	switch (filter) {
 		case "deviceIds": return kind.startsWith("device-") || kind.startsWith("stream-") || kind === "detection-blind" || kind === "alarm-triggered" || kind === "export-completed";
 		case "deviceTypes": return kind.startsWith("device-") || kind === "detection-blind";
-		case "nodeIds": return kind.startsWith("node-") || kind === "addon-update-available" || kind === "addon-updated" || kind === "server-update-available" || kind === "server-updated";
+		case "nodeIds": return kind.startsWith("node-") || kind === "addon-crash-loop" || kind === "addon-update-available" || kind === "addon-updated" || kind === "server-update-available" || kind === "server-updated";
 		case "packageNames": return kind.endsWith("update-available") || kind === "addon-updated" || kind === "server-updated";
 	}
 }
@@ -45286,6 +45291,305 @@ function maskUrlCredentials(rawUrl) {
 	}
 }
 //#endregion
+//#region src/utils/pool-memory-watchdog.ts
+var MB = 1024 * 1024;
+var DEFAULT_POOL_MEMORY_POLICY = {
+	sampleIntervalMs: 6e4,
+	baselineSettleSamples: 5,
+	baselineSampleCount: 3,
+	restartMultiple: 4,
+	floorBytes: 1024 * MB,
+	ceilingBytes: 3072 * MB,
+	cooldownMs: 30 * 6e4,
+	maxRestartsPerWindow: 6,
+	restartWindowMs: 1440 * 6e4
+};
+/**
+* Resolve the policy from env overrides (`CAMSTACK_POOL_MEM_*`). Garbage or
+* absent values keep the default — an operator typo must never disable the
+* bound or set it to zero.
+*/
+function resolvePoolMemoryPolicy(env) {
+	const num = (key, fallback, min) => {
+		const raw = env[key];
+		if (raw === void 0) return fallback;
+		const parsed = Number(raw);
+		return Number.isFinite(parsed) && parsed >= min ? parsed : fallback;
+	};
+	const d = DEFAULT_POOL_MEMORY_POLICY;
+	return {
+		sampleIntervalMs: num("CAMSTACK_POOL_MEM_INTERVAL_MS", d.sampleIntervalMs, 5e3),
+		baselineSettleSamples: d.baselineSettleSamples,
+		baselineSampleCount: d.baselineSampleCount,
+		restartMultiple: num("CAMSTACK_POOL_MEM_MULTIPLE", d.restartMultiple, 1.5),
+		floorBytes: num("CAMSTACK_POOL_MEM_FLOOR_MB", d.floorBytes / MB, 128) * MB,
+		ceilingBytes: num("CAMSTACK_POOL_MEM_CEILING_MB", d.ceilingBytes / MB, 256) * MB,
+		cooldownMs: num("CAMSTACK_POOL_MEM_COOLDOWN_MS", d.cooldownMs, 6e4),
+		maxRestartsPerWindow: num("CAMSTACK_POOL_MEM_MAX_RESTARTS", d.maxRestartsPerWindow, 1),
+		restartWindowMs: num("CAMSTACK_POOL_MEM_RESTART_WINDOW_MS", d.restartWindowMs, 6e4)
+	};
+}
+/** Parse the fields this watchdog needs out of `/proc/<pid>/status` text.
+*  Returns null when VmRSS is missing (dead pid, kernel thread, bad read). */
+function parseProcStatus(text) {
+	const kb = (label) => {
+		const match = text.match(new RegExp(`^${label}:\\s+(\\d+)\\s*kB`, "m"));
+		return match ? Number(match[1]) * 1024 : null;
+	};
+	const rssBytes = kb("VmRSS");
+	if (rssBytes === null) return null;
+	const threadsMatch = text.match(/^Threads:\s+(\d+)/m);
+	return {
+		rssBytes,
+		vmBytes: kb("VmSize") ?? 0,
+		hwmBytes: kb("VmHWM") ?? 0,
+		swapBytes: kb("VmSwap") ?? 0,
+		threads: threadsMatch ? Number(threadsMatch[1]) : 0
+	};
+}
+function initialPoolMemoryState() {
+	return {
+		settleSeen: 0,
+		baselineWindow: [],
+		baselineBytes: null,
+		restartsAt: [],
+		lastRestartAt: null
+	};
+}
+function median(values) {
+	const sorted = [...values].sort((a, b) => a - b);
+	return sorted[Math.floor(sorted.length / 2)];
+}
+function advanceBaseline(state, rssBytes, policy) {
+	if (state.baselineBytes !== null) return state;
+	if (state.settleSeen < policy.baselineSettleSamples) return {
+		...state,
+		settleSeen: state.settleSeen + 1
+	};
+	const window = [...state.baselineWindow, rssBytes];
+	if (window.length < policy.baselineSampleCount) return {
+		...state,
+		baselineWindow: window
+	};
+	return {
+		...state,
+		baselineWindow: window,
+		baselineBytes: median(window)
+	};
+}
+/** The one place a pool's restart threshold is computed. */
+function poolMemoryThreshold(baselineBytes, policy) {
+	if (baselineBytes === null) return policy.ceilingBytes;
+	return Math.min(policy.ceilingBytes, Math.max(policy.floorBytes, policy.restartMultiple * baselineBytes));
+}
+function pruneRestarts(restartsAt, nowMs, policy) {
+	return restartsAt.filter((t) => nowMs - t < policy.restartWindowMs);
+}
+/**
+* Evaluate one RSS sample. Pure: returns the successor state plus the verdict.
+* The caller performs (and logs) the restart, then commits it via
+* {@link commitWatchdogRestart}.
+*/
+function evaluatePoolMemory(state, rssBytes, nowMs, policy) {
+	const next = advanceBaseline(state, rssBytes, policy);
+	const thresholdBytes = poolMemoryThreshold(next.baselineBytes, policy);
+	if (rssBytes <= thresholdBytes) return {
+		state: next,
+		action: next.baselineBytes === null ? "baseline-pending" : "ok",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+	const inWindow = pruneRestarts(next.restartsAt, nowMs, policy);
+	const pruned = {
+		...next,
+		restartsAt: inWindow
+	};
+	if (inWindow.length >= policy.maxRestartsPerWindow) return {
+		state: pruned,
+		action: "exhausted",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+	if (pruned.lastRestartAt !== null && nowMs - pruned.lastRestartAt < policy.cooldownMs) return {
+		state: pruned,
+		action: "cooldown",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+	return {
+		state: pruned,
+		action: "restart",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+}
+/** Record a performed watchdog restart: budget consumed, baseline reset —
+*  the recreated pool is a NEW process and gets a fresh settle window. */
+function commitWatchdogRestart(state, nowMs) {
+	return {
+		...initialPoolMemoryState(),
+		restartsAt: [...state.restartsAt, nowMs],
+		lastRestartAt: nowMs
+	};
+}
+/** Reset baseline tracking (pool process replaced OUTSIDE the watchdog —
+*  kernel OOM kill + respawn, idle-reaper eviction, tuning respawn). The
+*  restart budget is kept: it bounds the WATCHDOG, not the pool. */
+function resetPoolBaseline(state) {
+	return {
+		...initialPoolMemoryState(),
+		restartsAt: state.restartsAt,
+		lastRestartAt: state.lastRestartAt
+	};
+}
+/**
+* Pick ONE pool to restart this sweep — the worst overage ratio. Restarting
+* several pools at once (e.g. after a model rollout grew every baseline)
+* would take detection down on every camera simultaneously; one per sweep
+* means the survivors keep serving while the worst offender reloads, and the
+* next sweep (one interval later) takes the next one.
+*/
+function pickRestartCandidate(candidates) {
+	if (candidates.length === 0) return null;
+	let worst = candidates[0];
+	for (const c of candidates) if (c.rssBytes / c.thresholdBytes > worst.rssBytes / worst.thresholdBytes) worst = c;
+	return worst.key;
+}
+var PoolMemoryWatchdog = class {
+	pools = /* @__PURE__ */ new Map();
+	timer = null;
+	sweeping = false;
+	stopped = false;
+	opts;
+	now;
+	setTimer;
+	clearTimer;
+	constructor(opts) {
+		this.opts = opts;
+		this.now = opts.now ?? (() => Date.now());
+		this.setTimer = opts.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
+		this.clearTimer = opts.clearTimer ?? ((t) => clearTimeout(t));
+	}
+	start() {
+		if (this.timer || this.stopped) return;
+		this.arm();
+	}
+	stop() {
+		this.stopped = true;
+		if (this.timer) {
+			this.clearTimer(this.timer);
+			this.timer = null;
+		}
+	}
+	arm() {
+		const timer = this.setTimer(() => {
+			this.sweep().finally(() => {
+				if (!this.stopped) this.arm();
+			});
+		}, this.opts.policy.sampleIntervalMs);
+		if (typeof timer.unref === "function") timer.unref();
+		this.timer = timer;
+	}
+	/** One sweep: sample → log every pool → restart at most one. Public so the
+	*  loop is testable without fake global timers. */
+	async sweep() {
+		if (this.sweeping) return;
+		this.sweeping = true;
+		try {
+			await this.doSweep();
+		} catch (err) {
+			this.opts.log.warn("pool memory sweep failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
+		} finally {
+			this.sweeping = false;
+		}
+	}
+	async doSweep() {
+		const nowMs = this.now();
+		const samples = await this.opts.sample();
+		const candidates = [];
+		const verdictByKey = /* @__PURE__ */ new Map();
+		for (const t of samples) {
+			const tracked = this.pools.get(t.key);
+			let state = tracked?.state ?? initialPoolMemoryState();
+			if (tracked && tracked.pids.length > 0 && !samePids(tracked.pids, t.pids)) {
+				this.opts.log.warn("pool process changed outside the watchdog — re-baselining", { meta: {
+					poolKey: t.key,
+					previousPids: tracked.pids,
+					pids: t.pids
+				} });
+				state = resetPoolBaseline(state);
+			}
+			const verdict = evaluatePoolMemory(state, t.rssBytes, nowMs, this.opts.policy);
+			this.pools.set(t.key, {
+				state: verdict.state,
+				pids: t.pids
+			});
+			verdictByKey.set(t.key, verdict);
+			this.opts.log.info("pool memory", { meta: {
+				poolKey: t.key,
+				pids: t.pids,
+				rssMb: Math.round(t.rssBytes / MB),
+				baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
+				thresholdMb: Math.round(verdict.thresholdBytes / MB),
+				action: verdict.action,
+				...t.meta
+			} });
+			if (verdict.action === "restart") candidates.push({
+				key: t.key,
+				rssBytes: t.rssBytes,
+				thresholdBytes: verdict.thresholdBytes
+			});
+			else if (verdict.action === "cooldown") this.opts.log.warn("pool over memory threshold but inside restart cooldown", { meta: {
+				poolKey: t.key,
+				rssMb: Math.round(t.rssBytes / MB),
+				thresholdMb: Math.round(verdict.thresholdBytes / MB),
+				cooldownMs: this.opts.policy.cooldownMs
+			} });
+			else if (verdict.action === "exhausted") this.opts.log.error("pool memory watchdog gave up — restart budget exhausted, pool leaks faster than restarts can pay for; needs a heap profile (py-spy / tracemalloc)", { meta: {
+				poolKey: t.key,
+				rssMb: Math.round(t.rssBytes / MB),
+				thresholdMb: Math.round(verdict.thresholdBytes / MB),
+				maxRestartsPerWindow: this.opts.policy.maxRestartsPerWindow,
+				restartWindowMs: this.opts.policy.restartWindowMs
+			} });
+		}
+		for (const key of [...this.pools.keys()]) if (!samples.some((t) => t.key === key)) this.pools.delete(key);
+		const pickedKey = pickRestartCandidate(candidates);
+		if (pickedKey === null) return;
+		const picked = candidates.find((c) => c.key === pickedKey);
+		const verdict = verdictByKey.get(pickedKey);
+		const tracked = this.pools.get(pickedKey);
+		const restartsInWindow = tracked.state.restartsAt.length + 1;
+		this.opts.log.warn("pool memory watchdog restarting pool — RSS over threshold", { meta: {
+			poolKey: pickedKey,
+			pids: tracked.pids,
+			rssMb: Math.round(picked.rssBytes / MB),
+			thresholdMb: Math.round(picked.thresholdBytes / MB),
+			baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
+			restartsInWindow,
+			deferredCandidates: candidates.filter((c) => c.key !== pickedKey).map((c) => c.key)
+		} });
+		try {
+			await this.opts.restart(pickedKey);
+			this.pools.set(pickedKey, {
+				state: commitWatchdogRestart(tracked.state, this.now()),
+				pids: []
+			});
+		} catch (err) {
+			this.opts.log.error("pool memory watchdog restart failed", { meta: {
+				poolKey: pickedKey,
+				error: err instanceof Error ? err.message : String(err)
+			} });
+		}
+	}
+};
+function samePids(a, b) {
+	if (a.length !== b.length) return false;
+	const sortedA = [...a].sort((x, y) => x - y);
+	const sortedB = [...b].sort((x, y) => x - y);
+	return sortedA.every((v, i) => v === sortedB[i]);
+}
+//#endregion
 //#region src/utils/privacy-grid-raster.ts
 /**
 * Rasterize a list of normalized rects into a row-major boolean cell grid.
@@ -46185,6 +46489,7 @@ exports.DEFAULT_EVENTS_BAND_BUFFER_SEC = DEFAULT_EVENTS_BAND_BUFFER_SEC;
 exports.DEFAULT_EVENT_COLOR = DEFAULT_EVENT_COLOR;
 exports.DEFAULT_FEATURES = DEFAULT_FEATURES;
 exports.DEFAULT_NATIVE_LEASE_SETTINGS = DEFAULT_NATIVE_LEASE_SETTINGS;
+exports.DEFAULT_POOL_MEMORY_POLICY = DEFAULT_POOL_MEMORY_POLICY;
 exports.DEFAULT_RECORDING_PROFILES = DEFAULT_RECORDING_PROFILES;
 exports.DEFAULT_RETENTION = DEFAULT_RETENTION;
 exports.DEFAULT_RUNTIME_STATE_DURABILITY = require_sleep.DEFAULT_RUNTIME_STATE_DURABILITY;
@@ -46582,6 +46887,7 @@ exports.PipelineValidationIssueSchema = PipelineValidationIssueSchema;
 exports.PipelineValidationResultSchema = PipelineValidationResultSchema;
 exports.PlaceholderReasonSchema = PlaceholderReasonSchema;
 exports.PolygonPointSchema = PolygonPointSchema;
+exports.PoolMemoryWatchdog = PoolMemoryWatchdog;
 exports.PowerMeterStatusSchema = PowerMeterStatusSchema;
 exports.PresenceStatusSchema = PresenceStatusSchema;
 exports.PressureSensorStatusSchema = PressureSensorStatusSchema;
@@ -46949,6 +47255,7 @@ exports.collectHydratedFieldEntries = require_sleep.collectHydratedFieldEntries;
 exports.collectHydratedFieldValues = require_sleep.collectHydratedFieldValues;
 exports.colorCapability = colorCapability;
 exports.colorForKind = colorForKind;
+exports.commitWatchdogRestart = commitWatchdogRestart;
 exports.compileExpression = compileExpression;
 exports.compileExpressionSafe = compileExpressionSafe;
 exports.composeSwitchedOff = composeSwitchedOff;
@@ -47018,6 +47325,7 @@ exports.enumerateSchemaFields = enumerateSchemaFields;
 exports.errMsg = require_err_msg.errMsg;
 exports.evaluateAst = evaluateAst;
 exports.evaluateExpressionSource = evaluateExpressionSource;
+exports.evaluatePoolMemory = evaluatePoolMemory;
 exports.evaluateZoneRules = evaluateZoneRules;
 exports.event = require_sleep.event;
 exports.eventEmitterCapability = eventEmitterCapability;
@@ -47047,6 +47355,7 @@ exports.humiditySensorCapability = humiditySensorCapability;
 exports.hydrateSchema = require_sleep.hydrateSchema;
 exports.imageCapability = imageCapability;
 exports.imageSettingsCapability = imageSettingsCapability;
+exports.initialPoolMemoryState = initialPoolMemoryState;
 exports.integrationsCapability = integrationsCapability;
 exports.intercomCapability = intercomCapability;
 exports.invocationFromEncodeProfile = require_canonical_hash.invocationFromEncodeProfile;
@@ -47129,6 +47438,7 @@ exports.parseExpression = parseExpression;
 exports.parseJsonArray = require_sleep.parseJsonArray;
 exports.parseJsonObject = require_sleep.parseJsonObject;
 exports.parseJsonUnknown = require_sleep.parseJsonUnknown;
+exports.parseProcStatus = parseProcStatus;
 exports.parseProfileBrokerId = require_sleep.parseProfileBrokerId;
 exports.parseStreamParamsFormPatch = parseStreamParamsFormPatch;
 exports.patchAudio = patchAudio;
@@ -47137,6 +47447,7 @@ exports.pickAccessoryControl = pickAccessoryControl;
 exports.pickDetailCropConvention = pickDetailCropConvention;
 exports.pickNativeLeaseOverride = pickNativeLeaseOverride;
 exports.pickPreferredRtspEntry = pickPreferredRtspEntry;
+exports.pickRestartCandidate = pickRestartCandidate;
 exports.pickVideoEncoder = require_canonical_hash.pickVideoEncoder;
 exports.pickerForCondition = pickerForCondition;
 exports.pipelineAnalyticsCapability = pipelineAnalyticsCapability;
@@ -47145,6 +47456,7 @@ exports.pipelineOrchestratorCapability = pipelineOrchestratorCapability;
 exports.pipelineRunnerCapability = pipelineRunnerCapability;
 exports.plateGalleryCapability = plateGalleryCapability;
 exports.platformProbeCapability = platformProbeCapability;
+exports.poolMemoryThreshold = poolMemoryThreshold;
 exports.powerMeterCapability = powerMeterCapability;
 exports.prepareNotification = prepareNotification;
 exports.presenceCapability = presenceCapability;
@@ -47166,6 +47478,7 @@ exports.recordingCapability = recordingCapability;
 exports.recordingExportCapability = recordingExportCapability;
 exports.rectsToCells = rectsToCells;
 exports.requiresPython = requiresPython;
+exports.resetPoolBaseline = resetPoolBaseline;
 exports.resolveAddonExecution = resolveAddonExecution;
 exports.resolveAddonGroup = resolveAddonGroup;
 exports.resolveAddonPlacement = resolveAddonPlacement;
@@ -47179,6 +47492,7 @@ exports.resolveFormat = resolveFormat;
 exports.resolveHydratedFieldValue = require_sleep.resolveHydratedFieldValue;
 exports.resolveModelFormat = resolveModelFormat;
 exports.resolveMutate = resolveMutate;
+exports.resolvePoolMemoryPolicy = resolvePoolMemoryPolicy;
 exports.resolveRecordingProfiles = resolveRecordingProfiles;
 exports.resolveRunnerId = resolveRunnerId;
 exports.resolveScrubThumbnailGeometry = resolveScrubThumbnailGeometry;