@camstack/addon-post-analysis 1.2.92 → 1.2.93

package/dist/{dist-VnaTfwAL.mjs → dist-3QRyOemL.mjs}

@@ -1,4 +1,4 @@
-//#region ../types/dist/event-category-C0lyLd5U.mjs
+//#region ../types/dist/event-category-XfKNtfCc.mjs
 var EventCategory = /* @__PURE__ */ function(EventCategory) {
 	EventCategory["SystemBoot"] = "system.boot";
 	EventCategory["SystemAddonsReady"] = "system.addons-ready";
@@ -37,6 +37,22 @@ var EventCategory = /* @__PURE__ */ function(EventCategory) {
 	EventCategory["AddonInstalled"] = "addon.installed";
 	EventCategory["AddonUninstalled"] = "addon.uninstalled";
 	EventCategory["AddonCrashed"] = "addon.crashed";
+	/**
+	* A RUNNER the D6 crash circuit-breaker gave up on — terminal.
+	*
+	* `AddonCrashed` is the routine, self-healing fact ("it crashed; it is being
+	* respawned"); this is the one that never resolves itself. Emitted exactly
+	* once per trip, by `process-service.ts`, carrying the node, the runner, the
+	* addons it hosted and the crash count that tripped the breaker.
+	*
+	* It exists because a runner marked terminally `failed` used to be SILENT:
+	* on 2026-08-18 the `recorder` runner died of three unhandled ffmpeg spawn
+	* errors, the breaker stopped respawning it (correctly), `/health` kept
+	* returning 200, and fleet-wide recording was gone for 3h15 before the
+	* operator's phone told him. The Notification Center's system-event intake
+	* consumes this category and maps it to the `addon-crash-loop` kind.
+	*/
+	EventCategory["AddonRunnerFailed"] = "addon.runner-failed";
 	EventCategory["AddonError"] = "addon.error";
 	EventCategory["AddonPageReady"] = "addon.page-ready";
 	EventCategory["AddonWidgetReady"] = "addon.widget-ready";
@@ -14814,6 +14830,7 @@ var NcSystemEventKindSchema = _enum([
 	"node-offline",
 	"node-inference-unavailable",
 	"detection-blind",
+	"addon-crash-loop",
 	"addon-update-available",
 	"server-update-available",
 	"alarm-triggered",
@@ -15939,6 +15956,10 @@ var NC_CONDITION_CATALOG = [
 				value: "detection-blind",
 				label: "Camera detecting nothing"
 			},
+			{
+				value: "addon-crash-loop",
+				label: "Addon stopped after repeated crashes"
+			},
 			{
 				value: "addon-update-available",
 				label: "Addon update available"
@@ -38537,7 +38558,7 @@ function systemEventFilterApplies(kind, filter) {
 	switch (filter) {
 		case "deviceIds": return kind.startsWith("device-") || kind.startsWith("stream-") || kind === "detection-blind" || kind === "alarm-triggered" || kind === "export-completed";
 		case "deviceTypes": return kind.startsWith("device-") || kind === "detection-blind";
-		case "nodeIds": return kind.startsWith("node-") || kind === "addon-update-available" || kind === "addon-updated" || kind === "server-update-available" || kind === "server-updated";
+		case "nodeIds": return kind.startsWith("node-") || kind === "addon-crash-loop" || kind === "addon-update-available" || kind === "addon-updated" || kind === "server-update-available" || kind === "server-updated";
 		case "packageNames": return kind.endsWith("update-available") || kind === "addon-updated" || kind === "server-updated";
 	}
 }
@@ -38994,6 +39015,303 @@ function cosineSimilarity(a, b) {
 function hfModelUrl(repo, path) {
 	return `https://huggingface.co/${repo}/resolve/main/${path}`;
 }
+var MB = 1024 * 1024;
+var DEFAULT_POOL_MEMORY_POLICY = {
+	sampleIntervalMs: 6e4,
+	baselineSettleSamples: 5,
+	baselineSampleCount: 3,
+	restartMultiple: 4,
+	floorBytes: 1024 * MB,
+	ceilingBytes: 3072 * MB,
+	cooldownMs: 30 * 6e4,
+	maxRestartsPerWindow: 6,
+	restartWindowMs: 1440 * 6e4
+};
+/**
+* Resolve the policy from env overrides (`CAMSTACK_POOL_MEM_*`). Garbage or
+* absent values keep the default — an operator typo must never disable the
+* bound or set it to zero.
+*/
+function resolvePoolMemoryPolicy(env) {
+	const num = (key, fallback, min) => {
+		const raw = env[key];
+		if (raw === void 0) return fallback;
+		const parsed = Number(raw);
+		return Number.isFinite(parsed) && parsed >= min ? parsed : fallback;
+	};
+	const d = DEFAULT_POOL_MEMORY_POLICY;
+	return {
+		sampleIntervalMs: num("CAMSTACK_POOL_MEM_INTERVAL_MS", d.sampleIntervalMs, 5e3),
+		baselineSettleSamples: d.baselineSettleSamples,
+		baselineSampleCount: d.baselineSampleCount,
+		restartMultiple: num("CAMSTACK_POOL_MEM_MULTIPLE", d.restartMultiple, 1.5),
+		floorBytes: num("CAMSTACK_POOL_MEM_FLOOR_MB", d.floorBytes / MB, 128) * MB,
+		ceilingBytes: num("CAMSTACK_POOL_MEM_CEILING_MB", d.ceilingBytes / MB, 256) * MB,
+		cooldownMs: num("CAMSTACK_POOL_MEM_COOLDOWN_MS", d.cooldownMs, 6e4),
+		maxRestartsPerWindow: num("CAMSTACK_POOL_MEM_MAX_RESTARTS", d.maxRestartsPerWindow, 1),
+		restartWindowMs: num("CAMSTACK_POOL_MEM_RESTART_WINDOW_MS", d.restartWindowMs, 6e4)
+	};
+}
+/** Parse the fields this watchdog needs out of `/proc/<pid>/status` text.
+*  Returns null when VmRSS is missing (dead pid, kernel thread, bad read). */
+function parseProcStatus(text) {
+	const kb = (label) => {
+		const match = text.match(new RegExp(`^${label}:\\s+(\\d+)\\s*kB`, "m"));
+		return match ? Number(match[1]) * 1024 : null;
+	};
+	const rssBytes = kb("VmRSS");
+	if (rssBytes === null) return null;
+	const threadsMatch = text.match(/^Threads:\s+(\d+)/m);
+	return {
+		rssBytes,
+		vmBytes: kb("VmSize") ?? 0,
+		hwmBytes: kb("VmHWM") ?? 0,
+		swapBytes: kb("VmSwap") ?? 0,
+		threads: threadsMatch ? Number(threadsMatch[1]) : 0
+	};
+}
+function initialPoolMemoryState() {
+	return {
+		settleSeen: 0,
+		baselineWindow: [],
+		baselineBytes: null,
+		restartsAt: [],
+		lastRestartAt: null
+	};
+}
+function median(values) {
+	const sorted = [...values].sort((a, b) => a - b);
+	return sorted[Math.floor(sorted.length / 2)];
+}
+function advanceBaseline(state, rssBytes, policy) {
+	if (state.baselineBytes !== null) return state;
+	if (state.settleSeen < policy.baselineSettleSamples) return {
+		...state,
+		settleSeen: state.settleSeen + 1
+	};
+	const window = [...state.baselineWindow, rssBytes];
+	if (window.length < policy.baselineSampleCount) return {
+		...state,
+		baselineWindow: window
+	};
+	return {
+		...state,
+		baselineWindow: window,
+		baselineBytes: median(window)
+	};
+}
+/** The one place a pool's restart threshold is computed. */
+function poolMemoryThreshold(baselineBytes, policy) {
+	if (baselineBytes === null) return policy.ceilingBytes;
+	return Math.min(policy.ceilingBytes, Math.max(policy.floorBytes, policy.restartMultiple * baselineBytes));
+}
+function pruneRestarts(restartsAt, nowMs, policy) {
+	return restartsAt.filter((t) => nowMs - t < policy.restartWindowMs);
+}
+/**
+* Evaluate one RSS sample. Pure: returns the successor state plus the verdict.
+* The caller performs (and logs) the restart, then commits it via
+* {@link commitWatchdogRestart}.
+*/
+function evaluatePoolMemory(state, rssBytes, nowMs, policy) {
+	const next = advanceBaseline(state, rssBytes, policy);
+	const thresholdBytes = poolMemoryThreshold(next.baselineBytes, policy);
+	if (rssBytes <= thresholdBytes) return {
+		state: next,
+		action: next.baselineBytes === null ? "baseline-pending" : "ok",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+	const inWindow = pruneRestarts(next.restartsAt, nowMs, policy);
+	const pruned = {
+		...next,
+		restartsAt: inWindow
+	};
+	if (inWindow.length >= policy.maxRestartsPerWindow) return {
+		state: pruned,
+		action: "exhausted",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+	if (pruned.lastRestartAt !== null && nowMs - pruned.lastRestartAt < policy.cooldownMs) return {
+		state: pruned,
+		action: "cooldown",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+	return {
+		state: pruned,
+		action: "restart",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+}
+/** Record a performed watchdog restart: budget consumed, baseline reset —
+*  the recreated pool is a NEW process and gets a fresh settle window. */
+function commitWatchdogRestart(state, nowMs) {
+	return {
+		...initialPoolMemoryState(),
+		restartsAt: [...state.restartsAt, nowMs],
+		lastRestartAt: nowMs
+	};
+}
+/** Reset baseline tracking (pool process replaced OUTSIDE the watchdog —
+*  kernel OOM kill + respawn, idle-reaper eviction, tuning respawn). The
+*  restart budget is kept: it bounds the WATCHDOG, not the pool. */
+function resetPoolBaseline(state) {
+	return {
+		...initialPoolMemoryState(),
+		restartsAt: state.restartsAt,
+		lastRestartAt: state.lastRestartAt
+	};
+}
+/**
+* Pick ONE pool to restart this sweep — the worst overage ratio. Restarting
+* several pools at once (e.g. after a model rollout grew every baseline)
+* would take detection down on every camera simultaneously; one per sweep
+* means the survivors keep serving while the worst offender reloads, and the
+* next sweep (one interval later) takes the next one.
+*/
+function pickRestartCandidate(candidates) {
+	if (candidates.length === 0) return null;
+	let worst = candidates[0];
+	for (const c of candidates) if (c.rssBytes / c.thresholdBytes > worst.rssBytes / worst.thresholdBytes) worst = c;
+	return worst.key;
+}
+var PoolMemoryWatchdog = class {
+	pools = /* @__PURE__ */ new Map();
+	timer = null;
+	sweeping = false;
+	stopped = false;
+	opts;
+	now;
+	setTimer;
+	clearTimer;
+	constructor(opts) {
+		this.opts = opts;
+		this.now = opts.now ?? (() => Date.now());
+		this.setTimer = opts.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
+		this.clearTimer = opts.clearTimer ?? ((t) => clearTimeout(t));
+	}
+	start() {
+		if (this.timer || this.stopped) return;
+		this.arm();
+	}
+	stop() {
+		this.stopped = true;
+		if (this.timer) {
+			this.clearTimer(this.timer);
+			this.timer = null;
+		}
+	}
+	arm() {
+		const timer = this.setTimer(() => {
+			this.sweep().finally(() => {
+				if (!this.stopped) this.arm();
+			});
+		}, this.opts.policy.sampleIntervalMs);
+		if (typeof timer.unref === "function") timer.unref();
+		this.timer = timer;
+	}
+	/** One sweep: sample → log every pool → restart at most one. Public so the
+	*  loop is testable without fake global timers. */
+	async sweep() {
+		if (this.sweeping) return;
+		this.sweeping = true;
+		try {
+			await this.doSweep();
+		} catch (err) {
+			this.opts.log.warn("pool memory sweep failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
+		} finally {
+			this.sweeping = false;
+		}
+	}
+	async doSweep() {
+		const nowMs = this.now();
+		const samples = await this.opts.sample();
+		const candidates = [];
+		const verdictByKey = /* @__PURE__ */ new Map();
+		for (const t of samples) {
+			const tracked = this.pools.get(t.key);
+			let state = tracked?.state ?? initialPoolMemoryState();
+			if (tracked && tracked.pids.length > 0 && !samePids(tracked.pids, t.pids)) {
+				this.opts.log.warn("pool process changed outside the watchdog — re-baselining", { meta: {
+					poolKey: t.key,
+					previousPids: tracked.pids,
+					pids: t.pids
+				} });
+				state = resetPoolBaseline(state);
+			}
+			const verdict = evaluatePoolMemory(state, t.rssBytes, nowMs, this.opts.policy);
+			this.pools.set(t.key, {
+				state: verdict.state,
+				pids: t.pids
+			});
+			verdictByKey.set(t.key, verdict);
+			this.opts.log.info("pool memory", { meta: {
+				poolKey: t.key,
+				pids: t.pids,
+				rssMb: Math.round(t.rssBytes / MB),
+				baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
+				thresholdMb: Math.round(verdict.thresholdBytes / MB),
+				action: verdict.action,
+				...t.meta
+			} });
+			if (verdict.action === "restart") candidates.push({
+				key: t.key,
+				rssBytes: t.rssBytes,
+				thresholdBytes: verdict.thresholdBytes
+			});
+			else if (verdict.action === "cooldown") this.opts.log.warn("pool over memory threshold but inside restart cooldown", { meta: {
+				poolKey: t.key,
+				rssMb: Math.round(t.rssBytes / MB),
+				thresholdMb: Math.round(verdict.thresholdBytes / MB),
+				cooldownMs: this.opts.policy.cooldownMs
+			} });
+			else if (verdict.action === "exhausted") this.opts.log.error("pool memory watchdog gave up — restart budget exhausted, pool leaks faster than restarts can pay for; needs a heap profile (py-spy / tracemalloc)", { meta: {
+				poolKey: t.key,
+				rssMb: Math.round(t.rssBytes / MB),
+				thresholdMb: Math.round(verdict.thresholdBytes / MB),
+				maxRestartsPerWindow: this.opts.policy.maxRestartsPerWindow,
+				restartWindowMs: this.opts.policy.restartWindowMs
+			} });
+		}
+		for (const key of [...this.pools.keys()]) if (!samples.some((t) => t.key === key)) this.pools.delete(key);
+		const pickedKey = pickRestartCandidate(candidates);
+		if (pickedKey === null) return;
+		const picked = candidates.find((c) => c.key === pickedKey);
+		const verdict = verdictByKey.get(pickedKey);
+		const tracked = this.pools.get(pickedKey);
+		const restartsInWindow = tracked.state.restartsAt.length + 1;
+		this.opts.log.warn("pool memory watchdog restarting pool — RSS over threshold", { meta: {
+			poolKey: pickedKey,
+			pids: tracked.pids,
+			rssMb: Math.round(picked.rssBytes / MB),
+			thresholdMb: Math.round(picked.thresholdBytes / MB),
+			baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
+			restartsInWindow,
+			deferredCandidates: candidates.filter((c) => c.key !== pickedKey).map((c) => c.key)
+		} });
+		try {
+			await this.opts.restart(pickedKey);
+			this.pools.set(pickedKey, {
+				state: commitWatchdogRestart(tracked.state, this.now()),
+				pids: []
+			});
+		} catch (err) {
+			this.opts.log.error("pool memory watchdog restart failed", { meta: {
+				poolKey: pickedKey,
+				error: err instanceof Error ? err.message : String(err)
+			} });
+		}
+	}
+};
+function samePids(a, b) {
+	if (a.length !== b.length) return false;
+	const sortedA = [...a].sort((x, y) => x - y);
+	const sortedB = [...b].sort((x, y) => x - y);
+	return sortedA.every((v, i) => v === sortedB[i]);
+}
 /**
 * Vector wire codec — base64 of little-endian Float32.
 *
@@ -39023,4 +39341,4 @@ function vectorDimFromBase64(encoded) {
 	return Math.floor(Buffer.from(encoded, "base64").byteLength / 4);
 }
 //#endregion
-export { faceGalleryCapability as $, SCENE_CONFIRM_DEFAULT_TIMEOUT_MS as A, partialRecord as At, VISIT_MERGE_GAP_MS as B, NcSnoozeSchema as C, _enum as Ct, OpsLogEntrySchema as D, literal as Dt, NcTaxonomySchema as E, discriminatedUnion as Et, TIMELAPSE_DENSE_FLOOR_SEC as F, audioModeOf as G, alarmPanelCapability as H, TimelapseRuleInputSchema as I, customAction as J, buildEventKindDescriptor as K, TimelapseRulePatchSchema as L, SCENE_DEFAULT_UNCOVERED_POLICY as M, string as Mt, SCENE_DIVERGED as N, unknown as Nt, RECORDING_EXPORT_MAX_READ_BYTES as O, number as Ot, SceneMonitorSchema as P, EventCategory as Pt, encodeVectorBase64 as Q, TimelapseRuleSchema as R, NcSnoozeInputSchema as S, sleep as St, NcSystemEventKindSchema as T, boolean as Tt, assertTimelapseCadences as U, addonWidgetsSourceCapability as V, audioMetricsCapability as W, deriveRecordingMode as X, defineCustomActions as Y, embeddingEncoderCapability as Z, NcRuleInputSchema as _, DeviceType as _t, DETECTION_PIPELINE_CAP_NAME as a, pipelineAnalyticsCapability as at, NcRuleTargetSchema as b, isDeviceScopedCap as bt, EVENT_PAD_MS as c, readTimelapseGeneratedAt as ct, NC_ALARM_SYSTEM_EVENT_KINDS as d, systemEventFilterApplies as dt, hfModelUrl as et, NC_CONDITION_CATALOG as f, vectorDimFromBase64 as ft, NcConditionDescriptorSchema as g, BaseAddon as gt, NC_TAXONOMY as h, errMsg as ht, DETECTION_MACRO_CLASSES as i, notificationRulesCapability as it, SCENE_DEFAULT_ANCHOR_THRESHOLD as j, record as jt, RetrainStatusSchema as k, object as kt, LabelAttributionSchema as l, sceneMonitorCapability as lt, NC_SNOOZE_MAX_MINUTES as m, zoneAnalyticsCapability as mt, DEFAULT_EVENT_COLOR as n, isScheduleActive as nt, DeclaredDevices as o, plateGalleryCapability as ot, NC_DEFAULT_SNOOZE_MINUTES as p, videoclipsCapability as pt, cosineSimilarity as q, DEFAULT_TIMELAPSE_PREVIEW_TEXT as r, kebabToCamel as rt, EVENT_KIND_BY_CAP as s, readDeviceStateFrom as st, BaseDevice as t, isDetectionMacroClass as tt, MACRO_LABELS as u, subKindsOf as ut, NcRulePatchSchema as v, createEvent as vt, NcSnoozeSuppressedSchema as w, array as wt, NcScheduleSchema as x, nodePin as xt, NcRuleSchema as y, hydrateSchema as yt, TrackSourceSchema as z };
+export { encodeVectorBase64 as $, RetrainStatusSchema as A, literal as At, TrackSourceSchema as B, NcSnoozeSchema as C, isDeviceScopedCap as Ct, OpsLogEntrySchema as D, array as Dt, NcTaxonomySchema as E, _enum as Et, SceneMonitorSchema as F, string as Ft, audioMetricsCapability as G, addonWidgetsSourceCapability as H, TIMELAPSE_DENSE_FLOOR_SEC as I, unknown as It, cosineSimilarity as J, audioModeOf as K, TimelapseRuleInputSchema as L, EventCategory as Lt, SCENE_DEFAULT_ANCHOR_THRESHOLD as M, object as Mt, SCENE_DEFAULT_UNCOVERED_POLICY as N, partialRecord as Nt, PoolMemoryWatchdog as O, boolean as Ot, SCENE_DIVERGED as P, record as Pt, embeddingEncoderCapability as Q, TimelapseRulePatchSchema as R, NcSnoozeInputSchema as S, hydrateSchema as St, NcSystemEventKindSchema as T, sleep as Tt, alarmPanelCapability as U, VISIT_MERGE_GAP_MS as V, assertTimelapseCadences as W, defineCustomActions as X, customAction as Y, deriveRecordingMode as Z, NcRuleInputSchema as _, zoneAnalyticsCapability as _t, DETECTION_PIPELINE_CAP_NAME as a, notificationRulesCapability as at, NcRuleTargetSchema as b, DeviceType as bt, EVENT_PAD_MS as c, plateGalleryCapability as ct, NC_ALARM_SYSTEM_EVENT_KINDS as d, resolvePoolMemoryPolicy as dt, faceGalleryCapability as et, NC_CONDITION_CATALOG as f, sceneMonitorCapability as ft, NcConditionDescriptorSchema as g, videoclipsCapability as gt, NC_TAXONOMY as h, vectorDimFromBase64 as ht, DETECTION_MACRO_CLASSES as i, kebabToCamel as it, SCENE_CONFIRM_DEFAULT_TIMEOUT_MS as j, number as jt, RECORDING_EXPORT_MAX_READ_BYTES as k, discriminatedUnion as kt, LabelAttributionSchema as l, readDeviceStateFrom as lt, NC_SNOOZE_MAX_MINUTES as m, systemEventFilterApplies as mt, DEFAULT_EVENT_COLOR as n, isDetectionMacroClass as nt, DeclaredDevices as o, parseProcStatus as ot, NC_DEFAULT_SNOOZE_MINUTES as p, subKindsOf as pt, buildEventKindDescriptor as q, DEFAULT_TIMELAPSE_PREVIEW_TEXT as r, isScheduleActive as rt, EVENT_KIND_BY_CAP as s, pipelineAnalyticsCapability as st, BaseDevice as t, hfModelUrl as tt, MACRO_LABELS as u, readTimelapseGeneratedAt as ut, NcRulePatchSchema as v, errMsg as vt, NcSnoozeSuppressedSchema as w, nodePin as wt, NcScheduleSchema as x, createEvent as xt, NcRuleSchema as y, BaseAddon as yt, TimelapseRuleSchema as z };