@camstack/addon-post-analysis 1.2.91 → 1.2.93

package/dist/{dist-CTIUuGRB.js → dist-DYoWeRRt.js}

@@ -29,7 +29,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 	enumerable: true
 }) : target, mod));
 //#endregion
-//#region ../types/dist/event-category-C0lyLd5U.mjs
+//#region ../types/dist/event-category-XfKNtfCc.mjs
 var EventCategory = /* @__PURE__ */ function(EventCategory) {
 	EventCategory["SystemBoot"] = "system.boot";
 	EventCategory["SystemAddonsReady"] = "system.addons-ready";
@@ -68,6 +68,22 @@ var EventCategory = /* @__PURE__ */ function(EventCategory) {
 	EventCategory["AddonInstalled"] = "addon.installed";
 	EventCategory["AddonUninstalled"] = "addon.uninstalled";
 	EventCategory["AddonCrashed"] = "addon.crashed";
+	/**
+	* A RUNNER the D6 crash circuit-breaker gave up on — terminal.
+	*
+	* `AddonCrashed` is the routine, self-healing fact ("it crashed; it is being
+	* respawned"); this is the one that never resolves itself. Emitted exactly
+	* once per trip, by `process-service.ts`, carrying the node, the runner, the
+	* addons it hosted and the crash count that tripped the breaker.
+	*
+	* It exists because a runner marked terminally `failed` used to be SILENT:
+	* on 2026-08-18 the `recorder` runner died of three unhandled ffmpeg spawn
+	* errors, the breaker stopped respawning it (correctly), `/health` kept
+	* returning 200, and fleet-wide recording was gone for 3h15 before the
+	* operator's phone told him. The Notification Center's system-event intake
+	* consumes this category and maps it to the `addon-crash-loop` kind.
+	*/
+	EventCategory["AddonRunnerFailed"] = "addon.runner-failed";
 	EventCategory["AddonError"] = "addon.error";
 	EventCategory["AddonPageReady"] = "addon.page-ready";
 	EventCategory["AddonWidgetReady"] = "addon.widget-ready";
@@ -14845,6 +14861,7 @@ var NcSystemEventKindSchema = _enum([
 	"node-offline",
 	"node-inference-unavailable",
 	"detection-blind",
+	"addon-crash-loop",
 	"addon-update-available",
 	"server-update-available",
 	"alarm-triggered",
@@ -15970,6 +15987,10 @@ var NC_CONDITION_CATALOG = [
 				value: "detection-blind",
 				label: "Camera detecting nothing"
 			},
+			{
+				value: "addon-crash-loop",
+				label: "Addon stopped after repeated crashes"
+			},
 			{
 				value: "addon-update-available",
 				label: "Addon update available"
@@ -21487,20 +21508,6 @@ var VectorStatsResultSchema = object({
 	exact: boolean()
 });
 method(VectorDeclareIndexInputSchema, _void(), { kind: "mutation" }), method(VectorUpsertInputSchema, VectorUpsertResultSchema, { kind: "mutation" }), method(VectorQueryInputSchema, VectorQueryResultSchema), method(VectorGetInputSchema, VectorGetResultSchema), method(VectorDeleteInputSchema, VectorDeleteResultSchema, { kind: "mutation" }), method(VectorDeleteByFilterInputSchema, VectorDeleteResultSchema, { kind: "mutation" }), method(VectorStatsInputSchema, VectorStatsResultSchema);
-/**
-* `videoclips` — the unified, navigable-clip surface for a camera.
-*
-* A device-scoped WRAPPER cap (like `pipeline-analytics`): exactly one active
-* provider per device, substitutable. The DEFAULT provider (registered by
-* `addon-post-analysis`, `defaultActive: true`) composes the analytics event
-* log (markers + thumbnails) with the recorder's `getPlaybackManifest` — a clip
-* is a time-WINDOW over existing footage, never a separate file. A camera that
-* exposes NATIVE onboard clips (Reolink/Hikvision NVR) can later substitute the
-* wrapper provider for its device and serve its own clip catalog + URLs.
-*
-* A `Clip` is purely time-based (subtree-blind): playback resolves segments by
-* temporal overlap, so the API never decides `continuous` vs `events`.
-*/
 var ClipSchema = object({
 	/** Opaque, provider-namespaced id. The default provider encodes the time
 	*  window so `getClipPlayback` is self-contained (no event re-query). */
@@ -21526,12 +21533,41 @@ var ClipSchema = object({
 	* of the event that owns `kind` (object > motion > audio). Do not extract
 	* a keyframe from the recorded segments. Other providers (onboard, HKSV)
 	* mint their own stills.
+	*
+	* **VOUCHED, never fabricated.** A provider emits this only for an event it
+	* has CONFIRMED owns at least one media row (its own, or its owning track's).
+	* Absent is meaningful — "this visit has no event still" — never "we did not
+	* look". Stamping it from a URL template made 35% of one camera's clips point
+	* at a 404 (device 3836, 2026-08-17: 64 of 179 clips dead, 63 of them motion).
+	* A read that FAILS drops the claim; it never invents it.
 	*/
 	thumbnail: string().optional(),
-	/** Analytics event ids that overlap this visit. Empty on footage-only clips.
-	*  The default provider's visit grain puts many motion heartbeats on one clip
-	*  instead of minting one clip per marker. */
+	/**
+	* An instant INSIDE this visit's footage, hole-safe, where a recorded still
+	* can be decoded. Present whenever the visit came from recorded availability;
+	* absent on a per-event padded window (there is no footage to promise).
+	*
+	* This is not a thumbnail and not a second byte path: it is the argument to
+	* the recorder's existing still route. The surface — never the provider —
+	* decides whether to use it. It is the midpoint of the visit's LONGEST
+	* contiguous range, not of the visit: a visit spans its holes by
+	* construction, so a naive midpoint lands in dead air.
+	*/
+	stillAtMs: number().optional(),
+	/**
+	* Analytics event ids that overlap this visit — a BOUNDED sample, newest
+	* first within kind (object → motion → audio), capped at
+	* {@link MAX_CLIP_EVENT_IDS}. Empty on footage-only clips.
+	*
+	* Bounded because it is not a payload the surface pages through: one visit on
+	* device 615 carried 2 345 ids, and `eventIds` was 99% of a 153 KB
+	* camera-day. Read {@link eventCount} for the true total.
+	*/
 	eventIds: array(string()).optional(),
+	/** How many analytics events actually overlap this visit. Differs from
+	*  `eventIds.length` exactly when the sample was capped — so a truncated
+	*  list is never mistaken for a quiet visit. */
+	eventCount: number().int().nonnegative().optional(),
 	/** Intra-visit footage holes (GOP rolls, discarded segments) still shorter
 	*  than `VISIT_MERGE_GAP_MS`. Playback concatenates around them; the timeline
 	*  bar keeps showing them via `recording.getAvailability`. */
@@ -38553,7 +38589,7 @@ function systemEventFilterApplies(kind, filter) {
 	switch (filter) {
 		case "deviceIds": return kind.startsWith("device-") || kind.startsWith("stream-") || kind === "detection-blind" || kind === "alarm-triggered" || kind === "export-completed";
 		case "deviceTypes": return kind.startsWith("device-") || kind === "detection-blind";
-		case "nodeIds": return kind.startsWith("node-") || kind === "addon-update-available" || kind === "addon-updated" || kind === "server-update-available" || kind === "server-updated";
+		case "nodeIds": return kind.startsWith("node-") || kind === "addon-crash-loop" || kind === "addon-update-available" || kind === "addon-updated" || kind === "server-update-available" || kind === "server-updated";
 		case "packageNames": return kind.endsWith("update-available") || kind === "addon-updated" || kind === "server-updated";
 	}
 }
@@ -39010,6 +39046,303 @@ function cosineSimilarity(a, b) {
 function hfModelUrl(repo, path) {
 	return `https://huggingface.co/${repo}/resolve/main/${path}`;
 }
+var MB = 1024 * 1024;
+var DEFAULT_POOL_MEMORY_POLICY = {
+	sampleIntervalMs: 6e4,
+	baselineSettleSamples: 5,
+	baselineSampleCount: 3,
+	restartMultiple: 4,
+	floorBytes: 1024 * MB,
+	ceilingBytes: 3072 * MB,
+	cooldownMs: 30 * 6e4,
+	maxRestartsPerWindow: 6,
+	restartWindowMs: 1440 * 6e4
+};
+/**
+* Resolve the policy from env overrides (`CAMSTACK_POOL_MEM_*`). Garbage or
+* absent values keep the default — an operator typo must never disable the
+* bound or set it to zero.
+*/
+function resolvePoolMemoryPolicy(env) {
+	const num = (key, fallback, min) => {
+		const raw = env[key];
+		if (raw === void 0) return fallback;
+		const parsed = Number(raw);
+		return Number.isFinite(parsed) && parsed >= min ? parsed : fallback;
+	};
+	const d = DEFAULT_POOL_MEMORY_POLICY;
+	return {
+		sampleIntervalMs: num("CAMSTACK_POOL_MEM_INTERVAL_MS", d.sampleIntervalMs, 5e3),
+		baselineSettleSamples: d.baselineSettleSamples,
+		baselineSampleCount: d.baselineSampleCount,
+		restartMultiple: num("CAMSTACK_POOL_MEM_MULTIPLE", d.restartMultiple, 1.5),
+		floorBytes: num("CAMSTACK_POOL_MEM_FLOOR_MB", d.floorBytes / MB, 128) * MB,
+		ceilingBytes: num("CAMSTACK_POOL_MEM_CEILING_MB", d.ceilingBytes / MB, 256) * MB,
+		cooldownMs: num("CAMSTACK_POOL_MEM_COOLDOWN_MS", d.cooldownMs, 6e4),
+		maxRestartsPerWindow: num("CAMSTACK_POOL_MEM_MAX_RESTARTS", d.maxRestartsPerWindow, 1),
+		restartWindowMs: num("CAMSTACK_POOL_MEM_RESTART_WINDOW_MS", d.restartWindowMs, 6e4)
+	};
+}
+/** Parse the fields this watchdog needs out of `/proc/<pid>/status` text.
+*  Returns null when VmRSS is missing (dead pid, kernel thread, bad read). */
+function parseProcStatus(text) {
+	const kb = (label) => {
+		const match = text.match(new RegExp(`^${label}:\\s+(\\d+)\\s*kB`, "m"));
+		return match ? Number(match[1]) * 1024 : null;
+	};
+	const rssBytes = kb("VmRSS");
+	if (rssBytes === null) return null;
+	const threadsMatch = text.match(/^Threads:\s+(\d+)/m);
+	return {
+		rssBytes,
+		vmBytes: kb("VmSize") ?? 0,
+		hwmBytes: kb("VmHWM") ?? 0,
+		swapBytes: kb("VmSwap") ?? 0,
+		threads: threadsMatch ? Number(threadsMatch[1]) : 0
+	};
+}
+function initialPoolMemoryState() {
+	return {
+		settleSeen: 0,
+		baselineWindow: [],
+		baselineBytes: null,
+		restartsAt: [],
+		lastRestartAt: null
+	};
+}
+function median(values) {
+	const sorted = [...values].sort((a, b) => a - b);
+	return sorted[Math.floor(sorted.length / 2)];
+}
+function advanceBaseline(state, rssBytes, policy) {
+	if (state.baselineBytes !== null) return state;
+	if (state.settleSeen < policy.baselineSettleSamples) return {
+		...state,
+		settleSeen: state.settleSeen + 1
+	};
+	const window = [...state.baselineWindow, rssBytes];
+	if (window.length < policy.baselineSampleCount) return {
+		...state,
+		baselineWindow: window
+	};
+	return {
+		...state,
+		baselineWindow: window,
+		baselineBytes: median(window)
+	};
+}
+/** The one place a pool's restart threshold is computed. */
+function poolMemoryThreshold(baselineBytes, policy) {
+	if (baselineBytes === null) return policy.ceilingBytes;
+	return Math.min(policy.ceilingBytes, Math.max(policy.floorBytes, policy.restartMultiple * baselineBytes));
+}
+function pruneRestarts(restartsAt, nowMs, policy) {
+	return restartsAt.filter((t) => nowMs - t < policy.restartWindowMs);
+}
+/**
+* Evaluate one RSS sample. Pure: returns the successor state plus the verdict.
+* The caller performs (and logs) the restart, then commits it via
+* {@link commitWatchdogRestart}.
+*/
+function evaluatePoolMemory(state, rssBytes, nowMs, policy) {
+	const next = advanceBaseline(state, rssBytes, policy);
+	const thresholdBytes = poolMemoryThreshold(next.baselineBytes, policy);
+	if (rssBytes <= thresholdBytes) return {
+		state: next,
+		action: next.baselineBytes === null ? "baseline-pending" : "ok",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+	const inWindow = pruneRestarts(next.restartsAt, nowMs, policy);
+	const pruned = {
+		...next,
+		restartsAt: inWindow
+	};
+	if (inWindow.length >= policy.maxRestartsPerWindow) return {
+		state: pruned,
+		action: "exhausted",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+	if (pruned.lastRestartAt !== null && nowMs - pruned.lastRestartAt < policy.cooldownMs) return {
+		state: pruned,
+		action: "cooldown",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+	return {
+		state: pruned,
+		action: "restart",
+		baselineBytes: next.baselineBytes,
+		thresholdBytes
+	};
+}
+/** Record a performed watchdog restart: budget consumed, baseline reset —
+*  the recreated pool is a NEW process and gets a fresh settle window. */
+function commitWatchdogRestart(state, nowMs) {
+	return {
+		...initialPoolMemoryState(),
+		restartsAt: [...state.restartsAt, nowMs],
+		lastRestartAt: nowMs
+	};
+}
+/** Reset baseline tracking (pool process replaced OUTSIDE the watchdog —
+*  kernel OOM kill + respawn, idle-reaper eviction, tuning respawn). The
+*  restart budget is kept: it bounds the WATCHDOG, not the pool. */
+function resetPoolBaseline(state) {
+	return {
+		...initialPoolMemoryState(),
+		restartsAt: state.restartsAt,
+		lastRestartAt: state.lastRestartAt
+	};
+}
+/**
+* Pick ONE pool to restart this sweep — the worst overage ratio. Restarting
+* several pools at once (e.g. after a model rollout grew every baseline)
+* would take detection down on every camera simultaneously; one per sweep
+* means the survivors keep serving while the worst offender reloads, and the
+* next sweep (one interval later) takes the next one.
+*/
+function pickRestartCandidate(candidates) {
+	if (candidates.length === 0) return null;
+	let worst = candidates[0];
+	for (const c of candidates) if (c.rssBytes / c.thresholdBytes > worst.rssBytes / worst.thresholdBytes) worst = c;
+	return worst.key;
+}
+var PoolMemoryWatchdog = class {
+	pools = /* @__PURE__ */ new Map();
+	timer = null;
+	sweeping = false;
+	stopped = false;
+	opts;
+	now;
+	setTimer;
+	clearTimer;
+	constructor(opts) {
+		this.opts = opts;
+		this.now = opts.now ?? (() => Date.now());
+		this.setTimer = opts.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
+		this.clearTimer = opts.clearTimer ?? ((t) => clearTimeout(t));
+	}
+	start() {
+		if (this.timer || this.stopped) return;
+		this.arm();
+	}
+	stop() {
+		this.stopped = true;
+		if (this.timer) {
+			this.clearTimer(this.timer);
+			this.timer = null;
+		}
+	}
+	arm() {
+		const timer = this.setTimer(() => {
+			this.sweep().finally(() => {
+				if (!this.stopped) this.arm();
+			});
+		}, this.opts.policy.sampleIntervalMs);
+		if (typeof timer.unref === "function") timer.unref();
+		this.timer = timer;
+	}
+	/** One sweep: sample → log every pool → restart at most one. Public so the
+	*  loop is testable without fake global timers. */
+	async sweep() {
+		if (this.sweeping) return;
+		this.sweeping = true;
+		try {
+			await this.doSweep();
+		} catch (err) {
+			this.opts.log.warn("pool memory sweep failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
+		} finally {
+			this.sweeping = false;
+		}
+	}
+	async doSweep() {
+		const nowMs = this.now();
+		const samples = await this.opts.sample();
+		const candidates = [];
+		const verdictByKey = /* @__PURE__ */ new Map();
+		for (const t of samples) {
+			const tracked = this.pools.get(t.key);
+			let state = tracked?.state ?? initialPoolMemoryState();
+			if (tracked && tracked.pids.length > 0 && !samePids(tracked.pids, t.pids)) {
+				this.opts.log.warn("pool process changed outside the watchdog — re-baselining", { meta: {
+					poolKey: t.key,
+					previousPids: tracked.pids,
+					pids: t.pids
+				} });
+				state = resetPoolBaseline(state);
+			}
+			const verdict = evaluatePoolMemory(state, t.rssBytes, nowMs, this.opts.policy);
+			this.pools.set(t.key, {
+				state: verdict.state,
+				pids: t.pids
+			});
+			verdictByKey.set(t.key, verdict);
+			this.opts.log.info("pool memory", { meta: {
+				poolKey: t.key,
+				pids: t.pids,
+				rssMb: Math.round(t.rssBytes / MB),
+				baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
+				thresholdMb: Math.round(verdict.thresholdBytes / MB),
+				action: verdict.action,
+				...t.meta
+			} });
+			if (verdict.action === "restart") candidates.push({
+				key: t.key,
+				rssBytes: t.rssBytes,
+				thresholdBytes: verdict.thresholdBytes
+			});
+			else if (verdict.action === "cooldown") this.opts.log.warn("pool over memory threshold but inside restart cooldown", { meta: {
+				poolKey: t.key,
+				rssMb: Math.round(t.rssBytes / MB),
+				thresholdMb: Math.round(verdict.thresholdBytes / MB),
+				cooldownMs: this.opts.policy.cooldownMs
+			} });
+			else if (verdict.action === "exhausted") this.opts.log.error("pool memory watchdog gave up — restart budget exhausted, pool leaks faster than restarts can pay for; needs a heap profile (py-spy / tracemalloc)", { meta: {
+				poolKey: t.key,
+				rssMb: Math.round(t.rssBytes / MB),
+				thresholdMb: Math.round(verdict.thresholdBytes / MB),
+				maxRestartsPerWindow: this.opts.policy.maxRestartsPerWindow,
+				restartWindowMs: this.opts.policy.restartWindowMs
+			} });
+		}
+		for (const key of [...this.pools.keys()]) if (!samples.some((t) => t.key === key)) this.pools.delete(key);
+		const pickedKey = pickRestartCandidate(candidates);
+		if (pickedKey === null) return;
+		const picked = candidates.find((c) => c.key === pickedKey);
+		const verdict = verdictByKey.get(pickedKey);
+		const tracked = this.pools.get(pickedKey);
+		const restartsInWindow = tracked.state.restartsAt.length + 1;
+		this.opts.log.warn("pool memory watchdog restarting pool — RSS over threshold", { meta: {
+			poolKey: pickedKey,
+			pids: tracked.pids,
+			rssMb: Math.round(picked.rssBytes / MB),
+			thresholdMb: Math.round(picked.thresholdBytes / MB),
+			baselineMb: verdict.baselineBytes === null ? null : Math.round(verdict.baselineBytes / MB),
+			restartsInWindow,
+			deferredCandidates: candidates.filter((c) => c.key !== pickedKey).map((c) => c.key)
+		} });
+		try {
+			await this.opts.restart(pickedKey);
+			this.pools.set(pickedKey, {
+				state: commitWatchdogRestart(tracked.state, this.now()),
+				pids: []
+			});
+		} catch (err) {
+			this.opts.log.error("pool memory watchdog restart failed", { meta: {
+				poolKey: pickedKey,
+				error: err instanceof Error ? err.message : String(err)
+			} });
+		}
+	}
+};
+function samePids(a, b) {
+	if (a.length !== b.length) return false;
+	const sortedA = [...a].sort((x, y) => x - y);
+	const sortedB = [...b].sort((x, y) => x - y);
+	return sortedA.every((v, i) => v === sortedB[i]);
+}
 /**
 * Vector wire codec — base64 of little-endian Float32.
 *
@@ -39219,6 +39552,12 @@ Object.defineProperty(exports, "OpsLogEntrySchema", {
 		return OpsLogEntrySchema;
 	}
 });
+Object.defineProperty(exports, "PoolMemoryWatchdog", {
+	enumerable: true,
+	get: function() {
+		return PoolMemoryWatchdog;
+	}
+});
 Object.defineProperty(exports, "RECORDING_EXPORT_MAX_READ_BYTES", {
 	enumerable: true,
 	get: function() {
@@ -39489,6 +39828,12 @@ Object.defineProperty(exports, "object", {
 		return object;
 	}
 });
+Object.defineProperty(exports, "parseProcStatus", {
+	enumerable: true,
+	get: function() {
+		return parseProcStatus;
+	}
+});
 Object.defineProperty(exports, "partialRecord", {
 	enumerable: true,
 	get: function() {
@@ -39525,6 +39870,12 @@ Object.defineProperty(exports, "record", {
 		return record;
 	}
 });
+Object.defineProperty(exports, "resolvePoolMemoryPolicy", {
+	enumerable: true,
+	get: function() {
+		return resolvePoolMemoryPolicy;
+	}
+});
 Object.defineProperty(exports, "sceneMonitorCapability", {
 	enumerable: true,
 	get: function() {