@camstack/addon-pipeline 1.2.38 → 1.2.40

package/dist/{step-definitions-C8tk9fPJ.mjs → event-loop-stall-monitor-BHqCesa1.mjs}

@@ -1,4 +1,5 @@
-import { D as hfModelUrl, a as COCO_TO_MACRO, i as COCO_80_LABELS, r as AUDIO_MACRO_LABELS } from "./dist-Du7_SNiR.mjs";
+import { D as hfModelUrl, a as COCO_TO_MACRO, i as COCO_80_LABELS, r as AUDIO_MACRO_LABELS } from "./dist-C6p8jcji.mjs";
+import { PerformanceObserver } from "node:perf_hooks";
 //#region src/detection-pipeline/registry/model-catalogs.ts
 var HF_REPO = "camstack/camstack-models";
 var HF_SCRYPTED = "scrypted/plugin-models";
@@ -1807,4 +1808,140 @@ function resolveModelForFormat(stepId, chosenModelId, format) {
 	return getDefaultModelForFormat(stepId, format);
 }
 //#endregion
-export { getStep as a, getDefaultModelForFormatFromDef as i, ALL_STEPS as n, getStepDefinition as o, getDefaultModelForFormat as r, resolveModelForFormat as s, ALL_PIPELINE_STEPS as t };
+//#region src/detection-pipeline/diagnostics/event-loop-stall-monitor.ts
+/**
+* event-loop-stall-monitor — names the thing that freezes the detection process.
+*
+* ## What this exists to settle
+*
+* On 2026-08-01/02 inference across the whole fleet stalled in bursts: 759
+* calls in 90 minutes crossed the runner's 20s deadline, p50 41s and max 80s,
+* and EVERY one of them eventually resolved. Six explanations were killed with
+* telemetry alone:
+*
+*  - not a slow GPU or a wedged pool — the work completes, just enormously late;
+*  - not CPU starvation — hosts sat at 8-48% during the freezes;
+*  - not swap — `pswpin/pswpout` are flat zero on every host;
+*  - not a logging artifact — Loki's timestamp trails the in-process one by
+*    0.1-1.0s at p95, so the gaps are real, not buffered delivery;
+*  - not the v1.2.30 release — the stalls predate it by a day;
+*  - not only the idle-pool recompile — that is real and hub-side, but the
+*    agent stalls with no eviction and no compile at all.
+*
+* What IS established: the 10s stats timer — which touches no inference code —
+* stops firing for 22-88s and resumes at the same instant the abandoned calls
+* settle. The event loop itself freezes. From outside the process, GC and a
+* synchronous block are indistinguishable, and guessing between them is how
+* this investigation would go wrong. So measure it.
+*
+* ## How
+*
+* A 1s interval whose LATENESS is the freeze: a tick that arrives 40s late means
+* the loop was blocked ~39s, whatever blocked it. In the same window the GC
+* observer accumulates pause time. The two numbers together answer the question
+* outright:
+*
+*  - `gcMs` ≈ `stallMs` → garbage collection. Look at heap growth.
+*  - `gcMs` ≈ 0 → something SYNCHRONOUS blocked the loop, and GC is innocent.
+*
+* Heap and RSS ride along because if it is GC, the next question is always
+* "growing since when?".
+*
+* Deliberately not a histogram (`monitorEventLoopDelay`): that reports a
+* distribution, and the question here is about ONE outlier and what coincided
+* with it.
+*/
+/** Tick period. Small enough to bound attribution error, large enough to cost
+*  nothing on a process already doing frame work. */
+var STALL_TICK_MS = 1e3;
+/**
+* Report a freeze only past this. The runner's own inference deadline is 20s
+* and the observed freezes run 22-88s, so this is far below the signal while
+* staying clear of ordinary scheduling jitter (a busy frame path routinely
+* overruns a 1s timer by tens of ms).
+*/
+var STALL_REPORT_MS = 2e3;
+/**
+* Attribute a freeze. Pure, because this is the ONE judgement the whole module
+* exists to make and it must be testable without freezing a real event loop.
+*
+* `gc` when garbage collection accounts for most of the stall; a
+* `synchronous-block` when it accounts for almost none — that verdict points
+* away from the heap entirely and at whatever ran without yielding.
+*/
+function attributeStall(stallMs, gcMs, gcCount, longestGcMs) {
+	const share = stallMs > 0 ? gcMs / stallMs : 0;
+	return {
+		stallMs,
+		gcMs,
+		gcCount,
+		longestGcMs,
+		cause: share >= .7 ? "gc" : share <= .2 ? "synchronous-block" : "mixed"
+	};
+}
+/**
+* Start watching. Returns a stop function; safe to call twice.
+*
+* Never throws into the caller: a diagnostic that can take the detection
+* process down is worse than no diagnostic. `PerformanceObserver` for `gc` is
+* unavailable in some embedded runtimes, and that must degrade to
+* stall-detection-without-attribution rather than a boot failure.
+*/
+function startEventLoopStallMonitor(logger, reportThresholdMs = STALL_REPORT_MS) {
+	let gc = {
+		totalMs: 0,
+		count: 0,
+		longestMs: 0
+	};
+	let observer = null;
+	try {
+		observer = new PerformanceObserver((list) => {
+			for (const entry of list.getEntries()) {
+				gc.totalMs += entry.duration;
+				gc.count += 1;
+				if (entry.duration > gc.longestMs) gc.longestMs = entry.duration;
+			}
+		});
+		observer.observe({ entryTypes: ["gc"] });
+	} catch {
+		observer = null;
+	}
+	let expected = Date.now() + STALL_TICK_MS;
+	const timer = setInterval(() => {
+		const now = Date.now();
+		const lateBy = now - expected;
+		const window = gc;
+		gc = {
+			totalMs: 0,
+			count: 0,
+			longestMs: 0
+		};
+		expected = now + STALL_TICK_MS;
+		if (lateBy < reportThresholdMs) return;
+		const verdict = attributeStall(lateBy, Math.round(window.totalMs), window.count, Math.round(window.longestMs));
+		const mem = process.memoryUsage();
+		logger.warn("event loop stalled", { meta: {
+			stallMs: verdict.stallMs,
+			cause: verdict.cause,
+			gcMs: verdict.gcMs,
+			gcCount: verdict.gcCount,
+			longestGcMs: verdict.longestGcMs,
+			gcObserved: observer !== null,
+			heapUsedMb: Math.round(mem.heapUsed / 1e6),
+			heapTotalMb: Math.round(mem.heapTotal / 1e6),
+			rssMb: Math.round(mem.rss / 1e6),
+			externalMb: Math.round(mem.external / 1e6),
+			arrayBuffersMb: Math.round(mem.arrayBuffers / 1e6)
+		} });
+	}, STALL_TICK_MS);
+	timer.unref?.();
+	let stopped = false;
+	return () => {
+		if (stopped) return;
+		stopped = true;
+		clearInterval(timer);
+		observer?.disconnect();
+	};
+}
+//#endregion
+export { getDefaultModelForFormat as a, getStepDefinition as c, ALL_STEPS as i, resolveModelForFormat as l, startEventLoopStallMonitor as n, getDefaultModelForFormatFromDef as o, ALL_PIPELINE_STEPS as r, getStep as s, attributeStall as t };

package/dist/{step-definitions-i8O-hxyI.js → event-loop-stall-monitor-oL9xanvX.js}

@@ -1,4 +1,5 @@
-const require_dist = require("./dist-BvFLZyhW.js");
+const require_dist = require("./dist-DcMBfoCg.js");
+let node_perf_hooks = require("node:perf_hooks");
 //#region src/detection-pipeline/registry/model-catalogs.ts
 var HF_REPO = "camstack/camstack-models";
 var HF_SCRYPTED = "scrypted/plugin-models";
@@ -1807,6 +1808,142 @@ function resolveModelForFormat(stepId, chosenModelId, format) {
 	return getDefaultModelForFormat(stepId, format);
 }
 //#endregion
+//#region src/detection-pipeline/diagnostics/event-loop-stall-monitor.ts
+/**
+* event-loop-stall-monitor — names the thing that freezes the detection process.
+*
+* ## What this exists to settle
+*
+* On 2026-08-01/02 inference across the whole fleet stalled in bursts: 759
+* calls in 90 minutes crossed the runner's 20s deadline, p50 41s and max 80s,
+* and EVERY one of them eventually resolved. Six explanations were killed with
+* telemetry alone:
+*
+*  - not a slow GPU or a wedged pool — the work completes, just enormously late;
+*  - not CPU starvation — hosts sat at 8-48% during the freezes;
+*  - not swap — `pswpin/pswpout` are flat zero on every host;
+*  - not a logging artifact — Loki's timestamp trails the in-process one by
+*    0.1-1.0s at p95, so the gaps are real, not buffered delivery;
+*  - not the v1.2.30 release — the stalls predate it by a day;
+*  - not only the idle-pool recompile — that is real and hub-side, but the
+*    agent stalls with no eviction and no compile at all.
+*
+* What IS established: the 10s stats timer — which touches no inference code —
+* stops firing for 22-88s and resumes at the same instant the abandoned calls
+* settle. The event loop itself freezes. From outside the process, GC and a
+* synchronous block are indistinguishable, and guessing between them is how
+* this investigation would go wrong. So measure it.
+*
+* ## How
+*
+* A 1s interval whose LATENESS is the freeze: a tick that arrives 40s late means
+* the loop was blocked ~39s, whatever blocked it. In the same window the GC
+* observer accumulates pause time. The two numbers together answer the question
+* outright:
+*
+*  - `gcMs` ≈ `stallMs` → garbage collection. Look at heap growth.
+*  - `gcMs` ≈ 0 → something SYNCHRONOUS blocked the loop, and GC is innocent.
+*
+* Heap and RSS ride along because if it is GC, the next question is always
+* "growing since when?".
+*
+* Deliberately not a histogram (`monitorEventLoopDelay`): that reports a
+* distribution, and the question here is about ONE outlier and what coincided
+* with it.
+*/
+/** Tick period. Small enough to bound attribution error, large enough to cost
+*  nothing on a process already doing frame work. */
+var STALL_TICK_MS = 1e3;
+/**
+* Report a freeze only past this. The runner's own inference deadline is 20s
+* and the observed freezes run 22-88s, so this is far below the signal while
+* staying clear of ordinary scheduling jitter (a busy frame path routinely
+* overruns a 1s timer by tens of ms).
+*/
+var STALL_REPORT_MS = 2e3;
+/**
+* Attribute a freeze. Pure, because this is the ONE judgement the whole module
+* exists to make and it must be testable without freezing a real event loop.
+*
+* `gc` when garbage collection accounts for most of the stall; a
+* `synchronous-block` when it accounts for almost none — that verdict points
+* away from the heap entirely and at whatever ran without yielding.
+*/
+function attributeStall(stallMs, gcMs, gcCount, longestGcMs) {
+	const share = stallMs > 0 ? gcMs / stallMs : 0;
+	return {
+		stallMs,
+		gcMs,
+		gcCount,
+		longestGcMs,
+		cause: share >= .7 ? "gc" : share <= .2 ? "synchronous-block" : "mixed"
+	};
+}
+/**
+* Start watching. Returns a stop function; safe to call twice.
+*
+* Never throws into the caller: a diagnostic that can take the detection
+* process down is worse than no diagnostic. `PerformanceObserver` for `gc` is
+* unavailable in some embedded runtimes, and that must degrade to
+* stall-detection-without-attribution rather than a boot failure.
+*/
+function startEventLoopStallMonitor(logger, reportThresholdMs = STALL_REPORT_MS) {
+	let gc = {
+		totalMs: 0,
+		count: 0,
+		longestMs: 0
+	};
+	let observer = null;
+	try {
+		observer = new node_perf_hooks.PerformanceObserver((list) => {
+			for (const entry of list.getEntries()) {
+				gc.totalMs += entry.duration;
+				gc.count += 1;
+				if (entry.duration > gc.longestMs) gc.longestMs = entry.duration;
+			}
+		});
+		observer.observe({ entryTypes: ["gc"] });
+	} catch {
+		observer = null;
+	}
+	let expected = Date.now() + STALL_TICK_MS;
+	const timer = setInterval(() => {
+		const now = Date.now();
+		const lateBy = now - expected;
+		const window = gc;
+		gc = {
+			totalMs: 0,
+			count: 0,
+			longestMs: 0
+		};
+		expected = now + STALL_TICK_MS;
+		if (lateBy < reportThresholdMs) return;
+		const verdict = attributeStall(lateBy, Math.round(window.totalMs), window.count, Math.round(window.longestMs));
+		const mem = process.memoryUsage();
+		logger.warn("event loop stalled", { meta: {
+			stallMs: verdict.stallMs,
+			cause: verdict.cause,
+			gcMs: verdict.gcMs,
+			gcCount: verdict.gcCount,
+			longestGcMs: verdict.longestGcMs,
+			gcObserved: observer !== null,
+			heapUsedMb: Math.round(mem.heapUsed / 1e6),
+			heapTotalMb: Math.round(mem.heapTotal / 1e6),
+			rssMb: Math.round(mem.rss / 1e6),
+			externalMb: Math.round(mem.external / 1e6),
+			arrayBuffersMb: Math.round(mem.arrayBuffers / 1e6)
+		} });
+	}, STALL_TICK_MS);
+	timer.unref?.();
+	let stopped = false;
+	return () => {
+		if (stopped) return;
+		stopped = true;
+		clearInterval(timer);
+		observer?.disconnect();
+	};
+}
+//#endregion
 Object.defineProperty(exports, "ALL_PIPELINE_STEPS", {
 	enumerable: true,
 	get: function() {
@@ -1819,6 +1956,12 @@ Object.defineProperty(exports, "ALL_STEPS", {
 		return ALL_STEPS;
 	}
 });
+Object.defineProperty(exports, "attributeStall", {
+	enumerable: true,
+	get: function() {
+		return attributeStall;
+	}
+});
 Object.defineProperty(exports, "getDefaultModelForFormat", {
 	enumerable: true,
 	get: function() {
@@ -1849,3 +1992,9 @@ Object.defineProperty(exports, "resolveModelForFormat", {
 		return resolveModelForFormat;
 	}
 });
+Object.defineProperty(exports, "startEventLoopStallMonitor", {
+	enumerable: true,
+	get: function() {
+		return startEventLoopStallMonitor;
+	}
+});

package/dist/lazy-sharp-B-mb8uWx.mjs

@@ -0,0 +1,20 @@
+//#region src/shared/lazy-sharp.ts
+/** Loaded on first use, then reused. `null` until someone asks. */
+var sharpPromise = null;
+/**
+* The `sharp` factory, resolved once per process.
+*
+* On failure the memo is CLEARED so a later call retries. A single transient
+* failure (a partially-installed native dep during a deploy) must not poison
+* the encoder for the process's whole lifetime — that would turn a blip into a
+* camera that never produces media again.
+*/
+async function getSharp() {
+	if (sharpPromise === null) sharpPromise = import("sharp").then((m) => m.default, (err) => {
+		sharpPromise = null;
+		throw err;
+	});
+	return sharpPromise;
+}
+//#endregion
+export { getSharp as t };

package/dist/lazy-sharp-CcYuXtsa.js

@@ -0,0 +1,25 @@
+//#region src/shared/lazy-sharp.ts
+/** Loaded on first use, then reused. `null` until someone asks. */
+var sharpPromise = null;
+/**
+* The `sharp` factory, resolved once per process.
+*
+* On failure the memo is CLEARED so a later call retries. A single transient
+* failure (a partially-installed native dep during a deploy) must not poison
+* the encoder for the process's whole lifetime — that would turn a blip into a
+* camera that never produces media again.
+*/
+async function getSharp() {
+	if (sharpPromise === null) sharpPromise = import("sharp").then((m) => m.default, (err) => {
+		sharpPromise = null;
+		throw err;
+	});
+	return sharpPromise;
+}
+//#endregion
+Object.defineProperty(exports, "getSharp", {
+	enumerable: true,
+	get: function() {
+		return getSharp;
+	}
+});

package/dist/motion-wasm/index.js

@@ -2,7 +2,8 @@ Object.defineProperties(exports, {
 	__esModule: { value: true },
 	[Symbol.toStringTag]: { value: "Module" }
 });
-const require_dist = require("../dist-BvFLZyhW.js");
+const require_dist = require("../dist-DcMBfoCg.js");
+const require_lazy_sharp = require("../lazy-sharp-CcYuXtsa.js");
 let node_fs = require("node:fs");
 let node_path = require("node:path");
 //#region src/motion-wasm/wasm-motion-detector.ts
@@ -226,8 +227,7 @@ var MotionWasmAddon = class MotionWasmAddon extends require_dist.BaseAddon {
 		let width;
 		let height;
 		if (frame.format === "jpeg") {
-			const sharp = (await import("sharp")).default;
-			const { data, info } = await sharp(Buffer.from(frame.data)).grayscale().raw().toBuffer({ resolveWithObject: true });
+			const { data, info } = await (await require_lazy_sharp.getSharp())(Buffer.from(frame.data)).grayscale().raw().toBuffer({ resolveWithObject: true });
 			gray = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
 			width = info.width;
 			height = info.height;

package/dist/motion-wasm/index.mjs

@@ -1,4 +1,5 @@
-import { A as motionDetectionCapability, E as evaluateZoneRules, K as hydrateSchema, V as BaseAddon, W as DeviceType } from "../dist-Du7_SNiR.mjs";
+import { A as motionDetectionCapability, E as evaluateZoneRules, K as hydrateSchema, V as BaseAddon, W as DeviceType } from "../dist-C6p8jcji.mjs";
+import { t as getSharp } from "../lazy-sharp-B-mb8uWx.mjs";
 import { readFileSync } from "node:fs";
 import { join } from "node:path";
 //#region src/motion-wasm/wasm-motion-detector.ts
@@ -222,8 +223,7 @@ var MotionWasmAddon = class MotionWasmAddon extends BaseAddon {
 		let width;
 		let height;
 		if (frame.format === "jpeg") {
-			const sharp = (await import("sharp")).default;
-			const { data, info } = await sharp(Buffer.from(frame.data)).grayscale().raw().toBuffer({ resolveWithObject: true });
+			const { data, info } = await (await getSharp())(Buffer.from(frame.data)).grayscale().raw().toBuffer({ resolveWithObject: true });
 			gray = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
 			width = info.width;
 			height = info.height;