@camstack/addon-pipeline 1.2.614 → 1.2.616

package/dist/stream-broker/index.mjs

@@ -1,9 +1,9 @@
 import { r as __toESM } from "../chunk-DnnnRqeS.mjs";
-import { $n as parseProfileBrokerId, $t as rankStreams, An as DeviceFeature, At as gifDeriveSpeed, Bn as createEvent, Cn as AUDIO_PRESETS, D as G711_SCALE_CORRECTION_DB, En as buildFfmpegArgs, Fn as CAM_PROFILE_ORDER, I as RATE_CONTROL_RELAXED, In as CamProfileSchema, It as loadContributionCapability, J as addonWidgetsSourceCapability, Jn as isSecretMarker, L as RATE_CONTROL_TIGHT, Lt as logChannelsCapability, Mn as errMsg, Nn as BOOT_RECOVERY_BACKOFF_MS, On as invocationFromEncodeProfile, Ot as ffmpegRate, Pn as BaseAddon, Qn as nodePin, Sn as webrtcSessionCapability, Vt as memoizePinnedFfmpeg, W as RingBuffer, Wn as expandAudioChunkToF32le, Xn as makeSourceBrokerId, Yn as makeProfileBrokerId, Z as armRemovalReconcile, Zn as maskUrlCredentials, ar as boolean, bt as egressTranscodeSharingKey, c as CLIP_STILL_REASON_CODE_HEADER, cr as literal, dt as declareLogChannel, fr as record, hr as EventCategory, ir as array, it as cameraStreamsCapability, jn as DeviceType, jt as gifFromMp4Args, kn as isSoftwareDecode, lr as number, lt as createHwAccelCache, mr as union, o as BatteryStatusSchema, on as resolveEgressDecodeHwAccel, or as discriminatedUnion, ot as clipExportFailureOf, pr as string, pt as deriveBatteryPresence, qn as isEvent, rr as _enum, s as CLIP_BROKER_PACED_RATES, tt as batteryCapability, ur as object, ut as createLogChannelsProvider, vn as streamBrokerCapability, vt as deviceRemovalHookCapability, w as EncodeProfileSchema, xt as egressTransportFromRequest } from "../dist-BGq_y1xM.mjs";
-import { a as defaultFfmpegSpawn, i as NO_COST_CLAIM, n as Fmp4FragmentChild, r as Fmp4FragmentPlane, s as probeFfmpegBinary, t as ChildCostRegistry } from "../node-DwTqHnch.mjs";
+import { $n as parseProfileBrokerId, $t as rankStreams, An as DeviceFeature, At as gifDeriveSpeed, Bn as createEvent, Cn as AUDIO_PRESETS, D as G711_SCALE_CORRECTION_DB, En as buildFfmpegArgs, Fn as CAM_PROFILE_ORDER, I as RATE_CONTROL_RELAXED, In as CamProfileSchema, It as loadContributionCapability, J as addonWidgetsSourceCapability, Jn as isSecretMarker, L as RATE_CONTROL_TIGHT, Lt as logChannelsCapability, Mn as errMsg, Nn as BOOT_RECOVERY_BACKOFF_MS, On as invocationFromEncodeProfile, Ot as ffmpegRate, Pn as BaseAddon, Qn as nodePin, Sn as webrtcSessionCapability, Vt as memoizePinnedFfmpeg, W as RingBuffer, Wn as expandAudioChunkToF32le, Xn as makeSourceBrokerId, Yn as makeProfileBrokerId, Z as armRemovalReconcile, Zn as maskUrlCredentials, ar as boolean, bt as egressTranscodeSharingKey, c as CLIP_STILL_REASON_CODE_HEADER, cr as literal, dt as declareLogChannel, fr as record, hr as EventCategory, ir as array, it as cameraStreamsCapability, jn as DeviceType, jt as gifFromMp4Args, kn as isSoftwareDecode, lr as number, lt as createHwAccelCache, mr as union, o as BatteryStatusSchema, on as resolveEgressDecodeHwAccel, or as discriminatedUnion, ot as clipExportFailureOf, pr as string, pt as deriveBatteryPresence, qn as isEvent, rr as _enum, s as CLIP_BROKER_PACED_RATES, tt as batteryCapability, ur as object, ut as createLogChannelsProvider, vn as streamBrokerCapability, vt as deviceRemovalHookCapability, w as EncodeProfileSchema, xt as egressTransportFromRequest } from "../dist-BWPVIuD_.mjs";
+import { a as defaultFfmpegSpawn, i as NO_COST_CLAIM, n as Fmp4FragmentChild, r as Fmp4FragmentPlane, s as probeFfmpegBinary, t as ChildCostRegistry } from "../node-D48F38hW.mjs";
 import { a as parseRestreamPath, d as PREBUFFER_MAX_MS, f as capPreRollRequestMs, i as isLiveEdgeOnlyDial, m as resolvePrebuffer, n as NO_DECLARATION, p as prebufferRetentionMs, s as sdpVariantFor, u as withSentryIntent } from "../restream-intent-DyGmLAU9.mjs";
 import { n as profileForStreamId } from "../remote-restream-BeHi78PZ.mjs";
-import { a as require_mp4box_all, c as parseNalLengthSize, i as runFfmpegOnce, n as demuxSegmentPreferJs, o as avccSampleToAnnexB, r as FfmpegFallbackRefusedError, s as buildKeyframeAnnexB, t as demuxSegmentFullJs } from "../segment-demux-js-CWJseQHd.mjs";
+import { a as require_mp4box_all, c as parseNalLengthSize, i as runFfmpegOnce, n as demuxSegmentPreferJs, o as avccSampleToAnnexB, r as FfmpegFallbackRefusedError, s as buildKeyframeAnnexB, t as demuxSegmentFullJs } from "../segment-demux-js-VeHvPIJe.mjs";
 import { createRequire } from "node:module";
 import * as os$1 from "node:os";
 import { networkInterfaces, tmpdir } from "node:os";
@@ -5684,6 +5684,13 @@ var RTSP_DESCRIBE_WARMUP_ATTRIBUTION = {
 	label: "describe-warmup",
 	media: "both"
 };
+/** How long a DESCRIBE waits for the broker to dial upstream and parse the SDP. */
+var SDP_WAIT_MS = 3e3;
+/** The numeric device id a broker id (`<deviceId>/<stream>`) belongs to, or undefined. */
+function deviceIdOfBroker(brokerId) {
+	const head = Number((brokerId ?? "").split("/")[0]);
+	return Number.isInteger(head) ? head : void 0;
+}
 /**
 * Single TCP server for all RTSP restream connections.
 * Routes clients to the correct RtspRestreamer by a random token path
@@ -5716,6 +5723,16 @@ var RtspListenServer = class RtspListenServer {
 	brokerWarmUps = /* @__PURE__ */ new Map();
 	/** brokerId → sentry picture-window opener */
 	brokerPrestreamOpeners = /* @__PURE__ */ new Map();
+	logger = null;
+	sdpWaitMs = SDP_WAIT_MS;
+	/** Shorten the DESCRIBE SDP wait (tests only). */
+	setSdpWaitMs(ms) {
+		this.sdpWaitMs = ms;
+	}
+	/** The logger the refusal branches write to; absent in unit specs that do not care. */
+	setLogger(logger) {
+		this.logger = logger;
+	}
 	/** Generate a random token for a stream path */
 	static generateToken() {
 		return randomBytes(16).toString("hex");
@@ -5911,8 +5928,23 @@ var RtspListenServer = class RtspListenServer {
 		};
 		socket.once("close", releaseOnce);
 		socket.once("error", releaseOnce);
-		await this.waitForSdp(restreamer, sdpVariant, 3e3);
-		const session = new RtspSession(socket, restreamer.getSdpFor(sdpVariant) ?? restreamer.getSdp() ?? "", () => {
+		const sdpReady = await this.waitForSdp(restreamer, sdpVariant, this.sdpWaitMs);
+		const sdp = restreamer.getSdpFor(sdpVariant) ?? restreamer.getSdp() ?? "";
+		if (!sdpReady || sdp.length === 0) {
+			releaseOnce();
+			this.logger?.warn("RTSP DESCRIBE refused: no SDP after the wait", {
+				tags: { deviceId: deviceIdOfBroker(brokerId) },
+				meta: {
+					brokerId,
+					waitedMs: this.sdpWaitMs,
+					variant: sdpVariant
+				}
+			});
+			socket.write(`RTSP/1.0 503 Service Unavailable\r\nCSeq: 1\r\n\r\n`);
+			socket.destroy();
+			return;
+		}
+		const session = new RtspSession(socket, sdp, () => {
 			restreamer.removeSession(session.getSessionId());
 			releaseOnce();
 		}, isMuted, declaration);
@@ -18017,6 +18049,7 @@ var StreamBrokerManager = class StreamBrokerManager {
 	};
 	constructor(logger) {
 		this.logger = logger;
+		this.rtspServer.setLogger(logger.child("rtsp-listen"));
 		this.rtspProvider = new RtspRestreamProviderImpl(this.rtspServer, logger.child("rtsp-restream"));
 	}
 	/**

package/dist/stream-broker/remoteEntry.js

@@ -30,7 +30,7 @@ async function d(e) {
 	}
 }
 async function f() {
-	return l ||= d(() => import("./_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-DrWysXZj.mjs")).catch((e) => {
+	return l ||= d(() => import("./_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-B6vm1OlX.mjs")).catch((e) => {
 		throw l = void 0, e;
 	}), l;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@camstack/addon-pipeline",
-  "version": "1.2.614",
+  "version": "1.2.616",
   "description": "Pipeline bundle — runner, detection, motion, audio + stream broker. Multi-entry npm package shipping pipeline addons under a single bundle.",
   "keywords": [
     "camstack",

package/python/inference_pool.py

@@ -1045,6 +1045,27 @@ def _pin_batch_for_device(core: Any, source: Any, path: str, device: str) -> Any
     return model
 
 
+# The OpenVINO compile probe (`openvino_compile_probe`) runs the pool's own
+# `_load_model` in a throwaway child; the child switches the probe off in itself.
+_ov_probe_enabled: bool = True
+
+
+def _ov_env_tag(core: Any) -> str:
+    """OpenVINO version + GPU driver: what a persisted probe verdict is valid for."""
+    parts: "list[str]" = []
+    try:
+        from openvino import get_version
+
+        parts.append(str(get_version()))
+    except Exception:
+        pass
+    try:
+        parts.append(str(core.get_property("GPU", "DEVICE_ARCHITECTURE")))
+    except Exception:
+        pass
+    return "|".join(parts)
+
+
 def _load_model(slot: ModelSlot, config: dict) -> None:
     """Load a model into a slot using the active runtime. Thin adapter per backend."""
     slot.config = dict(config)
@@ -1163,6 +1184,18 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
             ov_config["CACHE_DIR"] = _cache_dir
         except OSError:
             pass
+        # A GPU/NPU compile that has no blob yet runs in a CHILD first
+        # (`openvino_compile_probe`): a compiler SIGSEGV is refused here with
+        # `OpenvinoLoadCrashed` instead of killing this worker, and on success
+        # the child has written the blob, so the compile below is a cache hit.
+        if _ov_probe_enabled:
+            import openvino_compile_probe
+
+            openvino_compile_probe.ensure_openvino_loadable(
+                path, config, candidates,
+                cache_dir=ov_config.get("CACHE_DIR"),
+                env_tag=_ov_env_tag(core),
+            )
         # Fix 2 — fold uint8→float/scale/layout preprocessing INTO the compiled
         # graph via PrePostProcessor. When it succeeds we compile the PPP model
         # object (fed uint8 NHWC at inference time); on ANY failure we fall back

package/python/openvino_compile_probe.py

@@ -0,0 +1,245 @@
+"""Compile an OpenVINO GPU/NPU model in a CHILD process before the pool loads it.
+
+Why (2026-10-22, hub with a Core Ultra 7 265K iGPU, OpenVINO 2025.4.1): the
+iGPU compiler SIGSEGVed the pool worker three times in 15 s while
+`compile_model()` ran for a model with no compiled blob (`malloc(): unsorted
+double linked list corrupted`, then `longjmp causes uninitialized stack
+frame`). The compile runs in the same process as the predict threads and every
+other model, so one native crash took the whole `openvino:gpu` worker (and the
+camera's detection with it) down, and the respawn went straight back into the
+same load. Python cannot catch a SIGSEGV. A warm load (blob present) never
+crashed.
+
+So the first load of each (model x device request x compile config) happens in
+a throwaway interpreter that runs the pool's OWN `_load_model` — same
+PrePostProcessor, same candidate list, same properties, same CACHE_DIR — and
+therefore writes the blob the pool then loads warm. Mirrors CoreML's
+`coreml_load_probe.py` (D742):
+
+  ok           -> load in-process (a cache HIT), never probe this key again
+                  while the blobs the probe wrote are still on disk
+  crashed      -> refuse with `OpenvinoLoadCrashed` (reason
+                  `native-load-crashed`), now and on every later load of the
+                  same key, without re-probing
+  inconclusive -> (a Python exception in the child, a probe past its bound, a
+                  child that could not start) remembered NOWHERE; the
+                  in-process load runs exactly as it did before this gate
+
+Only a death by SIGNAL is a crash. CPU-only loads are never probed.
+"""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+from typing import Callable, Optional
+
+# One-way: inference_pool imports this module lazily, inside its OpenVINO branch.
+from coreml_load_probe import (
+    VERDICT_CRASHED,
+    VERDICT_INCONCLUSIVE,
+    VERDICT_OK,
+    _signal_name,
+    classify_probe_exit,
+)
+
+PROBE_VERDICT_VERSION = 1
+PROBE_MARKER = "@@CAMSTACK_OV_PROBE_OK@@"
+# A cold iGPU compile of a large detector takes tens of seconds; past this the
+# probe says nothing and the pool's own load bound (POOL_MODEL_LOAD_BOUNDS)
+# is the only fence, exactly as before the probe existed.
+PROBE_TIMEOUT_SEC = 300.0
+# The config keys that change what `compile_model` is handed (PrePostProcessor
+# plan, input shape, precision hint). A threshold or a class list does not.
+COMPILE_CONFIG_KEYS = (
+    "device",
+    "inputSize",
+    "inputWidth",
+    "inputHeight",
+    "inputChannels",
+    "preprocessMode",
+    "inputNormalization",
+    "gpuInferencePrecision",
+)
+PROBED_DEVICE_PREFIXES = ("GPU", "NPU")
+
+_HERE = os.path.dirname(os.path.abspath(__file__))
+
+# The child runs the pool's own loader, with the probe switched off in it.
+_CHILD_CODE = f"""
+import json, sys
+args = json.loads(sys.argv[1])
+sys.path.insert(0, args["pool_dir"])
+import inference_pool as ip
+ip._ov_probe_enabled = False
+ip._init_runtime("openvino", str(args["config"].get("device", "")))
+ip._load_model(ip.ModelSlot(), args["config"])
+print({PROBE_MARKER!r}, flush=True)
+"""
+
+ProbeRunner = Callable[[str, dict], "tuple[str, str]"]
+
+
+class OpenvinoLoadCrashed(Exception):
+    """The model kills the native compiler on this device; refused instead of loaded."""
+
+    reason = "native-load-crashed"
+
+    def __init__(self, model_path: str, device: str, detail: str) -> None:
+        self.model_path = model_path
+        self.device = device
+        self.detail = detail
+        super().__init__(
+            f"OpenVINO model {os.path.basename(model_path)} crashes the native compiler "
+            f"on device={device} ({detail}); refused so it cannot take the pool down. "
+            f"Run this model on another device, or replace it"
+        )
+
+
+def needs_probe(candidates: "list[str]") -> bool:
+    """True when any compile target is a GPU or an NPU (CPU-only loads are never probed)."""
+    return any(str(c).upper().startswith(PROBED_DEVICE_PREFIXES) for c in candidates)
+
+
+def run_probe(
+    model_path: str,
+    config: dict,
+    child_code: str = _CHILD_CODE,
+    timeout_sec: float = PROBE_TIMEOUT_SEC,
+) -> "tuple[str, str]":
+    """Compile `model_path` in a fresh interpreter; return (verdict, detail)."""
+    payload = json.dumps({"config": {**config, "path": model_path}, "pool_dir": _HERE})
+    try:
+        proc = subprocess.run(
+            [sys.executable, "-c", child_code, payload],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
+            timeout=timeout_sec,
+        )
+    except subprocess.TimeoutExpired:
+        return VERDICT_INCONCLUSIVE, f"probe timed out after {timeout_sec:g}s"
+    except (OSError, TypeError, ValueError) as exc:
+        return VERDICT_INCONCLUSIVE, f"probe could not start ({exc})"
+    marker_seen = any(line.strip() == PROBE_MARKER for line in proc.stdout.splitlines())
+    verdict = classify_probe_exit(proc.returncode, marker_seen)
+    tail = " | ".join(proc.stderr.strip().splitlines()[-3:])
+    if proc.returncode < 0:
+        detail = f"probe child died by signal {-proc.returncode} ({_signal_name(-proc.returncode)})"
+    else:
+        detail = f"probe child exited {proc.returncode}"
+    return verdict, f"{detail}{': ' + tail if tail else ''}"
+
+
+# Looked up at call time, so a test can replace it.
+DEFAULT_RUNNER: ProbeRunner = run_probe
+
+
+def _file_fingerprint(path: str) -> dict:
+    """size + mtime of the model file and, for an IR, its `.bin` weights."""
+    out: dict = {}
+    for candidate in (path, os.path.splitext(path)[0] + ".bin"):
+        try:
+            st = os.stat(candidate)
+        except OSError:
+            continue
+        out[os.path.basename(candidate)] = [st.st_size, st.st_mtime_ns]
+    return out
+
+
+def probe_key(path: str, config: dict, candidates: "list[str]", env_tag: str) -> dict:
+    return {
+        "version": PROBE_VERDICT_VERSION,
+        "model": _file_fingerprint(path),
+        "candidates": [str(c).upper() for c in candidates],
+        "compile": {k: config.get(k) for k in COMPILE_CONFIG_KEYS},
+        # An OpenVINO / GPU-driver update may fix or break a model.
+        "env": env_tag,
+    }
+
+
+def probe_verdict_path(path: str, config: dict, cache_dir: str) -> str:
+    stem = os.path.splitext(os.path.basename(path))[0]
+    device = str(config.get("device", "AUTO")).upper() or "AUTO"
+    return os.path.join(cache_dir, f"{stem}.{device}.ovprobe.json")
+
+
+def _blob_names(cache_dir: str) -> "set[str]":
+    try:
+        return {n for n in os.listdir(cache_dir) if n.endswith(".blob")}
+    except OSError:
+        return set()
+
+
+def _read_verdict(path: str, key: dict) -> Optional[dict]:
+    try:
+        with open(path, "r", encoding="utf-8") as fh:
+            stored = json.load(fh)
+    except (OSError, ValueError):
+        return None
+    if not isinstance(stored, dict) or stored.get("key") != key:
+        return None
+    if stored.get("verdict") not in (VERDICT_OK, VERDICT_CRASHED):
+        return None
+    return stored
+
+
+def _write_verdict(path: str, key: dict, verdict: str, detail: str, blobs: "list[str]") -> None:
+    """Atomic replace; a verdict that cannot be written only costs a re-probe."""
+    tmp = f"{path}.tmp-{os.getpid()}"
+    try:
+        with open(tmp, "w", encoding="utf-8") as fh:
+            json.dump({"key": key, "verdict": verdict, "detail": detail, "blobs": blobs}, fh)
+        os.replace(tmp, path)
+    except OSError as exc:
+        sys.stderr.write(f"OpenVINO probe: could not persist the verdict for {path} ({exc})\n")
+        sys.stderr.flush()
+        try:
+            os.remove(tmp)
+        except OSError:
+            pass
+
+
+def _warm(known: dict, cache_dir: str) -> bool:
+    """An `ok` verdict holds only while the blobs its probe wrote are still on disk."""
+    blobs = known.get("blobs")
+    if not isinstance(blobs, list):
+        return False
+    return all(os.path.isfile(os.path.join(cache_dir, str(b))) for b in blobs)
+
+
+def ensure_openvino_loadable(
+    model_path: str,
+    config: dict,
+    candidates: "list[str]",
+    cache_dir: Optional[str],
+    env_tag: str = "",
+    run: Optional[ProbeRunner] = None,
+) -> None:
+    """Return when `model_path` may be compiled in-process; raise when it would crash."""
+    if not needs_probe(candidates):
+        return
+    name = os.path.basename(model_path)
+    device = str(config.get("device", "AUTO"))
+    key = probe_key(model_path, config, candidates, env_tag)
+    verdict_path = probe_verdict_path(model_path, config, cache_dir) if cache_dir else None
+    if verdict_path is not None:
+        known = _read_verdict(verdict_path, key)
+        if known is not None:
+            if known["verdict"] == VERDICT_CRASHED:
+                sys.stderr.write(
+                    f"OpenVINO probe: {name} device={device} is a KNOWN compiler crash — refused\n"
+                )
+                sys.stderr.flush()
+                raise OpenvinoLoadCrashed(model_path, device, str(known.get("detail") or "known crash"))
+            if cache_dir is not None and _warm(known, cache_dir):
+                return  # blob present: the in-process load is a cache hit
+    before = _blob_names(cache_dir) if cache_dir else set()
+    runner = run if run is not None else DEFAULT_RUNNER
+    verdict, detail = runner(model_path, config)
+    sys.stderr.write(f"OpenVINO probe: {name} device={device} -> {verdict} ({detail})\n")
+    sys.stderr.flush()
+    if verdict in (VERDICT_OK, VERDICT_CRASHED) and verdict_path is not None and cache_dir is not None:
+        written = sorted(_blob_names(cache_dir) - before) if verdict == VERDICT_OK else []
+        _write_verdict(verdict_path, key, verdict, detail, written)
+    if verdict == VERDICT_CRASHED:
+        raise OpenvinoLoadCrashed(model_path, device, detail)