npm - @camstack/addon-pipeline - Versions diffs - 1.1.25 → 1.1.26 - Mend

@camstack/addon-pipeline 1.1.25 → 1.1.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (39) hide show

package/embed-dist/index.html CHANGED Viewed

@@ -4,7 +4,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <title>CamStack Embed</title>
-    <script type="module" crossorigin src="./assets/index-xpPLFfsT.js"></script>
+    <script type="module" crossorigin src="./assets/index-C-pL8ETk.js"></script>
     <link rel="stylesheet" crossorigin href="./assets/index-DrJ0ee3f.css">
   </head>
   <body>

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@camstack/addon-pipeline",
-  "version": "1.1.25",
+  "version": "1.1.26",
   "description": "CamStack Pipeline bundle — runner, detection, motion, decoders, audio + stream broker. Multi-entry npm package shipping 7 addons under a single bundle.",
   "keywords": [
     "camstack",

package/python/inference_pool.py CHANGED Viewed

@@ -44,10 +44,13 @@ import concurrent.futures
 import io
 import json
 import os
+import platform
+import shutil
 import struct
 import sys
 import threading
 import time
+from collections import deque
 from dataclasses import dataclass, field
 from typing import Any, Awaitable, Callable, Optional
@@ -169,6 +172,274 @@ _ov_optimal_reqs: int = 0
 # query the device's optimal request count). ≈ measured optimal on Intel
 # CPU/iGPU/NPU (4-5). Threads idle-block on infer, so over-provisioning is cheap.
 OV_DEFAULT_CONCURRENCY: int = 4
+# OpenVINO device-mode decision — EXACT port of Scrypted's OpenVINO plugin
+# device switch (scrypted-ov-init.py:109-194). Scrypted's rationale, kept
+# verbatim: "AUTO mode can cause conflicts or hide errors with NPU and GPU
+# so try to be explicit and fall back accordingly." The intel-iGPU-only case
+# (the N100) compiles explicit "GPU" — the exact host where the AUTO plugin's
+# mid-flight CPU-helper -> GPU handover invalidated in-flight InferRequest
+# output ports ("Cannot find tensor for port opset1::Result score_8").
+def _resolve_ov_mode(available: list[str], full_names: dict[str, str]) -> str:
+    """Scrypted's exact mode string for an AUTO/default device request.
+    `available` is `core.available_devices`; `full_names` maps each device to
+    its FULL_DEVICE_NAME property (entries whose property query failed are
+    absent — mirrors Scrypted's per-device `except: pass`, so such a device
+    is not classified). The branch matrix, byte-faithful to Scrypted:
+      npu & gpu          -> "AUTO:NPU,GPU,CPU"
+      npu & !gpu         -> "AUTO:NPU,CPU"
+      NVIDIA dGPU(s)     -> "AUTO:<dgpu[,dgpu...]>,CPU"
+      gpu (intel only)   -> "GPU"          (explicit — the N100 case)
+      else               -> "AUTO"
+    """
+    npu = False
+    gpu = False
+    dgpus: list[str] = []
+    for device in available:
+        full_device_name = full_names.get(device)
+        if full_device_name is None:
+            continue
+        if "NVIDIA" in full_device_name and "dGPU" in full_device_name:
+            dgpus.append(device)
+        if "NPU" in device:
+            npu = True
+        if "GPU" in device:
+            gpu = True
+    mode = "AUTO"
+    if npu:
+        if gpu:
+            mode = "AUTO:NPU,GPU,CPU"
+        else:
+            mode = "AUTO:NPU,CPU"
+    elif len(dgpus):
+        mode = f"AUTO:{','.join(dgpus)},CPU"
+    # forcing GPU can cause crashes on older GPU. (Scrypted's comment — the
+    # GPU-mode compile failure falls back to AUTO in the compile loop below.)
+    elif gpu:
+        mode = "GPU"
+    return mode
+# ONNX Runtime execution-provider names used by _resolve_onnx_providers.
+ONNX_CUDA_EP = "CUDAExecutionProvider"
+ONNX_COREML_EP = "CoreMLExecutionProvider"
+ONNX_CPU_EP = "CPUExecutionProvider"
+def _resolve_onnx_providers(
+    device: str,
+    plat: str,
+    machine: str,
+    available: list[str],
+    cuda_device_id: int = 0,
+) -> list:
+    """Ordered ONNX Runtime providers — hardware first, CPU always last.
+    Mirrors Scrypted's ONNX plugin ordering (scrypted-onnx-init.py:95-109)
+    for the AUTO/default case:
+      darwin                    -> CoreMLExecutionProvider
+      linux/win on x86_64/AMD64 -> ("CUDAExecutionProvider", {"device_id": N})
+      always                    -> CPUExecutionProvider appended LAST
+    Deliberate adaptations for camstack:
+      - an operator device pin is honored: "cuda"/"coreml" put that EP first
+        (CPU still last as fallback); "cpu" compiles CPU only.
+      - hardware entries not present in `available`
+        (ort.get_available_providers()) are dropped — camstack ships the
+        plain `onnxruntime` wheel on most nodes, and requesting an EP the
+        build lacks raises at session creation instead of falling back.
+      - platform match uses startswith so "darwin" never matches "win"
+        (Scrypted's `"win" in sys.platform` would).
+    """
+    dev = (device or "").strip().lower()
+    ordered: list = []
+    if dev == "cuda":
+        ordered.append((ONNX_CUDA_EP, {"device_id": cuda_device_id}))
+    elif dev == "coreml":
+        ordered.append(ONNX_COREML_EP)
+    elif dev == "cpu":
+        pass  # explicit CPU pin — no hardware EPs
+    else:
+        # AUTO/default — Scrypted's platform-driven ordering.
+        if plat == "darwin":
+            ordered.append(ONNX_COREML_EP)
+        if plat.startswith(("linux", "win")) and machine in ("x86_64", "AMD64"):
+            ordered.append((ONNX_CUDA_EP, {"device_id": cuda_device_id}))
+    providers: list = [
+        p for p in ordered
+        if (p[0] if isinstance(p, tuple) else p) in available
+    ]
+    providers.append(ONNX_CPU_EP)
+    return providers
+# ---------------------------------------------------------------------------
+# CoreML persistent compiled-model cache
+# ---------------------------------------------------------------------------
+# macOS bug (documented by Scrypted, coreml-init comment): the OS-level
+# compiled-model cache is NOT reused across process restarts — every
+# `ct.models.MLModel(.mlpackage)` load RECOMPILES the model, and the stale OS
+# cache is only cleared on reboot. camstack loads the .mlpackage on every pool
+# spawn, so each spawn paid the full recompile. Fix: after the first compile,
+# copy the compiled `.mlmodelc` (get_compiled_model_path() returns a TEMP dir
+# that dies with the MLModel object) into a persistent `.coreml-cache` dir next
+# to the model (mirrors the OpenVINO `.ov-cache` convention); subsequent spawns
+# load it via `ct.models.CompiledMLModel(...)` — no recompile. A stamp file
+# fingerprinting the source model invalidates the cache when the model changes.
+# HARD RULE: every failure in this machinery falls back to the plain
+# `ct.models.MLModel(path)` load — exactly the pre-cache behavior.
+COREML_CACHE_DIR_NAME = ".coreml-cache"
+COREML_CACHE_STAMP_VERSION = 1
+def _coreml_cache_paths(model_path: str) -> tuple[str, str]:
+    """(compiled .mlmodelc cache dir, stamp file) for a source model path.
+    The cache lives in a dot-dir next to the model file — same convention as
+    the OpenVINO `.ov-cache` — so it ships/wipes with the models dir.
+    """
+    models_dir = os.path.dirname(os.path.abspath(model_path))
+    stem = os.path.splitext(os.path.basename(model_path))[0]
+    cache_root = os.path.join(models_dir, COREML_CACHE_DIR_NAME)
+    return (
+        os.path.join(cache_root, stem + ".mlmodelc"),
+        os.path.join(cache_root, stem + ".stamp.json"),
+    )
+def _coreml_model_fingerprint(model_path: str) -> dict:
+    """Cheap source-model fingerprint — file count + total size + max mtime.
+    Works for both a plain file (.mlmodel) and a .mlpackage DIRECTORY (the
+    normal case), whose top-level mtime alone would not reflect content edits.
+    JSON-serializable so it round-trips through the stamp file unchanged.
+    """
+    if not os.path.isdir(model_path):
+        st = os.stat(model_path)
+        return {"kind": "file", "files": 1, "size": st.st_size, "mtimeNs": st.st_mtime_ns}
+    total_size = 0
+    latest_mtime = 0
+    file_count = 0
+    for root, _dirs, files in os.walk(model_path):
+        for name in files:
+            st = os.stat(os.path.join(root, name))
+            total_size += st.st_size
+            if st.st_mtime_ns > latest_mtime:
+                latest_mtime = st.st_mtime_ns
+            file_count += 1
+    return {"kind": "dir", "files": file_count, "size": total_size, "mtimeNs": latest_mtime}
+def _coreml_stamp_matches(stamp: Any, fingerprint: dict) -> bool:
+    """Pure decision: does a parsed stamp validate the cached compile?"""
+    return (
+        isinstance(stamp, dict)
+        and stamp.get("version") == COREML_CACHE_STAMP_VERSION
+        and stamp.get("fingerprint") == fingerprint
+    )
+def _persist_coreml_compiled(
+    compiled_src: str, cache_path: str, stamp_path: str, fingerprint: dict,
+) -> None:
+    """Copy coremltools' TEMP compiled dir into the persistent cache path.
+    Crash-safe ordering: copy into a tmp sibling on the SAME filesystem, then
+    `os.replace` into place, and write the stamp LAST — a stamp file only ever
+    exists for a fully-moved `.mlmodelc`, so a crash mid-copy can never
+    produce a half-written cache that validates. Raises on failure — the
+    caller logs and continues WITHOUT a cache (model already loaded fine).
+    """
+    cache_root = os.path.dirname(cache_path)
+    os.makedirs(cache_root, exist_ok=True)
+    tmp_dir = f"{cache_path}.tmp-{os.getpid()}"
+    if os.path.exists(tmp_dir):
+        shutil.rmtree(tmp_dir)
+    try:
+        shutil.copytree(compiled_src, tmp_dir)
+        # Drop any stale previous cache: its stamp first (so nothing validates
+        # the old dir), then the dir itself — POSIX rename cannot replace a
+        # non-empty directory.
+        if os.path.exists(stamp_path):
+            os.remove(stamp_path)
+        if os.path.exists(cache_path):
+            shutil.rmtree(cache_path)
+        os.replace(tmp_dir, cache_path)
+    except BaseException:
+        shutil.rmtree(tmp_dir, ignore_errors=True)
+        raise
+    stamp_tmp = f"{stamp_path}.tmp-{os.getpid()}"
+    try:
+        with open(stamp_tmp, "w", encoding="utf-8") as fh:
+            json.dump({"version": COREML_CACHE_STAMP_VERSION, "fingerprint": fingerprint}, fh)
+        os.replace(stamp_tmp, stamp_path)
+    except BaseException:
+        try:
+            os.remove(stamp_tmp)
+        except OSError:
+            pass
+        raise
+def _acquire_coreml_model(ct: Any, path: str, compute_units: Any) -> tuple[Any, Any]:
+    """Return (model, spec) for the CoreML branch via the persistent cache.
+    Cache HIT  → `CompiledMLModel(cache)` — loads the already-compiled
+                 `.mlmodelc`, no recompile. It has NO `get_spec()`, so the
+                 spec is read straight from the source .mlpackage with
+                 `ct.utils.load_spec` (cheap — parses the protobuf, does not
+                 compile). Spec-derived metadata is therefore identical to
+                 what `MLModel.get_spec()` returned before.
+    Cache MISS → plain `MLModel(path)` (compiles — pre-cache behavior), then
+                 best-effort persist of the compiled dir for the next spawn.
+    ANY failure in the cache machinery → the plain MLModel path, exactly as
+    before this cache existed. Never raises differently from pre-cache code.
+    """
+    try:
+        cache_path, stamp_path = _coreml_cache_paths(path)
+        fingerprint = _coreml_model_fingerprint(path)
+        stamp: Any = None
+        if os.path.isfile(stamp_path):
+            with open(stamp_path, "r", encoding="utf-8") as fh:
+                stamp = json.load(fh)
+        if _coreml_stamp_matches(stamp, fingerprint) and os.path.isdir(cache_path):
+            spec = ct.utils.load_spec(path)
+            model = ct.models.CompiledMLModel(cache_path, compute_units)
+            sys.stderr.write(f"CoreML: compiled-model cache HIT {cache_path}\n")
+            sys.stderr.flush()
+            return model, spec
+    except Exception as exc:
+        sys.stderr.write(
+            f"CoreML: compiled-model cache lookup failed ({exc}); "
+            f"falling back to plain compile of {os.path.basename(path)}\n"
+        )
+        sys.stderr.flush()
+    # Cache MISS / invalid / lookup failure — identical to pre-cache behavior.
+    model = ct.models.MLModel(path, compute_units=compute_units)
+    spec = model.get_spec()
+    try:
+        cache_path, stamp_path = _coreml_cache_paths(path)
+        fingerprint = _coreml_model_fingerprint(path)
+        compiled_src = model.get_compiled_model_path()
+        _persist_coreml_compiled(compiled_src, cache_path, stamp_path, fingerprint)
+        sys.stderr.write(
+            f"CoreML: compiled-model cache MISS — compiled and persisted {cache_path}\n"
+        )
+        sys.stderr.flush()
+    except Exception as exc:
+        sys.stderr.write(
+            f"CoreML: could not persist compiled-model cache ({exc}); "
+            f"continuing without cache\n"
+        )
+        sys.stderr.flush()
+    return model, spec
 def _init_runtime(runtime: str) -> None:
@@ -201,8 +472,28 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
             "ane": ct.ComputeUnit.CPU_AND_NE,
             "all": ct.ComputeUnit.ALL,
         }
-        model = ct.models.MLModel(path, compute_units=compute_map.get(device, ct.ComputeUnit.ALL))
-        spec = model.get_spec()
+        # NOTE (Scrypted caveat, coreml-init comment): a macOS bug can cause
+        # the .mlpackage to be RECOMPILED on every load with the compiled
+        # cache not reused until reboot. camstack loads the .mlpackage on
+        # every pool spawn, so it was exposed to the same trap.
+        # _acquire_coreml_model routes through a persistent `.coreml-cache`
+        # of the compiled `.mlmodelc` (CompiledMLModel on hit, plain MLModel
+        # compile + persist on miss) and returns the spec read via
+        # ct.utils.load_spec on hits — falling back to the plain
+        # MLModel/get_spec() path on ANY cache failure.
+        model, spec = _acquire_coreml_model(
+            ct, path, compute_map.get(device, ct.ComputeUnit.ALL),
+        )
+        # Report the compute unit that was ACTUALLY applied so a silent
+        # CPU-only placement is visible in the pool stderr (mirrors the
+        # OpenVINO EXECUTION_DEVICES / ONNX provider reports). Both MLModel
+        # and CompiledMLModel expose `compute_unit`.
+        _actual_cu = getattr(model, "compute_unit", None)
+        sys.stderr.write(
+            f"CoreML: loaded {os.path.basename(path)} device={device} "
+            f"compute_unit={_actual_cu}\n"
+        )
+        sys.stderr.flush()
         input_spec = spec.description.input[0]
         slot.config["_input_name"] = input_spec.name
         slot.config["_input_type"] = input_spec.type.WhichOneof("Type")
@@ -234,16 +525,29 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
     elif _runtime == "openvino":
         core = _runtime_lib
-        ov_device = config.get("device", "AUTO").upper()
-        if ov_device == "AUTO":
-            # Scrypted-style device priority, built from the devices OpenVINO
-            # actually enumerates (more accurate than a hardware probe): NPU >
-            # GPU > CPU. The AUTO plugin handles runtime selection + failover.
-            # On a CPU-only image this is just AUTO:CPU; once the Intel GPU/NPU
-            # runtime is present it becomes AUTO:NPU,GPU,CPU automatically.
-            order = [d for d in ("NPU", "GPU", "CPU") if d in core.available_devices]
-            if order:
-                ov_device = "AUTO:" + ",".join(order)
+        ov_device = str(config.get("device", "AUTO")).upper()
+        # Scrypted's exact device switch (see _resolve_ov_mode above): an
+        # AUTO/default request classifies the enumerated devices by their
+        # FULL_DEVICE_NAME and builds the same mode string Scrypted's
+        # OpenVINO plugin does. An operator-pinned device (config "device" =
+        # gpu/cpu/npu) is honored unchanged — compiled directly.
+        if ov_device in ("AUTO", "DEFAULT", ""):
+            available = list(core.available_devices)
+            full_names: dict[str, str] = {}
+            for dev in available:
+                try:
+                    full_names[dev] = str(core.get_property(dev, "FULL_DEVICE_NAME"))
+                except Exception:
+                    pass
+            ov_device = _resolve_ov_mode(available, full_names)
+            sys.stderr.write(
+                f"OpenVINO: AUTO resolved to mode {ov_device} "
+                f"(available: {available}, names: {full_names})\n"
+            )
+            sys.stderr.flush()
+        else:
+            sys.stderr.write(f"OpenVINO: operator-pinned device {ov_device}\n")
+            sys.stderr.flush()
         # THROUGHPUT hint lets OpenVINO spin up multiple internal execution
         # streams. Combined with one InferRequest per predict-pool thread
         # (below), concurrent frames from N cameras run in parallel — the
@@ -251,8 +555,53 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
         # every camera serialised through one stream regardless of how many
         # predict workers existed. Measured ~1.5–2.4x throughput on CPU/GPU/NPU.
         ov_config = {"PERFORMANCE_HINT": "THROUGHPUT"}
-        compiled = core.compile_model(path, device_name=ov_device, config=ov_config)
-        output_layers = [compiled.output(i) for i in range(len(compiled.outputs))]
+        # Persist compiled GPU/NPU kernels so the multi-second JIT compile is paid
+        # once (first boot) instead of on every pool spawn. The cold compile storm
+        # (~25s for the full model set on an N100 iGPU) is the window where the AUTO
+        # CPU->GPU handover races (see the index-fetch note below) and frames pile up.
+        try:
+            _cache_dir = os.path.join(os.path.dirname(os.path.abspath(path)), ".ov-cache")
+            os.makedirs(_cache_dir, exist_ok=True)
+            ov_config["CACHE_DIR"] = _cache_dir
+        except OSError:
+            pass
+        # Compile the resolved mode. Failure fallback is Scrypted's, exactly:
+        # any mode containing "GPU" reverts to plain "AUTO" and recompiles
+        # once; if THAT also fails (or the mode had no GPU), re-raise —
+        # camstack has no reset-and-restart like Scrypted, so the existing
+        # load/replace error handler reports the failure.
+        candidates = [ov_device]
+        if "GPU" in ov_device:
+            candidates.append("AUTO")
+        compiled = None
+        last_exc: Optional[BaseException] = None
+        for candidate in candidates:
+            try:
+                compiled = core.compile_model(path, device_name=candidate, config=ov_config)
+                ov_device = candidate
+                break
+            except Exception as exc:
+                last_exc = exc
+                sys.stderr.write(f"OpenVINO: compile with mode {candidate} failed: {exc}\n")
+                if candidate != candidates[-1]:
+                    sys.stderr.write(f"OpenVINO: {candidate} mode failed, reverting to AUTO.\n")
+                sys.stderr.flush()
+        if compiled is None:
+            if last_exc is not None:
+                raise last_exc
+            raise RuntimeError("OpenVINO: no device available to compile the model")
+        # Name the resolved mode + what OpenVINO actually placed the model on
+        # (mirrors Scrypted's EXECUTION_DEVICES print) so a silent CPU
+        # placement is visible in the pool stderr.
+        try:
+            _exec_devices = compiled.get_property("EXECUTION_DEVICES")
+        except Exception:
+            _exec_devices = "unknown"
+        sys.stderr.write(
+            f"OpenVINO: compiled {os.path.basename(path)} mode={ov_device} "
+            f"EXECUTION_DEVICES={_exec_devices}\n"
+        )
+        sys.stderr.flush()
         output_names = [o.get_any_name() for o in compiled.outputs]
         # Record the device's optimal infer-request count so the dispatcher
@@ -270,27 +619,55 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
         # what lets the THROUGHPUT streams run concurrently.
         _ov_tls = threading.local()
-        def predict(inp_dict: dict, _c=compiled, _layers=output_layers, _names=output_names, _tls=_ov_tls) -> dict:
+        def predict(inp_dict: dict, _c=compiled, _names=output_names, _tls=_ov_tls) -> dict:
             req = getattr(_tls, "req", None)
             if req is None:
                 req = _tls.req = _c.create_infer_request()
             inp = list(inp_dict.values())[0]
-            result = req.infer(inp)
-            return {name: result[layer] for name, layer in zip(_names, _layers)}
+            try:
+                result = req.infer(inp)
+            except RuntimeError:
+                # AUTO CPU->GPU handover race: a request created during the
+                # CPU-helper phase (while the GPU JIT-compiles) cannot map the
+                # served model's output ports and raises — from INSIDE infer() —
+                # "Cannot find tensor for port ... score_8". (Reproduced live on
+                # the N100 iGPU under concurrent THROUGHPUT threads; the raise is
+                # in req.infer(), not the output fetch below.) Recreate the
+                # thread-local request post-handover and retry once.
+                req = _tls.req = _c.create_infer_request()
+                result = req.infer(inp)
+            # Fetch outputs by positional index (stable across the handover)
+            # rather than ConstOutput port identity — belt-and-braces.
+            return {name: result[i] for i, name in enumerate(_names)}
         slot.model = compiled
         slot.predict_fn = predict
     elif _runtime == "onnxruntime":
         ort = _runtime_lib
-        ort_device = config.get("device", "cpu")
-        if ort_device == "cuda":
-            providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
-        elif ort_device == "coreml":
-            providers = ["CoreMLExecutionProvider", "CPUExecutionProvider"]
-        else:
-            providers = ["CPUExecutionProvider"]
+        ort_device = str(config.get("device", "") or "")
+        # Scrypted-style ordered providers — hardware EP first, CPU LAST as
+        # fallback (see _resolve_onnx_providers): a "cuda"/"coreml" pin puts
+        # that EP first, a "cpu" pin compiles CPU only, and the AUTO/default
+        # case derives the hardware EP from the platform.
+        try:
+            available_eps = list(ort.get_available_providers())
+        except Exception:
+            available_eps = [ONNX_CPU_EP]
+        providers = _resolve_onnx_providers(
+            ort_device, sys.platform, platform.machine(), available_eps,
+        )
         session = ort.InferenceSession(path, providers=providers)
+        # Report the EP that ACTUALLY initialized — Scrypted strips CPU and
+        # logs the remainder — so a silent CPU fallback is visible.
+        _active_eps = [p for p in session.get_providers() if p != ONNX_CPU_EP]
+        _requested = [p[0] if isinstance(p, tuple) else p for p in providers]
+        sys.stderr.write(
+            f"ONNX: execution provider {', '.join(_active_eps) or ONNX_CPU_EP} "
+            f"(device={ort_device or 'auto'}, requested={_requested}, "
+            f"available={available_eps})\n"
+        )
+        sys.stderr.flush()
         output_names = [o.name for o in session.get_outputs()]
         slot.config["_input_name"] = session.get_inputs()[0].name
@@ -650,6 +1027,85 @@ class RuntimeDispatcher:
         self._predict_pool.shutdown(wait=False)
+# ---------------------------------------------------------------------------
+# Per-model backpressure — bound in-flight inference, drop-oldest shedding
+# ---------------------------------------------------------------------------
+# Frames a single model may hold WAITING behind its running set before the
+# OLDEST waiting frame is shed. Running slots (= predict-pool size) keep the
+# device fed; a couple of pending frames cover dispatch gaps. Anything beyond
+# that only sits in an invisible executor queue inflating per-frame
+# "inference" time — on an overloaded node (N100 iGPU) that queue wait
+# ballooned to 9-22s per frame. Shedding it instead makes overload read as
+# `dropped` frames (fast `"dropped": true` responses) rather than latency.
+MAX_PENDING_PER_MODEL: int = 2
+@dataclass
+class _ModelQueueState:
+    """Mutable per-model counters for ModelBackpressure."""
+    running: int = 0
+    pending: "deque[Any]" = field(default_factory=deque)
+class ModelBackpressure:
+    """Per-model in-flight bound with drop-oldest shedding.
+    Pure decision core — no asyncio, no I/O — so it is unit-testable
+    (see test_inference_pool_backpressure.py). The event loop calls:
+      admit(model_idx, item)  on every single-frame inference arrival.
+        Returns (to_run, to_drop): items to dispatch NOW and items shed
+        (oldest queued first). At most one of the lists is non-empty.
+      complete(model_idx)     when a dispatched inference finishes.
+        Returns the next queued items to dispatch (0 or 1).
+    Why per-model and why these bounds: the predict pool has `workers`
+    threads shared by all models; a model gets at most `max_running`
+    concurrently-dispatched inferences (sized to the predict pool), plus
+    a `max_pending` waiting buffer. The previous GLOBAL ceiling
+    (`workers * 4`, min 16) was unreachable from the live pipeline — the
+    TS runner's semaphore keeps at most `maxConcurrentInferences` frames
+    in flight per node, well below 16 — so overload never shed frames; it
+    just queued inside the executor as multi-second "inference" time.
+    """
+    def __init__(self, max_running: int, max_pending: int) -> None:
+        self._max_running = max(1, max_running)
+        self._max_pending = max(0, max_pending)
+        self._states: dict[int, _ModelQueueState] = {}
+    def _state(self, model_idx: int) -> _ModelQueueState:
+        state = self._states.get(model_idx)
+        if state is None:
+            state = _ModelQueueState()
+            self._states[model_idx] = state
+        return state
+    def admit(self, model_idx: int, item: Any) -> tuple[list, list]:
+        """Admit one inference request; returns (to_run, to_drop)."""
+        state = self._state(model_idx)
+        if state.running < self._max_running:
+            state.running += 1
+            return [item], []
+        state.pending.append(item)
+        dropped: list = []
+        while len(state.pending) > self._max_pending:
+            # DROP-OLDEST: the newest frame is the most valuable one for a
+            # live detection pipeline — shed the stalest queued frame(s).
+            dropped.append(state.pending.popleft())
+        return [], dropped
+    def complete(self, model_idx: int) -> list:
+        """One running inference finished; returns queued items to start."""
+        state = self._state(model_idx)
+        state.running = max(0, state.running - 1)
+        if state.pending and state.running < self._max_running:
+            state.running += 1
+            return [state.pending.popleft()]
+        return []
 # ---------------------------------------------------------------------------
 # IPC — binary framing with request_id multiplexing
 # ---------------------------------------------------------------------------
@@ -832,6 +1288,17 @@ async def _run() -> None:
             concurrency = target
     dispatcher = RuntimeDispatcher(runtime, concurrency)
+    # Backpressure: bound in-flight single-frame inference PER MODEL, shedding
+    # the OLDEST queued frame when full (see ModelBackpressure). max_running is
+    # sized to the predict pool — dispatching more than `workers` concurrent
+    # inferences for one model can't run any faster, it only queues invisibly
+    # inside the executor and balloons per-frame "inference" time (9-22s
+    # observed on the N100 iGPU). Healthy nodes never fill the pending buffer,
+    # so this is a no-op except under genuine overload.
+    backpressure = ModelBackpressure(
+        max_running=dispatcher.workers,
+        max_pending=MAX_PENDING_PER_MODEL,
+    )
     startup_ms = round((time.perf_counter() - t_start) * 1000)
     loaded_count = sum(1 for s in models if s.loaded)
     sys.stderr.write(
@@ -963,6 +1430,34 @@ async def _run() -> None:
                 "detections": [],
                 "inferenceMs": 0,
             })
+        finally:
+            # Release this model's running slot; promote the next queued
+            # frame (if any) into the freed slot immediately.
+            for next_req_id, next_img in backpressure.complete(model_idx):
+                asyncio.create_task(handle_inference(next_req_id, next_img, model_idx))
+    def _send_dropped(req_id: int) -> None:
+        # Fast shed response — returned in microseconds instead of queuing the
+        # frame for seconds. The TS side (shared-inference-pool.ts) recognises
+        # `dropped: true` and counts it, so overload is measurable, not silent.
+        asyncio.create_task(writer.send(req_id, {
+            "kind": "detections",
+            "detections": [],
+            "inferenceMs": 0,
+            "dropped": True,
+        }))
+    def _dispatch_inference(req_id: int, img: Image.Image, model_idx: int) -> None:
+        # Gate every single-frame inference through the per-model in-flight
+        # bound. Slot accounting is synchronous (admit here, complete in the
+        # handle_inference `finally`), so a slot can never leak past a task's
+        # lifetime. Under overload the OLDEST queued frame is shed and answered
+        # immediately so the caller never hangs on a frame we chose not to run.
+        to_run, to_drop = backpressure.admit(model_idx, (req_id, img))
+        for dropped_req_id, _dropped_img in to_drop:
+            _send_dropped(dropped_req_id)
+        for run_req_id, run_img in to_run:
+            asyncio.create_task(handle_inference(run_req_id, run_img, model_idx))
     async def handle_batch(req_id: int, model_idx: int, items: list[Image.Image]) -> None:
         if model_idx >= len(models) or not models[model_idx].loaded:
@@ -1045,7 +1540,7 @@ async def _run() -> None:
             except Exception as exc:
                 await writer.send(req_id, {"error": f"jpeg decode failed: {exc}"})
                 continue
-            asyncio.create_task(handle_inference(req_id, img, model_idx))
+            _dispatch_inference(req_id, img, model_idx)
         elif msg_type == MSG_INFER_RAW:
             if len(payload) < 10:
@@ -1060,7 +1555,7 @@ async def _run() -> None:
             except Exception as exc:
                 await writer.send(req_id, {"error": f"raw wrap failed: {exc}"})
                 continue
-            asyncio.create_task(handle_inference(req_id, img, model_idx))
+            _dispatch_inference(req_id, img, model_idx)
         elif msg_type == MSG_INFER_BATCH:
             # Header: [1B model_idx][1B count]
@@ -1130,7 +1625,7 @@ async def _run() -> None:
             if model_idx >= len(models) or not models[model_idx].loaded:
                 await writer.send(req_id, {"error": f"model {model_idx} not loaded"})
                 continue
-            asyncio.create_task(handle_inference(req_id, img, model_idx))
+            _dispatch_inference(req_id, img, model_idx)
         else:
             await writer.send(req_id, {"error": f"unknown msg_type: {msg_type}"})