@camstack/addon-pipeline 1.1.25 → 1.1.27

package/python/inference_pool.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """Async inference pool — request_id multiplexed, per-runtime concurrency.
 
-Architecture mirrors Scrypted's ML plugins (coreml / openvino / onnx):
+Per-runtime ML backends (coreml / openvino / onnx):
   - asyncio main loop reads requests from stdin; inference is dispatched
     onto a runtime-specific executor so the reader never blocks.
   - Each request carries a 32-bit id; responses are tagged with the same
@@ -44,10 +44,13 @@ import concurrent.futures
 import io
 import json
 import os
+import platform
+import shutil
 import struct
 import sys
 import threading
 import time
+from collections import deque
 from dataclasses import dataclass, field
 from typing import Any, Awaitable, Callable, Optional
 
@@ -73,7 +76,7 @@ MSG_INFER_RAW = 0x02
 # Each item is dispatched concurrently via asyncio.gather so the
 # predict pool's existing parallelism applies; the saving over N
 # separate calls is one IPC round-trip per item collapsed to one,
-# matching Scrypted's batch=4 semantics for fair benchmarking.
+# with batch=4 semantics for fair benchmarking.
 MSG_INFER_BATCH = 0x03
 MSG_CACHE_FRAME = 0x04
 MSG_INFER_CACHED = 0x05
@@ -169,6 +172,272 @@ _ov_optimal_reqs: int = 0
 # query the device's optimal request count). ≈ measured optimal on Intel
 # CPU/iGPU/NPU (4-5). Threads idle-block on infer, so over-provisioning is cheap.
 OV_DEFAULT_CONCURRENCY: int = 4
+# OpenVINO device-mode decision. AUTO mode can hide errors / cause conflicts
+# with NPU and GPU, so we resolve an explicit device and fall back accordingly.
+# The intel-iGPU-only case (the N100) compiles explicit "GPU" — the exact host
+# where the AUTO plugin's mid-flight CPU-helper -> GPU handover invalidated
+# in-flight InferRequest output ports ("Cannot find tensor for port
+# opset1::Result score_8").
+
+
+def _resolve_ov_mode(available: list[str], full_names: dict[str, str]) -> str:
+    """Resolve the device-mode string for an AUTO/default device request.
+
+    `available` is `core.available_devices`; `full_names` maps each device to
+    its FULL_DEVICE_NAME property (entries whose property query failed are
+    absent — a device whose FULL_DEVICE_NAME probe failed is not classified).
+    The branch matrix:
+
+      npu & gpu          -> "AUTO:NPU,GPU,CPU"
+      npu & !gpu         -> "AUTO:NPU,CPU"
+      NVIDIA dGPU(s)     -> "AUTO:<dgpu[,dgpu...]>,CPU"
+      gpu (intel only)   -> "GPU"          (explicit — the N100 case)
+      else               -> "AUTO"
+    """
+    npu = False
+    gpu = False
+    dgpus: list[str] = []
+    for device in available:
+        full_device_name = full_names.get(device)
+        if full_device_name is None:
+            continue
+        if "NVIDIA" in full_device_name and "dGPU" in full_device_name:
+            dgpus.append(device)
+        if "NPU" in device:
+            npu = True
+        if "GPU" in device:
+            gpu = True
+
+    mode = "AUTO"
+    if npu:
+        if gpu:
+            mode = "AUTO:NPU,GPU,CPU"
+        else:
+            mode = "AUTO:NPU,CPU"
+    elif len(dgpus):
+        mode = f"AUTO:{','.join(dgpus)},CPU"
+    # forcing GPU can crash on older GPUs — the GPU-mode compile failure falls
+    # back to AUTO in the compile loop below.
+    elif gpu:
+        mode = "GPU"
+    return mode
+
+
+# ONNX Runtime execution-provider names used by _resolve_onnx_providers.
+ONNX_CUDA_EP = "CUDAExecutionProvider"
+ONNX_COREML_EP = "CoreMLExecutionProvider"
+ONNX_CPU_EP = "CPUExecutionProvider"
+
+
+def _resolve_onnx_providers(
+    device: str,
+    plat: str,
+    machine: str,
+    available: list[str],
+    cuda_device_id: int = 0,
+) -> list:
+    """Ordered ONNX Runtime providers — hardware first, CPU always last.
+
+    Ordering for the AUTO/default case:
+
+      darwin                    -> CoreMLExecutionProvider
+      linux/win on x86_64/AMD64 -> ("CUDAExecutionProvider", {"device_id": N})
+      always                    -> CPUExecutionProvider appended LAST
+
+    Deliberate adaptations for camstack:
+      - an operator device pin is honored: "cuda"/"coreml" put that EP first
+        (CPU still last as fallback); "cpu" compiles CPU only.
+      - hardware entries not present in `available`
+        (ort.get_available_providers()) are dropped — camstack ships the
+        plain `onnxruntime` wheel on most nodes, and requesting an EP the
+        build lacks raises at session creation instead of falling back.
+      - platform match uses startswith so a substring test on "win" never
+        wrongly matches "darwin".
+    """
+    dev = (device or "").strip().lower()
+    ordered: list = []
+    if dev == "cuda":
+        ordered.append((ONNX_CUDA_EP, {"device_id": cuda_device_id}))
+    elif dev == "coreml":
+        ordered.append(ONNX_COREML_EP)
+    elif dev == "cpu":
+        pass  # explicit CPU pin — no hardware EPs
+    else:
+        # AUTO/default — platform-driven ordering.
+        if plat == "darwin":
+            ordered.append(ONNX_COREML_EP)
+        if plat.startswith(("linux", "win")) and machine in ("x86_64", "AMD64"):
+            ordered.append((ONNX_CUDA_EP, {"device_id": cuda_device_id}))
+    providers: list = [
+        p for p in ordered
+        if (p[0] if isinstance(p, tuple) else p) in available
+    ]
+    providers.append(ONNX_CPU_EP)
+    return providers
+
+
+# ---------------------------------------------------------------------------
+# CoreML persistent compiled-model cache
+# ---------------------------------------------------------------------------
+# Known macOS behavior: the OS-level compiled-model cache is NOT reused across
+# process restarts — every
+# `ct.models.MLModel(.mlpackage)` load RECOMPILES the model, and the stale OS
+# cache is only cleared on reboot. camstack loads the .mlpackage on every pool
+# spawn, so each spawn paid the full recompile. Fix: after the first compile,
+# copy the compiled `.mlmodelc` (get_compiled_model_path() returns a TEMP dir
+# that dies with the MLModel object) into a persistent `.coreml-cache` dir next
+# to the model (mirrors the OpenVINO `.ov-cache` convention); subsequent spawns
+# load it via `ct.models.CompiledMLModel(...)` — no recompile. A stamp file
+# fingerprinting the source model invalidates the cache when the model changes.
+# HARD RULE: every failure in this machinery falls back to the plain
+# `ct.models.MLModel(path)` load — exactly the pre-cache behavior.
+
+COREML_CACHE_DIR_NAME = ".coreml-cache"
+COREML_CACHE_STAMP_VERSION = 1
+
+
+def _coreml_cache_paths(model_path: str) -> tuple[str, str]:
+    """(compiled .mlmodelc cache dir, stamp file) for a source model path.
+
+    The cache lives in a dot-dir next to the model file — same convention as
+    the OpenVINO `.ov-cache` — so it ships/wipes with the models dir.
+    """
+    models_dir = os.path.dirname(os.path.abspath(model_path))
+    stem = os.path.splitext(os.path.basename(model_path))[0]
+    cache_root = os.path.join(models_dir, COREML_CACHE_DIR_NAME)
+    return (
+        os.path.join(cache_root, stem + ".mlmodelc"),
+        os.path.join(cache_root, stem + ".stamp.json"),
+    )
+
+
+def _coreml_model_fingerprint(model_path: str) -> dict:
+    """Cheap source-model fingerprint — file count + total size + max mtime.
+
+    Works for both a plain file (.mlmodel) and a .mlpackage DIRECTORY (the
+    normal case), whose top-level mtime alone would not reflect content edits.
+    JSON-serializable so it round-trips through the stamp file unchanged.
+    """
+    if not os.path.isdir(model_path):
+        st = os.stat(model_path)
+        return {"kind": "file", "files": 1, "size": st.st_size, "mtimeNs": st.st_mtime_ns}
+    total_size = 0
+    latest_mtime = 0
+    file_count = 0
+    for root, _dirs, files in os.walk(model_path):
+        for name in files:
+            st = os.stat(os.path.join(root, name))
+            total_size += st.st_size
+            if st.st_mtime_ns > latest_mtime:
+                latest_mtime = st.st_mtime_ns
+            file_count += 1
+    return {"kind": "dir", "files": file_count, "size": total_size, "mtimeNs": latest_mtime}
+
+
+def _coreml_stamp_matches(stamp: Any, fingerprint: dict) -> bool:
+    """Pure decision: does a parsed stamp validate the cached compile?"""
+    return (
+        isinstance(stamp, dict)
+        and stamp.get("version") == COREML_CACHE_STAMP_VERSION
+        and stamp.get("fingerprint") == fingerprint
+    )
+
+
+def _persist_coreml_compiled(
+    compiled_src: str, cache_path: str, stamp_path: str, fingerprint: dict,
+) -> None:
+    """Copy coremltools' TEMP compiled dir into the persistent cache path.
+
+    Crash-safe ordering: copy into a tmp sibling on the SAME filesystem, then
+    `os.replace` into place, and write the stamp LAST — a stamp file only ever
+    exists for a fully-moved `.mlmodelc`, so a crash mid-copy can never
+    produce a half-written cache that validates. Raises on failure — the
+    caller logs and continues WITHOUT a cache (model already loaded fine).
+    """
+    cache_root = os.path.dirname(cache_path)
+    os.makedirs(cache_root, exist_ok=True)
+    tmp_dir = f"{cache_path}.tmp-{os.getpid()}"
+    if os.path.exists(tmp_dir):
+        shutil.rmtree(tmp_dir)
+    try:
+        shutil.copytree(compiled_src, tmp_dir)
+        # Drop any stale previous cache: its stamp first (so nothing validates
+        # the old dir), then the dir itself — POSIX rename cannot replace a
+        # non-empty directory.
+        if os.path.exists(stamp_path):
+            os.remove(stamp_path)
+        if os.path.exists(cache_path):
+            shutil.rmtree(cache_path)
+        os.replace(tmp_dir, cache_path)
+    except BaseException:
+        shutil.rmtree(tmp_dir, ignore_errors=True)
+        raise
+    stamp_tmp = f"{stamp_path}.tmp-{os.getpid()}"
+    try:
+        with open(stamp_tmp, "w", encoding="utf-8") as fh:
+            json.dump({"version": COREML_CACHE_STAMP_VERSION, "fingerprint": fingerprint}, fh)
+        os.replace(stamp_tmp, stamp_path)
+    except BaseException:
+        try:
+            os.remove(stamp_tmp)
+        except OSError:
+            pass
+        raise
+
+
+def _acquire_coreml_model(ct: Any, path: str, compute_units: Any) -> tuple[Any, Any]:
+    """Return (model, spec) for the CoreML branch via the persistent cache.
+
+    Cache HIT  → `CompiledMLModel(cache)` — loads the already-compiled
+                 `.mlmodelc`, no recompile. It has NO `get_spec()`, so the
+                 spec is read straight from the source .mlpackage with
+                 `ct.utils.load_spec` (cheap — parses the protobuf, does not
+                 compile). Spec-derived metadata is therefore identical to
+                 what `MLModel.get_spec()` returned before.
+    Cache MISS → plain `MLModel(path)` (compiles — pre-cache behavior), then
+                 best-effort persist of the compiled dir for the next spawn.
+    ANY failure in the cache machinery → the plain MLModel path, exactly as
+    before this cache existed. Never raises differently from pre-cache code.
+    """
+    try:
+        cache_path, stamp_path = _coreml_cache_paths(path)
+        fingerprint = _coreml_model_fingerprint(path)
+        stamp: Any = None
+        if os.path.isfile(stamp_path):
+            with open(stamp_path, "r", encoding="utf-8") as fh:
+                stamp = json.load(fh)
+        if _coreml_stamp_matches(stamp, fingerprint) and os.path.isdir(cache_path):
+            spec = ct.utils.load_spec(path)
+            model = ct.models.CompiledMLModel(cache_path, compute_units)
+            sys.stderr.write(f"CoreML: compiled-model cache HIT {cache_path}\n")
+            sys.stderr.flush()
+            return model, spec
+    except Exception as exc:
+        sys.stderr.write(
+            f"CoreML: compiled-model cache lookup failed ({exc}); "
+            f"falling back to plain compile of {os.path.basename(path)}\n"
+        )
+        sys.stderr.flush()
+
+    # Cache MISS / invalid / lookup failure — identical to pre-cache behavior.
+    model = ct.models.MLModel(path, compute_units=compute_units)
+    spec = model.get_spec()
+    try:
+        cache_path, stamp_path = _coreml_cache_paths(path)
+        fingerprint = _coreml_model_fingerprint(path)
+        compiled_src = model.get_compiled_model_path()
+        _persist_coreml_compiled(compiled_src, cache_path, stamp_path, fingerprint)
+        sys.stderr.write(
+            f"CoreML: compiled-model cache MISS — compiled and persisted {cache_path}\n"
+        )
+        sys.stderr.flush()
+    except Exception as exc:
+        sys.stderr.write(
+            f"CoreML: could not persist compiled-model cache ({exc}); "
+            f"continuing without cache\n"
+        )
+        sys.stderr.flush()
+    return model, spec
 
 
 def _init_runtime(runtime: str) -> None:
@@ -201,8 +470,28 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
             "ane": ct.ComputeUnit.CPU_AND_NE,
             "all": ct.ComputeUnit.ALL,
         }
-        model = ct.models.MLModel(path, compute_units=compute_map.get(device, ct.ComputeUnit.ALL))
-        spec = model.get_spec()
+        # NOTE: a macOS behavior can cause the .mlpackage to be RECOMPILED on
+        # every load with the compiled cache not reused until reboot. camstack
+        # loads the .mlpackage on
+        # every pool spawn, so it was exposed to the same trap.
+        # _acquire_coreml_model routes through a persistent `.coreml-cache`
+        # of the compiled `.mlmodelc` (CompiledMLModel on hit, plain MLModel
+        # compile + persist on miss) and returns the spec read via
+        # ct.utils.load_spec on hits — falling back to the plain
+        # MLModel/get_spec() path on ANY cache failure.
+        model, spec = _acquire_coreml_model(
+            ct, path, compute_map.get(device, ct.ComputeUnit.ALL),
+        )
+        # Report the compute unit that was ACTUALLY applied so a silent
+        # CPU-only placement is visible in the pool stderr (mirrors the
+        # OpenVINO EXECUTION_DEVICES / ONNX provider reports). Both MLModel
+        # and CompiledMLModel expose `compute_unit`.
+        _actual_cu = getattr(model, "compute_unit", None)
+        sys.stderr.write(
+            f"CoreML: loaded {os.path.basename(path)} device={device} "
+            f"compute_unit={_actual_cu}\n"
+        )
+        sys.stderr.flush()
         input_spec = spec.description.input[0]
         slot.config["_input_name"] = input_spec.name
         slot.config["_input_type"] = input_spec.type.WhichOneof("Type")
@@ -234,16 +523,28 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
 
     elif _runtime == "openvino":
         core = _runtime_lib
-        ov_device = config.get("device", "AUTO").upper()
-        if ov_device == "AUTO":
-            # Scrypted-style device priority, built from the devices OpenVINO
-            # actually enumerates (more accurate than a hardware probe): NPU >
-            # GPU > CPU. The AUTO plugin handles runtime selection + failover.
-            # On a CPU-only image this is just AUTO:CPU; once the Intel GPU/NPU
-            # runtime is present it becomes AUTO:NPU,GPU,CPU automatically.
-            order = [d for d in ("NPU", "GPU", "CPU") if d in core.available_devices]
-            if order:
-                ov_device = "AUTO:" + ",".join(order)
+        ov_device = str(config.get("device", "AUTO")).upper()
+        # Device switch (see _resolve_ov_mode above): an AUTO/default request
+        # classifies the enumerated devices by their FULL_DEVICE_NAME and
+        # builds an explicit mode string. An operator-pinned device (config
+        # "device" = gpu/cpu/npu) is honored unchanged — compiled directly.
+        if ov_device in ("AUTO", "DEFAULT", ""):
+            available = list(core.available_devices)
+            full_names: dict[str, str] = {}
+            for dev in available:
+                try:
+                    full_names[dev] = str(core.get_property(dev, "FULL_DEVICE_NAME"))
+                except Exception:
+                    pass
+            ov_device = _resolve_ov_mode(available, full_names)
+            sys.stderr.write(
+                f"OpenVINO: AUTO resolved to mode {ov_device} "
+                f"(available: {available}, names: {full_names})\n"
+            )
+            sys.stderr.flush()
+        else:
+            sys.stderr.write(f"OpenVINO: operator-pinned device {ov_device}\n")
+            sys.stderr.flush()
         # THROUGHPUT hint lets OpenVINO spin up multiple internal execution
         # streams. Combined with one InferRequest per predict-pool thread
         # (below), concurrent frames from N cameras run in parallel — the
@@ -251,8 +552,52 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
         # every camera serialised through one stream regardless of how many
         # predict workers existed. Measured ~1.5–2.4x throughput on CPU/GPU/NPU.
         ov_config = {"PERFORMANCE_HINT": "THROUGHPUT"}
-        compiled = core.compile_model(path, device_name=ov_device, config=ov_config)
-        output_layers = [compiled.output(i) for i in range(len(compiled.outputs))]
+        # Persist compiled GPU/NPU kernels so the multi-second JIT compile is paid
+        # once (first boot) instead of on every pool spawn. The cold compile storm
+        # (~25s for the full model set on an N100 iGPU) is the window where the AUTO
+        # CPU->GPU handover races (see the index-fetch note below) and frames pile up.
+        try:
+            _cache_dir = os.path.join(os.path.dirname(os.path.abspath(path)), ".ov-cache")
+            os.makedirs(_cache_dir, exist_ok=True)
+            ov_config["CACHE_DIR"] = _cache_dir
+        except OSError:
+            pass
+        # Compile the resolved mode. Failure fallback: any mode containing
+        # "GPU" reverts to plain "AUTO" and recompiles once; if THAT also
+        # fails (or the mode had no GPU), re-raise — the existing load/replace
+        # error handler reports the failure.
+        candidates = [ov_device]
+        if "GPU" in ov_device:
+            candidates.append("AUTO")
+        compiled = None
+        last_exc: Optional[BaseException] = None
+        for candidate in candidates:
+            try:
+                compiled = core.compile_model(path, device_name=candidate, config=ov_config)
+                ov_device = candidate
+                break
+            except Exception as exc:
+                last_exc = exc
+                sys.stderr.write(f"OpenVINO: compile with mode {candidate} failed: {exc}\n")
+                if candidate != candidates[-1]:
+                    sys.stderr.write(f"OpenVINO: {candidate} mode failed, reverting to AUTO.\n")
+                sys.stderr.flush()
+        if compiled is None:
+            if last_exc is not None:
+                raise last_exc
+            raise RuntimeError("OpenVINO: no device available to compile the model")
+        # Name the resolved mode + what OpenVINO actually placed the model on
+        # (the EXECUTION_DEVICES property) so a silent CPU
+        # placement is visible in the pool stderr.
+        try:
+            _exec_devices = compiled.get_property("EXECUTION_DEVICES")
+        except Exception:
+            _exec_devices = "unknown"
+        sys.stderr.write(
+            f"OpenVINO: compiled {os.path.basename(path)} mode={ov_device} "
+            f"EXECUTION_DEVICES={_exec_devices}\n"
+        )
+        sys.stderr.flush()
         output_names = [o.get_any_name() for o in compiled.outputs]
 
         # Record the device's optimal infer-request count so the dispatcher
@@ -270,27 +615,55 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
         # what lets the THROUGHPUT streams run concurrently.
         _ov_tls = threading.local()
 
-        def predict(inp_dict: dict, _c=compiled, _layers=output_layers, _names=output_names, _tls=_ov_tls) -> dict:
+        def predict(inp_dict: dict, _c=compiled, _names=output_names, _tls=_ov_tls) -> dict:
             req = getattr(_tls, "req", None)
             if req is None:
                 req = _tls.req = _c.create_infer_request()
             inp = list(inp_dict.values())[0]
-            result = req.infer(inp)
-            return {name: result[layer] for name, layer in zip(_names, _layers)}
+            try:
+                result = req.infer(inp)
+            except RuntimeError:
+                # AUTO CPU->GPU handover race: a request created during the
+                # CPU-helper phase (while the GPU JIT-compiles) cannot map the
+                # served model's output ports and raises — from INSIDE infer() —
+                # "Cannot find tensor for port ... score_8". (Reproduced live on
+                # the N100 iGPU under concurrent THROUGHPUT threads; the raise is
+                # in req.infer(), not the output fetch below.) Recreate the
+                # thread-local request post-handover and retry once.
+                req = _tls.req = _c.create_infer_request()
+                result = req.infer(inp)
+            # Fetch outputs by positional index (stable across the handover)
+            # rather than ConstOutput port identity — belt-and-braces.
+            return {name: result[i] for i, name in enumerate(_names)}
 
         slot.model = compiled
         slot.predict_fn = predict
 
     elif _runtime == "onnxruntime":
         ort = _runtime_lib
-        ort_device = config.get("device", "cpu")
-        if ort_device == "cuda":
-            providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
-        elif ort_device == "coreml":
-            providers = ["CoreMLExecutionProvider", "CPUExecutionProvider"]
-        else:
-            providers = ["CPUExecutionProvider"]
+        ort_device = str(config.get("device", "") or "")
+        # Ordered providers — hardware EP first, CPU LAST as
+        # fallback (see _resolve_onnx_providers): a "cuda"/"coreml" pin puts
+        # that EP first, a "cpu" pin compiles CPU only, and the AUTO/default
+        # case derives the hardware EP from the platform.
+        try:
+            available_eps = list(ort.get_available_providers())
+        except Exception:
+            available_eps = [ONNX_CPU_EP]
+        providers = _resolve_onnx_providers(
+            ort_device, sys.platform, platform.machine(), available_eps,
+        )
         session = ort.InferenceSession(path, providers=providers)
+        # Report the EP that ACTUALLY initialized (CPU stripped from the log)
+        # so a silent CPU fallback is visible.
+        _active_eps = [p for p in session.get_providers() if p != ONNX_CPU_EP]
+        _requested = [p[0] if isinstance(p, tuple) else p for p in providers]
+        sys.stderr.write(
+            f"ONNX: execution provider {', '.join(_active_eps) or ONNX_CPU_EP} "
+            f"(device={ort_device or 'auto'}, requested={_requested}, "
+            f"available={available_eps})\n"
+        )
+        sys.stderr.flush()
         output_names = [o.name for o in session.get_outputs()]
         slot.config["_input_name"] = session.get_inputs()[0].name
 
@@ -650,6 +1023,85 @@ class RuntimeDispatcher:
         self._predict_pool.shutdown(wait=False)
 
 
+# ---------------------------------------------------------------------------
+# Per-model backpressure — bound in-flight inference, drop-oldest shedding
+# ---------------------------------------------------------------------------
+
+# Frames a single model may hold WAITING behind its running set before the
+# OLDEST waiting frame is shed. Running slots (= predict-pool size) keep the
+# device fed; a couple of pending frames cover dispatch gaps. Anything beyond
+# that only sits in an invisible executor queue inflating per-frame
+# "inference" time — on an overloaded node (N100 iGPU) that queue wait
+# ballooned to 9-22s per frame. Shedding it instead makes overload read as
+# `dropped` frames (fast `"dropped": true` responses) rather than latency.
+MAX_PENDING_PER_MODEL: int = 2
+
+
+@dataclass
+class _ModelQueueState:
+    """Mutable per-model counters for ModelBackpressure."""
+    running: int = 0
+    pending: "deque[Any]" = field(default_factory=deque)
+
+
+class ModelBackpressure:
+    """Per-model in-flight bound with drop-oldest shedding.
+
+    Pure decision core — no asyncio, no I/O — so it is unit-testable
+    (see test_inference_pool_backpressure.py). The event loop calls:
+
+      admit(model_idx, item)  on every single-frame inference arrival.
+        Returns (to_run, to_drop): items to dispatch NOW and items shed
+        (oldest queued first). At most one of the lists is non-empty.
+      complete(model_idx)     when a dispatched inference finishes.
+        Returns the next queued items to dispatch (0 or 1).
+
+    Why per-model and why these bounds: the predict pool has `workers`
+    threads shared by all models; a model gets at most `max_running`
+    concurrently-dispatched inferences (sized to the predict pool), plus
+    a `max_pending` waiting buffer. The previous GLOBAL ceiling
+    (`workers * 4`, min 16) was unreachable from the live pipeline — the
+    TS runner's semaphore keeps at most `maxConcurrentInferences` frames
+    in flight per node, well below 16 — so overload never shed frames; it
+    just queued inside the executor as multi-second "inference" time.
+    """
+
+    def __init__(self, max_running: int, max_pending: int) -> None:
+        self._max_running = max(1, max_running)
+        self._max_pending = max(0, max_pending)
+        self._states: dict[int, _ModelQueueState] = {}
+
+    def _state(self, model_idx: int) -> _ModelQueueState:
+        state = self._states.get(model_idx)
+        if state is None:
+            state = _ModelQueueState()
+            self._states[model_idx] = state
+        return state
+
+    def admit(self, model_idx: int, item: Any) -> tuple[list, list]:
+        """Admit one inference request; returns (to_run, to_drop)."""
+        state = self._state(model_idx)
+        if state.running < self._max_running:
+            state.running += 1
+            return [item], []
+        state.pending.append(item)
+        dropped: list = []
+        while len(state.pending) > self._max_pending:
+            # DROP-OLDEST: the newest frame is the most valuable one for a
+            # live detection pipeline — shed the stalest queued frame(s).
+            dropped.append(state.pending.popleft())
+        return [], dropped
+
+    def complete(self, model_idx: int) -> list:
+        """One running inference finished; returns queued items to start."""
+        state = self._state(model_idx)
+        state.running = max(0, state.running - 1)
+        if state.pending and state.running < self._max_running:
+            state.running += 1
+            return [state.pending.popleft()]
+        return []
+
+
 # ---------------------------------------------------------------------------
 # IPC — binary framing with request_id multiplexing
 # ---------------------------------------------------------------------------
@@ -832,6 +1284,17 @@ async def _run() -> None:
             concurrency = target
 
     dispatcher = RuntimeDispatcher(runtime, concurrency)
+    # Backpressure: bound in-flight single-frame inference PER MODEL, shedding
+    # the OLDEST queued frame when full (see ModelBackpressure). max_running is
+    # sized to the predict pool — dispatching more than `workers` concurrent
+    # inferences for one model can't run any faster, it only queues invisibly
+    # inside the executor and balloons per-frame "inference" time (9-22s
+    # observed on the N100 iGPU). Healthy nodes never fill the pending buffer,
+    # so this is a no-op except under genuine overload.
+    backpressure = ModelBackpressure(
+        max_running=dispatcher.workers,
+        max_pending=MAX_PENDING_PER_MODEL,
+    )
     startup_ms = round((time.perf_counter() - t_start) * 1000)
     loaded_count = sum(1 for s in models if s.loaded)
     sys.stderr.write(
@@ -963,6 +1426,34 @@ async def _run() -> None:
                 "detections": [],
                 "inferenceMs": 0,
             })
+        finally:
+            # Release this model's running slot; promote the next queued
+            # frame (if any) into the freed slot immediately.
+            for next_req_id, next_img in backpressure.complete(model_idx):
+                asyncio.create_task(handle_inference(next_req_id, next_img, model_idx))
+
+    def _send_dropped(req_id: int) -> None:
+        # Fast shed response — returned in microseconds instead of queuing the
+        # frame for seconds. The TS side (shared-inference-pool.ts) recognises
+        # `dropped: true` and counts it, so overload is measurable, not silent.
+        asyncio.create_task(writer.send(req_id, {
+            "kind": "detections",
+            "detections": [],
+            "inferenceMs": 0,
+            "dropped": True,
+        }))
+
+    def _dispatch_inference(req_id: int, img: Image.Image, model_idx: int) -> None:
+        # Gate every single-frame inference through the per-model in-flight
+        # bound. Slot accounting is synchronous (admit here, complete in the
+        # handle_inference `finally`), so a slot can never leak past a task's
+        # lifetime. Under overload the OLDEST queued frame is shed and answered
+        # immediately so the caller never hangs on a frame we chose not to run.
+        to_run, to_drop = backpressure.admit(model_idx, (req_id, img))
+        for dropped_req_id, _dropped_img in to_drop:
+            _send_dropped(dropped_req_id)
+        for run_req_id, run_img in to_run:
+            asyncio.create_task(handle_inference(run_req_id, run_img, model_idx))
 
     async def handle_batch(req_id: int, model_idx: int, items: list[Image.Image]) -> None:
         if model_idx >= len(models) or not models[model_idx].loaded:
@@ -1045,7 +1536,7 @@ async def _run() -> None:
             except Exception as exc:
                 await writer.send(req_id, {"error": f"jpeg decode failed: {exc}"})
                 continue
-            asyncio.create_task(handle_inference(req_id, img, model_idx))
+            _dispatch_inference(req_id, img, model_idx)
 
         elif msg_type == MSG_INFER_RAW:
             if len(payload) < 10:
@@ -1060,7 +1551,7 @@ async def _run() -> None:
             except Exception as exc:
                 await writer.send(req_id, {"error": f"raw wrap failed: {exc}"})
                 continue
-            asyncio.create_task(handle_inference(req_id, img, model_idx))
+            _dispatch_inference(req_id, img, model_idx)
 
         elif msg_type == MSG_INFER_BATCH:
             # Header: [1B model_idx][1B count]
@@ -1130,7 +1621,7 @@ async def _run() -> None:
             if model_idx >= len(models) or not models[model_idx].loaded:
                 await writer.send(req_id, {"error": f"model {model_idx} not loaded"})
                 continue
-            asyncio.create_task(handle_inference(req_id, img, model_idx))
+            _dispatch_inference(req_id, img, model_idx)
 
         else:
             await writer.send(req_id, {"error": f"unknown msg_type: {msg_type}"})