@camstack/addon-pipeline 1.2.294 → 1.2.296

package/python/inference_pool.py

@@ -52,6 +52,9 @@ host and gets byte-identical v1 behaviour; skew in either direction is the
 NORMAL state during a rolling deploy.
 
 Commands: load, unload, replace, reconfigure, status, uncache_frame, mem_stats.
+`load`/`replace` compile on their own thread under a hard bound and never block
+the loop; a load past the bound answers `reason: "compile-timeout"` and poisons
+the worker (D653, see `ModelLoader`).
 """
 from __future__ import annotations
 
@@ -68,6 +71,7 @@ import re
 import shutil
 import struct
 import sys
+import threading
 import time
 from collections import deque
 from dataclasses import dataclass, field
@@ -2358,6 +2362,7 @@ def _mem_stats_payload(
     preprocess_cache_len: int,
     bench_cache_len: int,
     uptime_sec: int,
+    model_load: Optional[dict] = None,
 ) -> dict:
     """Assemble the mem_stats reply. Pure given its inputs except the two
     process-wide reads (memory + arenas), so the shape is unit-testable."""
@@ -2374,6 +2379,9 @@ def _mem_stats_payload(
         "preprocessCacheEntries": preprocess_cache_len,
         "benchPreprocessCacheEntries": bench_cache_len,
         "uptimeSec": uptime_sec,
+        # What is compiling, and whether a compile ever failed to return
+        # (D653) — the one fact a host needs to tell "slow" from "poisoned".
+        "modelLoad": model_load,
     }
 
 
@@ -2486,25 +2494,13 @@ RECONFIGURABLE_CONFIG_KEYS = frozenset({"nmsIouThreshold"})
 
 
 def _handle_command(models: list[ModelSlot], cmd: dict) -> dict:
-    action = cmd.get("cmd")
-    if action == "load":
-        index = cmd["index"]
-        config = cmd["config"]
-        while len(models) <= index:
-            models.append(ModelSlot())
-        slot = models[index]
-        if slot.loaded:
-            _unload_model(slot)
-        try:
-            t0 = time.perf_counter()
-            _load_model(slot, config)
-            load_ms = round((time.perf_counter() - t0) * 1000)
-            sys.stderr.write(f"Model {index} loaded: {config['path']} ({load_ms}ms)\n")
-            sys.stderr.flush()
-            return {"cmd": "load", "index": index, "status": "ok", "loadMs": load_ms}
-        except Exception as exc:
-            return {"cmd": "load", "index": index, "status": "error", "error": str(exc)}
+    """The model commands that need no compile: unload, reconfigure, status.
 
+    `load` and `replace` are NOT here — they compile, and a compile must never
+    run on the event loop (D653). They go through `_handle_model_command`,
+    which the `CommandRouter` awaits off the loop's reader.
+    """
+    action = cmd.get("cmd")
     if action == "unload":
         index = cmd["index"]
         if index < len(models) and models[index].loaded:
@@ -2550,24 +2546,6 @@ def _handle_command(models: list[ModelSlot], cmd: dict) -> dict:
         sys.stderr.flush()
         return {"cmd": "reconfigure", "index": index, "status": "ok"}
 
-    if action == "replace":
-        index = cmd["index"]
-        config = cmd["config"]
-        while len(models) <= index:
-            models.append(ModelSlot())
-        slot = models[index]
-        if slot.loaded:
-            _unload_model(slot)
-        try:
-            t0 = time.perf_counter()
-            _load_model(slot, config)
-            load_ms = round((time.perf_counter() - t0) * 1000)
-            sys.stderr.write(f"Model {index} replaced: {config['path']} ({load_ms}ms)\n")
-            sys.stderr.flush()
-            return {"cmd": "replace", "index": index, "status": "ok", "loadMs": load_ms}
-        except Exception as exc:
-            return {"cmd": "replace", "index": index, "status": "error", "error": str(exc)}
-
     if action == "status":
         status = []
         for i, slot in enumerate(models):
@@ -2582,6 +2560,374 @@ def _handle_command(models: list[ModelSlot], cmd: dict) -> dict:
     return {"cmd": action or "unknown", "status": "error", "error": f"Unknown command: {action}"}
 
 
+# ---------------------------------------------------------------------------
+# Model loads run OFF the event loop, one at a time, under a hard bound (D653)
+# ---------------------------------------------------------------------------
+#
+# 2026-09-26, hub `openvino:gpu`: one `core.compile_model()` of
+# `camstack-yunet-2023mar.xml` for the iGPU never returned. It ran INLINE on this
+# loop, so from 18:33:48 until a container restart at 22:41 the worker read no
+# request and wrote no reply: 786 client-side timeouts on six cameras, 123 of
+# 123 `mem_stats` probes lost, and a process that looked perfectly alive.
+#
+# So a load now runs on its own DAEMON thread and the loop only awaits it:
+# inference and `mem_stats` keep being served while a compile runs. And it is
+# BOUNDED: past `MODEL_LOAD_TIMEOUT_SEC` the load answers `compile-timeout`,
+# naming the model, instead of never answering.
+#
+# A native thread stuck in a GPU compiler cannot be cancelled or killed from
+# Python. So a timeout POISONS the worker FOR NEW LOADS: every later load is
+# refused at once (`worker-poisoned`) rather than queued behind a thread that
+# may never free the compiler, and the models already loaded keep serving.
+#
+# The abandoned compile is NOT abandoned work (fix round 1): it keeps running,
+# so a slow but finite compile still writes its cache. When it comes back the
+# worker says so on the event channel (`compile-finished-late`) and is usable
+# for loads again — the next load of that model is a warm cache hit. If it
+# never comes back, the host kills the process at its HARD bound
+# (shared-inference-pool.ts `POOL_MODEL_LOAD_BOUNDS`). Daemon, not an executor
+# thread: `concurrent.futures` joins its threads at interpreter exit, so a stuck
+# compile would also have made the worker impossible to shut down.
+
+# SOFT bound on ONE model load — compile included — in seconds. A cold GPU
+# compile of the whole model set on the N100 iGPU measured ~25 s; one model is
+# normally 1-3 s. The host declares its own value in the startup config
+# (`modelLoadTimeoutMs`) so its reply deadline and this bound cannot drift;
+# this is what an old host that declares nothing gets.
+MODEL_LOAD_TIMEOUT_SEC = 120.0
+
+# Request id reserved for UNSOLICITED worker events (a reply to no request).
+# The host never allocates it (shared-inference-pool.ts `WORKER_EVENT_REQ_ID`).
+WORKER_EVENT_REQ_ID = 0xFFFFFFFF
+
+# Commands answered by the loop immediately, never queued behind a load: the
+# probe the host uses to tell "slow" from "wedged" must never wait for the very
+# compile it is asking about.
+INLINE_COMMANDS = frozenset({"mem_stats", "uncache_frame"})
+
+# Commands that compile a model — the only ones that leave the loop.
+LOAD_COMMANDS = frozenset({"load", "replace"})
+
+
+def _model_load_timeout_sec(config: dict) -> float:
+    """The load bound for this connection: the host's, else the default."""
+    raw = config.get("modelLoadTimeoutMs")
+    if isinstance(raw, (int, float)) and not isinstance(raw, bool) and raw > 0:
+        return float(raw) / 1000.0
+    return MODEL_LOAD_TIMEOUT_SEC
+
+
+def _model_id_of(config: dict) -> str:
+    """The model's file stem — what a log line and the host's reply carry."""
+    path = str(config.get("path") or "")
+    stem = os.path.splitext(os.path.basename(path))[0]
+    return stem or "unknown"
+
+
+class ModelLoadTimeout(Exception):
+    """A load did not finish inside its bound; new loads are refused until it does."""
+
+
+class WorkerPoisoned(Exception):
+    """A previous load is still running past its bound; no new load starts."""
+
+
+class ModelLoader:
+    """Runs `_load_model` on a daemon thread under a hard bound.
+
+    One load at a time is the CALLER's guarantee (the `CommandRouter` consumes
+    model commands in order); this class only owns the thread, the bound and
+    the poisoning. `load_fn` is injectable for tests; production passes
+    nothing and gets the module's `_load_model`, looked up at call time.
+    `notify` sends an unsolicited event to the host (the `compile-finished-late`
+    line); absent, the event is only written to stderr.
+    """
+
+    def __init__(
+        self,
+        timeout_sec: float = MODEL_LOAD_TIMEOUT_SEC,
+        load_fn: "Optional[Callable[[ModelSlot, dict], None]]" = None,
+        notify: "Optional[Callable[[dict], Awaitable[None]]]" = None,
+    ) -> None:
+        self.timeout_sec = timeout_sec
+        self._load_fn = load_fn
+        self._notify = notify
+        self.finished_late = 0
+        # Test seam: runs on the load thread right after it has claimed the
+        # outcome — lets a test hold the thread exactly in the race window.
+        self._after_claim: Optional[Callable[[], None]] = None
+        # Monotonic start of the compile running past its bound, if any — the
+        # host counts its HARD bound from here, not from a later refusal.
+        self._abandoned_started: Optional[float] = None
+        # `ensure_future` tasks are only weakly referenced by the loop; keep
+        # each notification alive until it is sent.
+        self._pending_notifications: "set[asyncio.Future[None]]" = set()
+        self.in_flight: Optional[dict] = None
+        self.poisoned_by: Optional[dict] = None
+        self.completed = 0
+        self.failed = 0
+        self.timed_out = 0
+
+    def snapshot(self) -> dict:
+        """What the `mem_stats` reply carries about loads."""
+        in_flight = None
+        if self.in_flight is not None:
+            in_flight = {
+                "index": self.in_flight["index"],
+                "modelId": self.in_flight["modelId"],
+                "elapsedMs": round((time.monotonic() - self.in_flight["startedAt"]) * 1000),
+            }
+        return {
+            "inFlight": in_flight,
+            "poisonedBy": dict(self.poisoned_by) if self.poisoned_by is not None else None,
+            "boundMs": round(self.timeout_sec * 1000),
+            "completed": self.completed,
+            "failed": self.failed,
+            "timedOut": self.timed_out,
+            "finishedLate": self.finished_late,
+        }
+
+    def _finish_abandoned(
+        self, index: int, model_id: str, outcome: Optional[BaseException], elapsed_ms: int,
+    ) -> None:
+        """On the loop: an abandoned compile came back. The compiler is free
+        again, so the worker loads again; a successful one has written its
+        cache, so the host's next load of the model is warm. Its RESULT is
+        still discarded — the host was told it failed and has moved on."""
+        self.finished_late += 1
+        self.poisoned_by = None
+        self._abandoned_started = None
+        payload = {
+            "event": "compile-finished-late",
+            "index": index,
+            "modelId": model_id,
+            "ok": outcome is None,
+            "elapsedMs": elapsed_ms,
+        }
+        if outcome is not None:
+            payload["error"] = str(outcome)
+        if self._notify is not None:
+            task = asyncio.ensure_future(self._notify(payload))
+            self._pending_notifications.add(task)
+            task.add_done_callback(self._pending_notifications.discard)
+
+    def abandoned_elapsed_ms(self) -> Optional[int]:
+        """How long the compile running past its bound has been running."""
+        if self._abandoned_started is None:
+            return None
+        return round((time.monotonic() - self._abandoned_started) * 1000)
+
+    async def load(self, slot: ModelSlot, config: dict, index: int) -> None:
+        """Load `config` into the DETACHED `slot`, or raise.
+
+        Raises `WorkerPoisoned` without starting anything once a previous load
+        timed out, `ModelLoadTimeout` when this one exceeds the bound, and the
+        load's own exception when it fails in time.
+        """
+        model_id = _model_id_of(config)
+        if self.poisoned_by is not None:
+            raise WorkerPoisoned(
+                f"worker poisoned for loads: the load of {self.poisoned_by['modelId']} is still "
+                f"running past its bound ({self.poisoned_by['boundMs']}ms) — no new load starts "
+                f"until it returns"
+            )
+        load_fn = self._load_fn if self._load_fn is not None else _load_model
+        loop = asyncio.get_running_loop()
+        done: "asyncio.Future[None]" = loop.create_future()
+        # Exactly ONE of "the load returned in time" and "the bound fired"
+        # wins, decided under this lock (release review). Without it the
+        # thread could see "not abandoned", the bound fire before its settle
+        # ran, and the worker stay poisoned forever for a load that finished.
+        decision = threading.Lock()
+        claimed: dict = {"by": None}  # "thread" | "bound"
+
+        def settle(exc: Optional[BaseException]) -> None:
+            if done.done():
+                return
+            if exc is None:
+                done.set_result(None)
+            else:
+                done.set_exception(exc)
+
+        def target() -> None:
+            outcome: Optional[BaseException] = None
+            try:
+                load_fn(slot, config)
+            except BaseException as exc:  # noqa: BLE001 — handed to the awaiting loop
+                outcome = exc
+            with decision:
+                late = claimed["by"] == "bound"
+                if not late:
+                    claimed["by"] = "thread"
+            if self._after_claim is not None:
+                self._after_claim()
+            if late:
+                # The bound already fired. Say that the native call DID come
+                # back, and when — the only evidence of how long a slow
+                # compile really takes — and hand the worker back its loads.
+                elapsed_ms = round((time.monotonic() - started) * 1000)
+                sys.stderr.write(
+                    f"Model {index} load of {model_id} returned AFTER its bound "
+                    f"in {elapsed_ms}ms ({'error: ' + str(outcome) if outcome else 'ok'}) "
+                    f"— result discarded, loads re-enabled\n"
+                )
+                sys.stderr.flush()
+                try:
+                    loop.call_soon_threadsafe(
+                        self._finish_abandoned, index, model_id, outcome, elapsed_ms,
+                    )
+                except RuntimeError:
+                    pass  # loop closed: the process is shutting down
+                return
+            try:
+                loop.call_soon_threadsafe(settle, outcome)
+            except RuntimeError:
+                pass  # loop closed: the process is shutting down
+
+        started = time.monotonic()
+        self.in_flight = {"index": index, "modelId": model_id, "startedAt": started}
+        thread = threading.Thread(target=target, name=f"model-load-{index}", daemon=True)
+        thread.start()
+        try:
+            try:
+                await asyncio.wait_for(asyncio.shield(done), timeout=self.timeout_sec)
+            except asyncio.TimeoutError:
+                with decision:
+                    bound_won = claimed["by"] is None
+                    if bound_won:
+                        claimed["by"] = "bound"
+                if bound_won:
+                    raise
+                # The load returned AT the bound: its settle is already on
+                # its way to this loop. It counts as in time.
+                await done
+        except asyncio.TimeoutError:
+            self.timed_out += 1
+            self._abandoned_started = started
+            self.poisoned_by = {
+                "index": index,
+                "modelId": model_id,
+                "boundMs": round(self.timeout_sec * 1000),
+                "at": round(time.time() * 1000),
+            }
+            raise ModelLoadTimeout(
+                f"load of {model_id} (index {index}) did not return within "
+                f"{round(self.timeout_sec * 1000)}ms — it keeps running so its cache "
+                f"can be written; no new load starts until it returns"
+            ) from None
+        except Exception:
+            self.failed += 1
+            raise
+        finally:
+            self.in_flight = None
+        self.completed += 1
+
+
+async def _handle_model_command(
+    models: "list[ModelSlot]", cmd: dict, loader: ModelLoader,
+) -> dict:
+    """Every model-management command, in order; `load`/`replace` off the loop.
+
+    The new model is loaded into a DETACHED slot and installed only once its
+    load completed inside the bound, so a compile that returns late — after the
+    host was told `compile-timeout` — can never surface as a model the host
+    believes failed. Whatever the index held before keeps serving until that
+    swap: a `replace` that fails or times out leaves the old model in place,
+    and the old slot object is only dropped, never mutated, so a predict
+    already holding it finishes normally.
+    """
+    action = cmd.get("cmd")
+    if action not in LOAD_COMMANDS:
+        return _handle_command(models, cmd)
+    index = cmd["index"]
+    config = cmd["config"]
+    model_id = _model_id_of(config)
+    while len(models) <= index:
+        models.append(ModelSlot())
+    slot = ModelSlot()
+    sys.stderr.write(
+        f"Model {index} load started: {config['path']} "
+        f"(bound {round(loader.timeout_sec * 1000)}ms)\n"
+    )
+    sys.stderr.flush()
+    t0 = time.perf_counter()
+    try:
+        await loader.load(slot, config, index)
+    except WorkerPoisoned as exc:
+        return {
+            "cmd": action, "index": index, "status": "error",
+            "reason": "worker-poisoned", "modelId": model_id, "poisoned": True,
+            # How long the EARLIER compile has run: the host's hard bound
+            # counts from its start, not from this refusal.
+            "compileElapsedMs": loader.abandoned_elapsed_ms(),
+            "error": str(exc),
+        }
+    except ModelLoadTimeout as exc:
+        sys.stderr.write(f"Model {index} load TIMED OUT: {exc}\n")
+        sys.stderr.flush()
+        return {
+            "cmd": action, "index": index, "status": "error",
+            "reason": "compile-timeout", "modelId": model_id, "poisoned": True,
+            "timeoutMs": round(loader.timeout_sec * 1000),
+            "compileElapsedMs": loader.abandoned_elapsed_ms(),
+            "error": str(exc),
+        }
+    except Exception as exc:
+        return {"cmd": action, "index": index, "status": "error", "error": str(exc)}
+    models[index] = slot
+    load_ms = round((time.perf_counter() - t0) * 1000)
+    verb = "loaded" if action == "load" else "replaced"
+    sys.stderr.write(f"Model {index} {verb}: {config['path']} ({load_ms}ms)\n")
+    sys.stderr.flush()
+    return {"cmd": action, "index": index, "status": "ok", "loadMs": load_ms}
+
+
+class CommandRouter:
+    """Routes MSG_COMMAND frames without ever blocking the reader.
+
+    `INLINE_COMMANDS` are answered at once by `inline` (it needs the loop's own
+    state: caches, backpressure, counters). Everything else is queued and
+    consumed IN ORDER by one task — model slots are indices shared with the
+    host, so a `status` or `unload` must never overtake the `load` sent before
+    it.
+    """
+
+    def __init__(
+        self,
+        models: "list[ModelSlot]",
+        loader: ModelLoader,
+        send: "Callable[[int, dict], Awaitable[None]]",
+        inline: "Callable[[dict], dict]",
+    ) -> None:
+        self._models = models
+        self._loader = loader
+        self._send = send
+        self._inline = inline
+        self._queue: "asyncio.Queue[tuple[int, dict]]" = asyncio.Queue()
+        self._consumer: "Optional[asyncio.Task[None]]" = None
+
+    async def submit(self, req_id: int, cmd: dict) -> None:
+        if cmd.get("cmd") in INLINE_COMMANDS:
+            try:
+                response = self._inline(cmd)
+            except Exception as exc:
+                response = {"cmd": cmd.get("cmd") or "unknown", "status": "error", "error": str(exc)}
+            await self._send(req_id, response)
+            return
+        self._queue.put_nowait((req_id, cmd))
+        if self._consumer is None or self._consumer.done():
+            self._consumer = asyncio.create_task(self._consume())
+
+    async def _consume(self) -> None:
+        while not self._queue.empty():
+            req_id, cmd = self._queue.get_nowait()
+            try:
+                response = await _handle_model_command(self._models, cmd, self._loader)
+            except Exception as exc:
+                response = {"cmd": cmd.get("cmd") or "unknown", "status": "error", "error": str(exc)}
+            await self._send(req_id, response)
+
+
 # ---------------------------------------------------------------------------
 # Main async loop
 # ---------------------------------------------------------------------------
@@ -2631,6 +2977,13 @@ async def _run() -> None:
     )
     sys.stderr.flush()
     _init_runtime(runtime, pool_device)
+    # Off-loop, bounded model loads (D653). Startup-config models below still
+    # load inline: nothing is served before `ready`, and the host bounds that
+    # handshake itself.
+    model_loader = ModelLoader(
+        timeout_sec=_model_load_timeout_sec(config),
+        notify=lambda event: writer.send(WORKER_EVENT_REQ_ID, event),
+    )
 
     models: list[ModelSlot] = []
     for i, mc in enumerate(config.get("models", [])):
@@ -2782,6 +3135,33 @@ async def _run() -> None:
     # Saves ~10ms/call (the PIL resize + numpy + tensor pack cost).
     _preprocess_cache: dict[tuple[int, int], tuple[dict, float, tuple[int, int], int, int]] = {}
 
+    def inline_command(cmd: dict) -> dict:
+        # Needs the loop's own state (caches, backpressure, counters), which
+        # the model-command path never sees. Answered at once — this is the
+        # host's liveness probe, and it must not wait for a compile.
+        if cmd.get("cmd") == "mem_stats":
+            return _mem_stats_payload(
+                models,
+                counters,
+                backpressure,
+                len(frame_cache),
+                len(_preprocess_cache),
+                len(_bench_preprocess_cache),
+                round(time.monotonic() - pool_started_monotonic),
+                model_load=model_loader.snapshot(),
+            )
+        fid = cmd.get("frameId", -1)
+        frame_cache.pop(fid, None)
+        # Purge preprocessed tensor caches for this frame
+        for k in [k for k in _preprocess_cache if k[0] == fid]:
+            del _preprocess_cache[k]
+        # Also purge the bench preprocess cache
+        for k in [k for k in _bench_preprocess_cache if k[0] == fid]:
+            del _bench_preprocess_cache[k]
+        return {"cmd": "uncache_frame", "status": "ok", "frameId": fid}
+
+    command_router = CommandRouter(models, model_loader, writer.send, inline_command)
+
     def get_accumulator(model_idx: int) -> WindowAccumulator:
         acc = accumulators.get(model_idx)
         if acc is None:
@@ -2924,35 +3304,13 @@ async def _run() -> None:
             counters.commands += 1
             try:
                 cmd = json.loads(payload)
-                # Handled inline (needs the loop's own state: caches,
-                # backpressure, counters) — _handle_command only sees models.
-                if cmd.get("cmd") == "mem_stats":
-                    response = _mem_stats_payload(
-                        models,
-                        counters,
-                        backpressure,
-                        len(frame_cache),
-                        len(_preprocess_cache),
-                        len(_bench_preprocess_cache),
-                        round(time.monotonic() - pool_started_monotonic),
-                    )
-                elif cmd.get("cmd") == "uncache_frame":
-                    fid = cmd.get("frameId", -1)
-                    removed_img = frame_cache.pop(fid, None)
-                    # Purge preprocessed tensor caches for this frame
-                    to_del = [k for k in _preprocess_cache if k[0] == fid]
-                    for k in to_del:
-                        del _preprocess_cache[k]
-                    # Also purge the bench preprocess cache
-                    to_del2 = [k for k in _bench_preprocess_cache if k[0] == fid]
-                    for k in to_del2:
-                        del _bench_preprocess_cache[k]
-                    response = {"cmd": "uncache_frame", "status": "ok", "frameId": fid}
-                else:
-                    response = _handle_command(models, cmd)
             except Exception as exc:
-                response = {"cmd": "unknown", "status": "error", "error": str(exc)}
-            await writer.send(req_id, response)
+                await writer.send(req_id, {"cmd": "unknown", "status": "error", "error": str(exc)})
+                continue
+            # NEVER awaits a model load: `load`/`replace` are queued and
+            # compiled on their own thread under a bound (D653), so the next
+            # frame is read while a GPU compile runs.
+            await command_router.submit(req_id, cmd)
 
         elif msg_type == MSG_INFER_JPEG:
             counters.infer_jpeg += 1

package/python/postprocessors/__init__.py

@@ -16,6 +16,7 @@ from .yamnet import postprocess_yamnet
 from .ssd import postprocess_ssd
 from .rfdetr import postprocess_rfdetr
 from .yolonas import postprocess_yolonas
+from .yunet import postprocess_yunet
 
 POSTPROCESSORS = {
     "yolo": postprocess_yolo,
@@ -31,4 +32,5 @@ POSTPROCESSORS = {
     "ssd": postprocess_ssd,
     "rfdetr": postprocess_rfdetr,
     "yolonas": postprocess_yolonas,
+    "yunet": postprocess_yunet,
 }