npm - @camstack/addon-pipeline - Versions diffs - 1.1.66 → 1.1.68 - Mend

@camstack/addon-pipeline 1.1.66 → 1.1.68

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (32) hide show

package/embed-dist/index.html CHANGED Viewed

@@ -4,7 +4,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <title>CamStack Embed</title>
-    <script type="module" crossorigin src="./assets/index-BRG_Fqb5.js"></script>
+    <script type="module" crossorigin src="./assets/index-BkoPcGtQ.js"></script>
     <link rel="stylesheet" crossorigin href="./assets/index-D2pF9Z3W.css">
   </head>
   <body>

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@camstack/addon-pipeline",
-  "version": "1.1.66",
+  "version": "1.1.68",
   "description": "CamStack Pipeline bundle — runner, detection, motion, audio + stream broker. Multi-entry npm package shipping pipeline addons under a single bundle.",
   "keywords": [
     "camstack",

package/python/inference_pool.py CHANGED Viewed

@@ -180,6 +180,27 @@ OV_DEFAULT_CONCURRENCY: int = 4
 # live on the N100 iGPU). Compiling ONE explicit device removes the handover
 # entirely. We resolve an ordered candidate list (NPU > GPU > CPU) and compile the
 # first that succeeds, with CPU as the guaranteed floor.
+#
+# OpenVINO async execution (CAMSTACK_OV_ASYNC, default OFF / "0"). When ON, the
+# OpenVINO runtime drives an ov.AsyncInferQueue sized to
+# OPTIMAL_NUMBER_OF_INFER_REQUESTS instead of the synchronous InferRequest pool +
+# req.infer(). This mirrors how the reference Intel-iGPU async pipeline saturates
+# the GPU: keep N inferences in flight, collect each in the queue's callback.
+# start_async() BLOCKS once all N jobs are busy, so in-flight work is bounded to N
+# with NO extra buffer (respects the "never buffer frames at frame-rate" rule).
+# OFF is byte-for-byte the historical synchronous path — the ship-dark default.
+_OV_ASYNC_ENABLED: bool = os.environ.get("CAMSTACK_OV_ASYNC", "0") not in (
+    "0", "false", "no",
+)
+# OpenVINO PERFORMANCE_HINT (CAMSTACK_OV_HINT, default "LATENCY"). THROUGHPUT
+# raises OPTIMAL_NUMBER_OF_INFER_REQUESTS (a larger async queue → more device
+# saturation) but spins up multiple internal streams that multiply memory and
+# lengthen tail latency — a known N100 OOM-freeze factor (see the ov_config note
+# in _load_model). Kept at LATENCY by default; only flip to THROUGHPUT on a node
+# with headroom. Any unrecognised value falls back to LATENCY.
+_OV_PERF_HINT: str = os.environ.get("CAMSTACK_OV_HINT", "LATENCY").upper()
+if _OV_PERF_HINT not in ("LATENCY", "THROUGHPUT"):
+    _OV_PERF_HINT = "LATENCY"
 def _resolve_ov_devices(available: list[str], full_names: dict[str, str]) -> list[str]:
@@ -556,12 +577,16 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
             candidates = [ov_device_req]  # operator pin — compiled directly
             sys.stderr.write(f"OpenVINO: operator-pinned device {ov_device_req}\n")
             sys.stderr.flush()
-        # LATENCY (not THROUGHPUT): one deterministic execution stream instead of
-        # N. THROUGHPUT spun up multiple internal streams whose lazy request
-        # creation collided with the AUTO handover AND multiplied memory (a factor
-        # in the N100 OOM-freeze). LATENCY keeps a single stream; camera-level
-        # concurrency comes from the fixed InferRequest pool built below.
-        ov_config = {"PERFORMANCE_HINT": "LATENCY"}
+        # PERFORMANCE_HINT defaults to LATENCY (CAMSTACK_OV_HINT): one
+        # deterministic execution stream instead of N. THROUGHPUT spins up
+        # multiple internal streams whose lazy request creation historically
+        # collided with the AUTO handover AND multiplied memory (a factor in the
+        # N100 OOM-freeze); it also RAISES OPTIMAL_NUMBER_OF_INFER_REQUESTS, which
+        # enlarges the async queue built below at a memory / tail-latency cost.
+        # LATENCY keeps a single stream; camera-level concurrency comes from the
+        # fixed InferRequest pool / async queue built below. Operators can opt into
+        # THROUGHPUT on a node with headroom via CAMSTACK_OV_HINT=THROUGHPUT.
+        ov_config = {"PERFORMANCE_HINT": _OV_PERF_HINT}
         # Persist compiled GPU/NPU kernels so the multi-second JIT compile is paid
         # once (first boot) instead of on every pool spawn. The cold compile storm
         # (~25s for the full model set on an N100 iGPU) is where frames pile up;
@@ -572,6 +597,31 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
             ov_config["CACHE_DIR"] = _cache_dir
         except OSError:
             pass
+        # Fix 2 — fold uint8→float/scale/layout preprocessing INTO the compiled
+        # graph via PrePostProcessor. When it succeeds we compile the PPP model
+        # object (fed uint8 NHWC at inference time); on ANY failure we fall back
+        # to compiling the plain `path` (byte-identical to the historical float
+        # preprocess). Default on for OpenVINO — CAMSTACK_OV_PPP=0 disables.
+        ppp_meta: Optional[dict] = None
+        compile_source: Any = path
+        if _OV_PPP_ENABLED:
+            try:
+                ppp_model, ppp_meta = _build_ov_ppp_model(core, path, config)
+                if ppp_model is not None:
+                    compile_source = ppp_model
+                    sys.stderr.write(
+                        f"OpenVINO: PrePostProcessor folded for {os.path.basename(path)} "
+                        f"(uint8 NHWC in, {ppp_meta}); Python preprocess stays uint8\n"
+                    )
+                    sys.stderr.flush()
+            except Exception as exc:
+                ppp_meta = None
+                compile_source = path
+                sys.stderr.write(
+                    f"OpenVINO: PrePostProcessor setup failed for {os.path.basename(path)} "
+                    f"({exc}); falling back to Python float preprocess\n"
+                )
+                sys.stderr.flush()
         # Compile-time failover over the explicit candidate list (NO AUTO): try
         # each device in order (NPU>GPU>CPU) and stop at the first that compiles.
         # CPU is the guaranteed floor. If every candidate fails, re-raise — the
@@ -581,7 +631,9 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
         last_exc: Optional[BaseException] = None
         for candidate in candidates:
             try:
-                compiled = core.compile_model(path, device_name=candidate, config=ov_config)
+                compiled = core.compile_model(
+                    compile_source, device_name=candidate, config=ov_config,
+                )
                 ov_device = candidate
                 break
             except Exception as exc:
@@ -606,7 +658,18 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
             f"EXECUTION_DEVICES={_exec_devices}\n"
         )
         sys.stderr.flush()
-        output_names = [o.get_any_name() for o in compiled.outputs]
+        # An OpenVINO IR converted from ONNX can carry an UNNAMED output tensor
+        # (e.g. `yolov8n-package`): `get_any_name()` then raises
+        # "Attempt to get a name for a Tensor without names", which previously
+        # failed the whole model load on OpenVINO nodes (the package-detection
+        # step could never run on OpenVINO). Outputs are fetched POSITIONALLY in
+        # `predict` below, so a synthetic key per unnamed port is sufficient.
+        output_names = []
+        for _i, _o in enumerate(compiled.outputs):
+            try:
+                output_names.append(_o.get_any_name())
+            except Exception:
+                output_names.append(f"output{_i}")
         # Record the device's optimal infer-request count so the dispatcher
         # can size the predict pool to actually feed the streams.
@@ -618,14 +681,10 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
         except Exception:
             pass
-        # Create ALL InferRequests ONCE, now, post-compile. With AUTO gone there
-        # is no CPU->GPU handover, so no request can be born mid-handover and bind
-        # to stale ports — the score_8 race is structurally removed. Requests are
-        # handed out through a thread-safe queue: an InferRequest is never touched
-        # by two threads at once (checkout/return), so sharing a fixed pool is safe
-        # and avoids the old thread-local lazy creation. Size the pool to the
-        # device's optimal count (>= OV_DEFAULT_CONCURRENCY) so it matches the
-        # predict-pool worker count sized in _run().
+        # Size N ONCE from the device's optimal infer-request count (fallback to
+        # OV_DEFAULT_CONCURRENCY when the property is missing/zero) so it matches
+        # the predict-pool worker count sized in _run(). Both the sync request-pool
+        # and the async queue below are sized to this same N.
         import queue as _queue
         try:
@@ -633,32 +692,112 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
         except Exception:
             _n_req = 0
         _n_req = max(_n_req, OV_DEFAULT_CONCURRENCY)
-        _req_pool: "_queue.Queue" = _queue.Queue()
-        for _ in range(_n_req):
-            _req_pool.put(compiled.create_infer_request())
-        sys.stderr.write(f"OpenVINO: created {_n_req} InferRequest(s) up-front on {ov_device}\n")
-        sys.stderr.flush()
-        def predict(inp_dict: dict, _names=output_names, _pool=_req_pool) -> dict:
-            inp = list(inp_dict.values())[0]
-            req = _pool.get()
+        if _OV_ASYNC_ENABLED:
+            # Async pipelining path (CAMSTACK_OV_ASYNC=1). An ov.AsyncInferQueue of
+            # N jobs keeps up to N inferences in flight on the device at once — the
+            # way the Intel-iGPU reference path saturates the GPU. Each completion
+            # fires the queue callback; start_async() BLOCKS when all N jobs are
+            # busy, so in-flight submissions are bounded to N with NO extra buffer
+            # (repo rule: never buffer frames at frame-rate). The RuntimeDispatcher
+            # still calls a BLOCKING predict_fn(input)->dict, so we bridge async->
+            # sync per call with a threading.Event + a 1-slot result holder passed
+            # as the submission's userdata; the callback fills it and wakes us.
+            import threading as _threading
             try:
-                result = req.infer(inp)
-            finally:
-                _pool.put(req)
-            # Output fetch by positional index — stable and harmless now that the
-            # ports no longer diverge (single explicit device, no handover).
-            return {name: result[i] for i, name in enumerate(_names)}
-        slot.model = compiled
-        slot.predict_fn = predict
-        # Capture the model's declared input layout so `_preprocess` feeds
-        # NHWC vs NCHW correctly (ArcFace/CLIP are NHWC). Best-effort — a
-        # failure leaves `_input_shape` unset → NCHW fallback (unchanged).
-        try:
-            slot.config["_input_shape"] = list(compiled.inputs[0].get_partial_shape())
-        except Exception:
-            pass
+                from openvino import AsyncInferQueue as _AsyncInferQueue
+            except Exception:
+                from openvino.runtime import AsyncInferQueue as _AsyncInferQueue
+            _infer_queue = _AsyncInferQueue(compiled, _n_req)
+            # Callback runs on an OpenVINO worker thread when a job finishes. Fetch
+            # outputs POSITIONALLY (get_output_tensor(i)) so an UNNAMED IR output
+            # tensor is tolerated (same reason the sync path indexes by position —
+            # see the output_names note above), and .copy() detaches the result
+            # from the request's reusable output buffer before the job is freed
+            # back into the queue and possibly overwritten by the next inference.
+            def _on_done(_request, _userdata, _names=output_names) -> None:
+                _event, _holder = _userdata
+                try:
+                    _holder[0] = {
+                        name: _request.get_output_tensor(i).data.copy()
+                        for i, name in enumerate(_names)
+                    }
+                except BaseException as _exc:  # surface to the waiting caller
+                    _holder[1] = _exc
+                finally:
+                    _event.set()
+            _infer_queue.set_callback(_on_done)
+            # Serialise only the (cheap) hand-off. start_async internally waits for
+            # an idle request; guarding it means two dispatcher threads can never
+            # claim the same idle slot. The inference itself stays fully async —
+            # N requests keep running on the device concurrently — so this lock
+            # does not reduce pipelining, it only makes concurrent submit safe.
+            _submit_lock = _threading.Lock()
+            sys.stderr.write(
+                f"OpenVINO: AsyncInferQueue with {_n_req} job(s) on {ov_device} "
+                f"(CAMSTACK_OV_ASYNC=1, hint={_OV_PERF_HINT})\n"
+            )
+            sys.stderr.flush()
+            def predict(inp_dict: dict, _q=_infer_queue, _lock=_submit_lock) -> dict:
+                inp = list(inp_dict.values())[0]
+                done = _threading.Event()
+                holder: list = [None, None]  # [result_dict, exception]
+                with _lock:
+                    # Blocks here when all N jobs are busy -> bounded in-flight.
+                    _q.start_async(inp, (done, holder))
+                done.wait()
+                if holder[1] is not None:
+                    raise holder[1]
+                return holder[0]
+            slot.model = compiled
+            slot.predict_fn = predict
+        else:
+            # Synchronous path (default, CAMSTACK_OV_ASYNC=0) — byte-for-byte the
+            # historical behaviour. Create ALL InferRequests ONCE, now, post-
+            # compile. With AUTO gone there is no CPU->GPU handover, so no request
+            # can be born mid-handover and bind to stale ports — the score_8 race
+            # is structurally removed. Requests are handed out through a thread-
+            # safe queue: an InferRequest is never touched by two threads at once
+            # (checkout/return), so sharing a fixed pool is safe and avoids the old
+            # thread-local lazy creation.
+            _req_pool: "_queue.Queue" = _queue.Queue()
+            for _ in range(_n_req):
+                _req_pool.put(compiled.create_infer_request())
+            sys.stderr.write(f"OpenVINO: created {_n_req} InferRequest(s) up-front on {ov_device}\n")
+            sys.stderr.flush()
+            def predict(inp_dict: dict, _names=output_names, _pool=_req_pool) -> dict:
+                inp = list(inp_dict.values())[0]
+                req = _pool.get()
+                try:
+                    result = req.infer(inp)
+                finally:
+                    _pool.put(req)
+                # Output fetch by positional index — stable and harmless now that
+                # the ports no longer diverge (single explicit device, no handover).
+                return {name: result[i] for i, name in enumerate(_names)}
+            slot.model = compiled
+            slot.predict_fn = predict
+        # When the PPP preprocessor is folded in, `_preprocess` produces uint8
+        # NHWC and short-circuits BEFORE the NHWC/NCHW float logic — so the
+        # `_input_shape` heuristic below is irrelevant for PPP models. Record
+        # the plan so `_preprocess` takes the uint8 fast path.
+        if ppp_meta is not None:
+            slot.config["_ov_ppp"] = ppp_meta
+        else:
+            # Capture the model's declared input layout so `_preprocess` feeds
+            # NHWC vs NCHW correctly (ArcFace/CLIP are NHWC). Best-effort — a
+            # failure leaves `_input_shape` unset → NCHW fallback (unchanged).
+            try:
+                slot.config["_input_shape"] = list(compiled.inputs[0].get_partial_shape())
+            except Exception:
+                pass
     elif _runtime == "onnxruntime":
         ort = _runtime_lib
@@ -759,6 +898,22 @@ def _unload_model(slot: ModelSlot) -> None:
 # Avoids the CPython id() reuse problem that caused stale detections.
 _bench_preprocess_cache: dict[tuple[int, int], tuple[dict, float, tuple[int, int]]] = {}
+# Verifiable proof that the bench preprocess cache is actually HIT on the
+# sustained-throughput BATCH path. A tagged frame that misses runs the FULL
+# decode-independent preprocess (letterbox + float/255 + transpose); a hit
+# returns the pre-built input tensor untouched — pure inference, no CPU
+# preprocess. The historical bug was that MSG_INFER_BATCH wrapped each item
+# fresh WITHOUT the tag, so the cache was NEVER hit on the batch path and every
+# inference re-ran the full preprocess. The sustained benchmark now asserts
+# hits ≫ misses (surfaced in the `status` command) so that regression is caught.
+_bench_cache_hits: int = 0
+_bench_cache_misses: int = 0
+def _bench_cache_stats() -> dict:
+    """(hits, misses) for the bench preprocess cache — surfaced in `status`."""
+    return {"hits": _bench_cache_hits, "misses": _bench_cache_misses}
 def _is_channels_last(input_shape) -> bool:
     """True when a 4-D model input is NHWC ([N,H,W,C]) rather than NCHW
     ([N,C,H,W]). Heuristic: the LAST dim is a channel count (1 or 3) and the
@@ -873,16 +1028,152 @@ def _preprocess_ctc_gray(img: "Image.Image", input_w: int, input_h: int) -> "np.
     return arr[np.newaxis, np.newaxis].astype(np.float32)  # (1, 1, H, W)
+# ---------------------------------------------------------------------------
+# Fix 2 — OpenVINO PrePostProcessor (preprocess folded into the compiled graph)
+# ---------------------------------------------------------------------------
+# Default ON for OpenVINO; set CAMSTACK_OV_PPP=0 to fall back to the pure-Python
+# float preprocess (the historical path) if a model/postprocessor misbehaves.
+_OV_PPP_ENABLED: bool = os.environ.get("CAMSTACK_OV_PPP", "1") not in ("0", "false", "no")
+def _ov_dim_to_int(dim: Any) -> "int | None":
+    """Best-effort static-length extraction from an OpenVINO Dimension (or a
+    plain int). Dynamic / unknown dims return None. Pure — no numpy."""
+    try:
+        return int(dim)
+    except (TypeError, ValueError):
+        pass
+    try:
+        if getattr(dim, "is_static", False):
+            return int(dim.get_length())
+    except (TypeError, ValueError, AttributeError):
+        pass
+    return None
+def _ov_ppp_plan(
+    dims: "list[int | None] | None",
+    input_channels: int,
+    preprocess_mode: str,
+    normalization: "str | None",
+    enabled: bool,
+) -> "dict | None":
+    """Pure decision: should preprocessing be folded into the OpenVINO graph?
+    `dims` is the model's declared input partial-shape as a list of int|None in
+    NCHW order. We only fold for standard 3-channel NCHW detectors/classifiers
+    with a STATIC spatial size — grayscale CTC recognizers (channels==1) and
+    NHWC-native embedders (ArcFace/CLIP, channel dim last) keep the pure-Python
+    path. Returns a `ppp_meta` dict describing the Python-side uint8 producer,
+    or None when PPP is not applicable. No OpenVINO / numpy imports —
+    unit-testable in isolation."""
+    if not enabled:
+        return None
+    if input_channels == 1:
+        return None
+    if dims is None or len(dims) != 4:
+        return None
+    _n, c, h, w = dims
+    # NCHW with a concrete 3-channel dim. A last-dim of 3 (NHWC-native) leaves
+    # c as a spatial value → rejected here, so embedders stay on the float path.
+    if c != 3:
+        return None
+    if h is None or w is None or h <= 0 or w <= 0:
+        return None
+    return {
+        "inputW": int(w),
+        "inputH": int(h),
+        "size": int(max(w, h)),
+        "letterbox": preprocess_mode == "letterbox",
+        "normalization": normalization or "none",
+    }
+def _preprocess_ov_uint8(
+    img: "Image.Image", plan: dict,
+) -> tuple["np.ndarray", float, tuple[int, int]]:
+    """Produce the uint8 NHWC ``[1,H,W,3]`` input for a PPP-folded OV model.
+    Letterbox (aspect-preserving pad) or plain resize stays in cheap uint8 PIL
+    space; the compiled OV graph does the float convert + /255 + optional
+    mean/std + NHWC→NCHW transpose on the inference device. Returns
+    ``(tensor, scale, pad)`` — the scale/pad still drive the postprocessor's box
+    mapping exactly like the letterbox float path. Pure numpy/PIL."""
+    if plan["letterbox"]:
+        canvas, scale_val, pad = letterbox_image(img, plan["size"])
+    else:
+        w, h = plan["inputW"], plan["inputH"]
+        if img.size == (w, h):
+            canvas, scale_val, pad = img, 1.0, (0, 0)
+        else:
+            canvas, scale_val, pad = img.resize((w, h), Image.BILINEAR), 1.0, (0, 0)
+    pix = np.asarray(canvas, dtype=np.uint8)
+    if pix.ndim == 2:
+        pix = np.stack([pix] * 3, axis=-1)
+    return pix[np.newaxis], scale_val, pad
+def _build_ov_ppp_model(core: Any, path: str, config: dict) -> "tuple[Any, dict] | tuple[None, None]":
+    """Read the OpenVINO IR and, when eligible, attach a PrePostProcessor that
+    accepts a uint8 NHWC tensor and folds convert→scale(→mean/std)→layout into
+    the compiled graph. Returns ``(model, ppp_meta)`` to compile, or
+    ``(None, None)`` when PPP is not applied (caller compiles the path directly,
+    byte-identical to the historical float path). Raises on hard OV errors so
+    the caller's try/except falls back safely."""
+    from openvino import Layout, Type
+    from openvino.preprocess import PrePostProcessor
+    model = core.read_model(path)
+    inp = model.input()
+    try:
+        pshape = inp.get_partial_shape()
+        dims = [_ov_dim_to_int(pshape[i]) for i in range(len(pshape))]
+    except Exception:
+        dims = None
+    _iw, _ih, input_channels = _resolve_input_dims(config)
+    plan = _ov_ppp_plan(
+        dims,
+        input_channels,
+        config.get("preprocessMode", "letterbox"),
+        config.get("inputNormalization"),
+        _OV_PPP_ENABLED,
+    )
+    if plan is None:
+        return None, None
+    ppp = PrePostProcessor(model)
+    # Tensor as the caller feeds it: uint8, NHWC, already at the model's spatial
+    # size (Python letterboxes/resizes to size in uint8) → OV needs no resize.
+    ppp.input().tensor().set_element_type(Type.u8).set_layout(Layout("NHWC"))
+    ppp.input().model().set_layout(Layout("NCHW"))
+    steps = ppp.input().preprocess().convert_element_type(Type.f32)
+    if plan["normalization"] == "imagenet":
+        # ((x/255) - mean) / std, applied in add-order (scale divides, mean
+        # subtracts). RGB channel order matches the fed PIL tensor.
+        steps.scale([255.0, 255.0, 255.0])
+        steps.mean([float(v) for v in _IMAGENET_MEAN])
+        steps.scale([float(v) for v in _IMAGENET_STD])
+    else:
+        steps.scale([255.0, 255.0, 255.0])
+    built = ppp.build()
+    return built, plan
 def _preprocess(img: Image.Image, config: dict) -> tuple[dict, float, tuple[int, int]]:
     input_size = config.get("inputSize", 640)
     input_w, input_h, input_channels = _resolve_input_dims(config)
     # Bench frames have _bench_frame_id tag → use preprocess cache
     bench_fid = getattr(img, '_bench_frame_id', None)
     if bench_fid is not None:
+        global _bench_cache_hits, _bench_cache_misses
         cache_key = (bench_fid, input_size)
         cached = _bench_preprocess_cache.get(cache_key)
         if cached is not None:
+            _bench_cache_hits += 1
             return cached
+        _bench_cache_misses += 1
     preprocess_mode = config.get("preprocessMode", "letterbox")
     input_dict: dict = {}
     if _runtime == "coreml":
@@ -958,6 +1249,21 @@ def _preprocess(img: Image.Image, config: dict) -> tuple[dict, float, tuple[int,
     else:
         # OpenVINO / ONNX — always need a float tensor.
         input_name = config.get("_input_name", "images")
+        # Fix 2 — OpenVINO PrePostProcessor fast path. When the model was
+        # compiled with a folded uint8→float/scale/layout preprocessor
+        # (`_ov_ppp` plan present), skip ALL numpy float work here: produce a
+        # uint8 NHWC tensor (aspect-preserving letterbox stays in cheap PIL
+        # space) and let the compiled OV graph do convert/scale/transpose on
+        # the inference device. Removes the per-frame /255 + transpose that
+        # dominated the non-inference CPU cost in production.
+        ov_ppp = config.get("_ov_ppp") if _runtime == "openvino" else None
+        if ov_ppp is not None and input_channels != 1:
+            input_arr, scale_val, pad = _preprocess_ov_uint8(img, ov_ppp)
+            input_dict[input_name] = input_arr
+            result = (input_dict, scale_val, pad)
+            if bench_fid is not None:
+                _bench_preprocess_cache[(bench_fid, input_size)] = result
+            return result
         if input_channels == 1:
             # Grayscale CTC recognizer (EasyOCR plate-OCR): aspect-preserving
             # resize + edge-pad + [-1,1] norm → [1,1,H,W]. NOT the squared
@@ -1424,7 +1730,7 @@ def _handle_command(models: list[ModelSlot], cmd: dict) -> dict:
                 "loaded": slot.loaded,
                 "postprocessor": slot.config.get("postprocessor") if slot.loaded else None,
             })
-        return {"cmd": "status", "models": status}
+        return {"cmd": "status", "models": status, "benchCache": _bench_cache_stats()}
     return {"cmd": action or "unknown", "status": "error", "error": f"Unknown command: {action}"}
@@ -1772,13 +2078,21 @@ async def _run() -> None:
             _dispatch_inference(req_id, img, model_idx)
         elif msg_type == MSG_INFER_BATCH:
-            # Header: [1B model_idx][1B count]
-            if len(payload) < 2:
+            # Header: [1B model_idx][1B count][4B frame_id]
+            # frame_id (0 = untagged, the live/default case). A NONZERO frame_id
+            # tags every wrapped item with `_bench_frame_id` so `_preprocess`
+            # uses the bench cache — the whole batch is the SAME pinned frame in
+            # the sustained-throughput benchmark, so decode+preprocess runs ONCE
+            # (first miss) and every later inference is a pure-inference cache
+            # hit. Historically the batch path shipped no frame_id and re-ran the
+            # full preprocess on every inference (the ~78fps ceiling).
+            if len(payload) < 6:
                 await writer.send(req_id, {"error": "truncated infer_batch header"})
                 continue
             model_idx = payload[0]
             count = payload[1]
-            offset = 2
+            batch_frame_id = struct.unpack("<I", payload[2:6])[0]
+            offset = 6
             items: list[Image.Image] = []
             parse_err: Optional[str] = None
             for _ in range(count):
@@ -1796,7 +2110,10 @@ async def _run() -> None:
                 raw = payload[offset:offset + size]
                 offset += size
                 try:
-                    items.append(wrap_raw(raw, width, height, fmt))
+                    item_img = wrap_raw(raw, width, height, fmt)
+                    if batch_frame_id != 0:
+                        item_img._bench_frame_id = batch_frame_id
+                    items.append(item_img)
                 except Exception as exc:
                     parse_err = f"raw wrap failed: {exc}"
                     break

package/python/test_inference_pool_ov_ppp.py ADDED Viewed

@@ -0,0 +1,117 @@
+"""Fix 2 — OpenVINO PrePostProcessor eligibility (`_ov_ppp_plan`) + the uint8
+dimension helper (`_ov_dim_to_int`).
+`_ov_ppp_plan` is the PURE decision (no OpenVINO / numpy) of WHETHER to fold
+preprocessing into the compiled graph: only standard 3-channel NCHW
+detectors/classifiers with a static spatial size qualify; grayscale CTC
+recognizers and NHWC-native embedders (ArcFace/CLIP) keep the pure-Python float
+path. Heavy deps are stubbed so the module imports in a minimal sandbox.
+Run: python3 -m unittest test_inference_pool_ov_ppp -v
+"""
+import sys
+import types
+import unittest
+if "numpy" not in sys.modules:
+    _np = types.ModuleType("numpy")
+    # inference_pool builds `_IMAGENET_MEAN = np.array(...)` at import time; the
+    # PPP plan decision under test needs no real numpy, so a passthrough stub
+    # keeps this pure test runnable in a numpy-less sandbox.
+    _np.array = lambda seq, dtype=None: list(seq)  # noqa: E731
+    _np.float32 = "float32"
+    sys.modules["numpy"] = _np
+for _mod in ("PIL", "PIL.Image"):
+    if _mod not in sys.modules:
+        sys.modules[_mod] = types.ModuleType(_mod)
+if not hasattr(sys.modules["PIL"], "Image"):
+    sys.modules["PIL"].Image = types.ModuleType("PIL.Image")
+if "postprocessors" not in sys.modules:
+    _pp = types.ModuleType("postprocessors")
+    _pp.POSTPROCESSORS = {}
+    sys.modules["postprocessors"] = _pp
+from inference_pool import _ov_dim_to_int, _ov_ppp_plan  # noqa: E402
+class _StaticDim:
+    """Mimics an OpenVINO static Dimension: int() raises, get_length() works."""
+    def __init__(self, length: int) -> None:
+        self._length = length
+        self.is_static = True
+    def __int__(self) -> int:
+        raise TypeError("Dimension is not directly int()-able")
+    def get_length(self) -> int:
+        return self._length
+class _DynamicDim:
+    is_static = False
+    def __int__(self) -> int:
+        raise TypeError("dynamic dimension")
+class OvDimToIntTest(unittest.TestCase):
+    def test_plain_int_passthrough(self) -> None:
+        self.assertEqual(_ov_dim_to_int(640), 640)
+    def test_static_dimension_uses_get_length(self) -> None:
+        self.assertEqual(_ov_dim_to_int(_StaticDim(416)), 416)
+    def test_dynamic_dimension_is_none(self) -> None:
+        self.assertIsNone(_ov_dim_to_int(_DynamicDim()))
+    def test_garbage_is_none(self) -> None:
+        self.assertIsNone(_ov_dim_to_int(object()))
+class OvPppPlanTest(unittest.TestCase):
+    def test_standard_square_detector_folds_with_letterbox(self) -> None:
+        plan = _ov_ppp_plan([1, 3, 640, 640], 3, "letterbox", None, True)
+        self.assertIsNotNone(plan)
+        assert plan is not None
+        self.assertEqual((plan["inputW"], plan["inputH"], plan["size"]), (640, 640, 640))
+        self.assertTrue(plan["letterbox"])
+        self.assertEqual(plan["normalization"], "none")
+    def test_non_letterbox_classifier_plan_is_plain_resize(self) -> None:
+        plan = _ov_ppp_plan([1, 3, 224, 224], 3, "resize", "imagenet", True)
+        assert plan is not None
+        self.assertFalse(plan["letterbox"])
+        self.assertEqual(plan["normalization"], "imagenet")
+    def test_non_square_detector_uses_max_side_for_letterbox(self) -> None:
+        # `_ov_ppp_plan` takes ALREADY-resolved int|None dims (the caller runs
+        # `_ov_dim_to_int` over the OV Dimensions first — see _ov_dim_to_int
+        # tests above). Non-square 384×640 → letterbox square uses the max side.
+        plan = _ov_ppp_plan([1, 3, 384, 640], 3, "letterbox", None, True)
+        assert plan is not None
+        self.assertEqual((plan["inputW"], plan["inputH"]), (640, 384))
+        self.assertEqual(plan["size"], 640)
+    def test_disabled_returns_none(self) -> None:
+        self.assertIsNone(_ov_ppp_plan([1, 3, 640, 640], 3, "letterbox", None, False))
+    def test_grayscale_ctc_is_rejected(self) -> None:
+        # Plate-OCR: channels==1 → keep the pure-Python EasyOCR path.
+        self.assertIsNone(_ov_ppp_plan([1, 1, 64, 384], 1, "resize", None, True))
+    def test_nhwc_native_embedder_is_rejected(self) -> None:
+        # ArcFace/CLIP declare [N,H,W,C] → the NCHW channel slot holds a spatial
+        # value (112), not 3 → rejected, embedder stays on the float NHWC path.
+        self.assertIsNone(_ov_ppp_plan([1, 112, 112, 3], 3, "resize", None, True))
+    def test_dynamic_spatial_is_rejected(self) -> None:
+        self.assertIsNone(_ov_ppp_plan([1, 3, None, None], 3, "letterbox", None, True))
+    def test_wrong_rank_is_rejected(self) -> None:
+        self.assertIsNone(_ov_ppp_plan([3, 640, 640], 3, "letterbox", None, True))
+        self.assertIsNone(_ov_ppp_plan(None, 3, "letterbox", None, True))
+if __name__ == "__main__":
+    unittest.main()