@camstack/addon-pipeline 1.1.65 → 1.1.67

package/python/inference_pool.py

@@ -180,6 +180,27 @@ OV_DEFAULT_CONCURRENCY: int = 4
 # live on the N100 iGPU). Compiling ONE explicit device removes the handover
 # entirely. We resolve an ordered candidate list (NPU > GPU > CPU) and compile the
 # first that succeeds, with CPU as the guaranteed floor.
+#
+# OpenVINO async execution (CAMSTACK_OV_ASYNC, default OFF / "0"). When ON, the
+# OpenVINO runtime drives an ov.AsyncInferQueue sized to
+# OPTIMAL_NUMBER_OF_INFER_REQUESTS instead of the synchronous InferRequest pool +
+# req.infer(). This mirrors how the reference Intel-iGPU async pipeline saturates
+# the GPU: keep N inferences in flight, collect each in the queue's callback.
+# start_async() BLOCKS once all N jobs are busy, so in-flight work is bounded to N
+# with NO extra buffer (respects the "never buffer frames at frame-rate" rule).
+# OFF is byte-for-byte the historical synchronous path — the ship-dark default.
+_OV_ASYNC_ENABLED: bool = os.environ.get("CAMSTACK_OV_ASYNC", "0") not in (
+    "0", "false", "no",
+)
+# OpenVINO PERFORMANCE_HINT (CAMSTACK_OV_HINT, default "LATENCY"). THROUGHPUT
+# raises OPTIMAL_NUMBER_OF_INFER_REQUESTS (a larger async queue → more device
+# saturation) but spins up multiple internal streams that multiply memory and
+# lengthen tail latency — a known N100 OOM-freeze factor (see the ov_config note
+# in _load_model). Kept at LATENCY by default; only flip to THROUGHPUT on a node
+# with headroom. Any unrecognised value falls back to LATENCY.
+_OV_PERF_HINT: str = os.environ.get("CAMSTACK_OV_HINT", "LATENCY").upper()
+if _OV_PERF_HINT not in ("LATENCY", "THROUGHPUT"):
+    _OV_PERF_HINT = "LATENCY"
 
 
 def _resolve_ov_devices(available: list[str], full_names: dict[str, str]) -> list[str]:
@@ -556,12 +577,16 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
             candidates = [ov_device_req]  # operator pin — compiled directly
             sys.stderr.write(f"OpenVINO: operator-pinned device {ov_device_req}\n")
             sys.stderr.flush()
-        # LATENCY (not THROUGHPUT): one deterministic execution stream instead of
-        # N. THROUGHPUT spun up multiple internal streams whose lazy request
-        # creation collided with the AUTO handover AND multiplied memory (a factor
-        # in the N100 OOM-freeze). LATENCY keeps a single stream; camera-level
-        # concurrency comes from the fixed InferRequest pool built below.
-        ov_config = {"PERFORMANCE_HINT": "LATENCY"}
+        # PERFORMANCE_HINT defaults to LATENCY (CAMSTACK_OV_HINT): one
+        # deterministic execution stream instead of N. THROUGHPUT spins up
+        # multiple internal streams whose lazy request creation historically
+        # collided with the AUTO handover AND multiplied memory (a factor in the
+        # N100 OOM-freeze); it also RAISES OPTIMAL_NUMBER_OF_INFER_REQUESTS, which
+        # enlarges the async queue built below at a memory / tail-latency cost.
+        # LATENCY keeps a single stream; camera-level concurrency comes from the
+        # fixed InferRequest pool / async queue built below. Operators can opt into
+        # THROUGHPUT on a node with headroom via CAMSTACK_OV_HINT=THROUGHPUT.
+        ov_config = {"PERFORMANCE_HINT": _OV_PERF_HINT}
         # Persist compiled GPU/NPU kernels so the multi-second JIT compile is paid
         # once (first boot) instead of on every pool spawn. The cold compile storm
         # (~25s for the full model set on an N100 iGPU) is where frames pile up;
@@ -572,6 +597,31 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
             ov_config["CACHE_DIR"] = _cache_dir
         except OSError:
             pass
+        # Fix 2 — fold uint8→float/scale/layout preprocessing INTO the compiled
+        # graph via PrePostProcessor. When it succeeds we compile the PPP model
+        # object (fed uint8 NHWC at inference time); on ANY failure we fall back
+        # to compiling the plain `path` (byte-identical to the historical float
+        # preprocess). Default on for OpenVINO — CAMSTACK_OV_PPP=0 disables.
+        ppp_meta: Optional[dict] = None
+        compile_source: Any = path
+        if _OV_PPP_ENABLED:
+            try:
+                ppp_model, ppp_meta = _build_ov_ppp_model(core, path, config)
+                if ppp_model is not None:
+                    compile_source = ppp_model
+                    sys.stderr.write(
+                        f"OpenVINO: PrePostProcessor folded for {os.path.basename(path)} "
+                        f"(uint8 NHWC in, {ppp_meta}); Python preprocess stays uint8\n"
+                    )
+                    sys.stderr.flush()
+            except Exception as exc:
+                ppp_meta = None
+                compile_source = path
+                sys.stderr.write(
+                    f"OpenVINO: PrePostProcessor setup failed for {os.path.basename(path)} "
+                    f"({exc}); falling back to Python float preprocess\n"
+                )
+                sys.stderr.flush()
         # Compile-time failover over the explicit candidate list (NO AUTO): try
         # each device in order (NPU>GPU>CPU) and stop at the first that compiles.
         # CPU is the guaranteed floor. If every candidate fails, re-raise — the
@@ -581,7 +631,9 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
         last_exc: Optional[BaseException] = None
         for candidate in candidates:
             try:
-                compiled = core.compile_model(path, device_name=candidate, config=ov_config)
+                compiled = core.compile_model(
+                    compile_source, device_name=candidate, config=ov_config,
+                )
                 ov_device = candidate
                 break
             except Exception as exc:
@@ -606,7 +658,18 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
             f"EXECUTION_DEVICES={_exec_devices}\n"
         )
         sys.stderr.flush()
-        output_names = [o.get_any_name() for o in compiled.outputs]
+        # An OpenVINO IR converted from ONNX can carry an UNNAMED output tensor
+        # (e.g. `yolov8n-package`): `get_any_name()` then raises
+        # "Attempt to get a name for a Tensor without names", which previously
+        # failed the whole model load on OpenVINO nodes (the package-detection
+        # step could never run on OpenVINO). Outputs are fetched POSITIONALLY in
+        # `predict` below, so a synthetic key per unnamed port is sufficient.
+        output_names = []
+        for _i, _o in enumerate(compiled.outputs):
+            try:
+                output_names.append(_o.get_any_name())
+            except Exception:
+                output_names.append(f"output{_i}")
 
         # Record the device's optimal infer-request count so the dispatcher
         # can size the predict pool to actually feed the streams.
@@ -618,14 +681,10 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
         except Exception:
             pass
 
-        # Create ALL InferRequests ONCE, now, post-compile. With AUTO gone there
-        # is no CPU->GPU handover, so no request can be born mid-handover and bind
-        # to stale ports — the score_8 race is structurally removed. Requests are
-        # handed out through a thread-safe queue: an InferRequest is never touched
-        # by two threads at once (checkout/return), so sharing a fixed pool is safe
-        # and avoids the old thread-local lazy creation. Size the pool to the
-        # device's optimal count (>= OV_DEFAULT_CONCURRENCY) so it matches the
-        # predict-pool worker count sized in _run().
+        # Size N ONCE from the device's optimal infer-request count (fallback to
+        # OV_DEFAULT_CONCURRENCY when the property is missing/zero) so it matches
+        # the predict-pool worker count sized in _run(). Both the sync request-pool
+        # and the async queue below are sized to this same N.
         import queue as _queue
 
         try:
@@ -633,32 +692,112 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
         except Exception:
             _n_req = 0
         _n_req = max(_n_req, OV_DEFAULT_CONCURRENCY)
-        _req_pool: "_queue.Queue" = _queue.Queue()
-        for _ in range(_n_req):
-            _req_pool.put(compiled.create_infer_request())
-        sys.stderr.write(f"OpenVINO: created {_n_req} InferRequest(s) up-front on {ov_device}\n")
-        sys.stderr.flush()
 
-        def predict(inp_dict: dict, _names=output_names, _pool=_req_pool) -> dict:
-            inp = list(inp_dict.values())[0]
-            req = _pool.get()
+        if _OV_ASYNC_ENABLED:
+            # Async pipelining path (CAMSTACK_OV_ASYNC=1). An ov.AsyncInferQueue of
+            # N jobs keeps up to N inferences in flight on the device at once — the
+            # way the Intel-iGPU reference path saturates the GPU. Each completion
+            # fires the queue callback; start_async() BLOCKS when all N jobs are
+            # busy, so in-flight submissions are bounded to N with NO extra buffer
+            # (repo rule: never buffer frames at frame-rate). The RuntimeDispatcher
+            # still calls a BLOCKING predict_fn(input)->dict, so we bridge async->
+            # sync per call with a threading.Event + a 1-slot result holder passed
+            # as the submission's userdata; the callback fills it and wakes us.
+            import threading as _threading
             try:
-                result = req.infer(inp)
-            finally:
-                _pool.put(req)
-            # Output fetch by positional index — stable and harmless now that the
-            # ports no longer diverge (single explicit device, no handover).
-            return {name: result[i] for i, name in enumerate(_names)}
-
-        slot.model = compiled
-        slot.predict_fn = predict
-        # Capture the model's declared input layout so `_preprocess` feeds
-        # NHWC vs NCHW correctly (ArcFace/CLIP are NHWC). Best-effort — a
-        # failure leaves `_input_shape` unset → NCHW fallback (unchanged).
-        try:
-            slot.config["_input_shape"] = list(compiled.inputs[0].get_partial_shape())
-        except Exception:
-            pass
+                from openvino import AsyncInferQueue as _AsyncInferQueue
+            except Exception:
+                from openvino.runtime import AsyncInferQueue as _AsyncInferQueue
+
+            _infer_queue = _AsyncInferQueue(compiled, _n_req)
+
+            # Callback runs on an OpenVINO worker thread when a job finishes. Fetch
+            # outputs POSITIONALLY (get_output_tensor(i)) so an UNNAMED IR output
+            # tensor is tolerated (same reason the sync path indexes by position —
+            # see the output_names note above), and .copy() detaches the result
+            # from the request's reusable output buffer before the job is freed
+            # back into the queue and possibly overwritten by the next inference.
+            def _on_done(_request, _userdata, _names=output_names) -> None:
+                _event, _holder = _userdata
+                try:
+                    _holder[0] = {
+                        name: _request.get_output_tensor(i).data.copy()
+                        for i, name in enumerate(_names)
+                    }
+                except BaseException as _exc:  # surface to the waiting caller
+                    _holder[1] = _exc
+                finally:
+                    _event.set()
+
+            _infer_queue.set_callback(_on_done)
+            # Serialise only the (cheap) hand-off. start_async internally waits for
+            # an idle request; guarding it means two dispatcher threads can never
+            # claim the same idle slot. The inference itself stays fully async —
+            # N requests keep running on the device concurrently — so this lock
+            # does not reduce pipelining, it only makes concurrent submit safe.
+            _submit_lock = _threading.Lock()
+            sys.stderr.write(
+                f"OpenVINO: AsyncInferQueue with {_n_req} job(s) on {ov_device} "
+                f"(CAMSTACK_OV_ASYNC=1, hint={_OV_PERF_HINT})\n"
+            )
+            sys.stderr.flush()
+
+            def predict(inp_dict: dict, _q=_infer_queue, _lock=_submit_lock) -> dict:
+                inp = list(inp_dict.values())[0]
+                done = _threading.Event()
+                holder: list = [None, None]  # [result_dict, exception]
+                with _lock:
+                    # Blocks here when all N jobs are busy -> bounded in-flight.
+                    _q.start_async(inp, (done, holder))
+                done.wait()
+                if holder[1] is not None:
+                    raise holder[1]
+                return holder[0]
+
+            slot.model = compiled
+            slot.predict_fn = predict
+        else:
+            # Synchronous path (default, CAMSTACK_OV_ASYNC=0) — byte-for-byte the
+            # historical behaviour. Create ALL InferRequests ONCE, now, post-
+            # compile. With AUTO gone there is no CPU->GPU handover, so no request
+            # can be born mid-handover and bind to stale ports — the score_8 race
+            # is structurally removed. Requests are handed out through a thread-
+            # safe queue: an InferRequest is never touched by two threads at once
+            # (checkout/return), so sharing a fixed pool is safe and avoids the old
+            # thread-local lazy creation.
+            _req_pool: "_queue.Queue" = _queue.Queue()
+            for _ in range(_n_req):
+                _req_pool.put(compiled.create_infer_request())
+            sys.stderr.write(f"OpenVINO: created {_n_req} InferRequest(s) up-front on {ov_device}\n")
+            sys.stderr.flush()
+
+            def predict(inp_dict: dict, _names=output_names, _pool=_req_pool) -> dict:
+                inp = list(inp_dict.values())[0]
+                req = _pool.get()
+                try:
+                    result = req.infer(inp)
+                finally:
+                    _pool.put(req)
+                # Output fetch by positional index — stable and harmless now that
+                # the ports no longer diverge (single explicit device, no handover).
+                return {name: result[i] for i, name in enumerate(_names)}
+
+            slot.model = compiled
+            slot.predict_fn = predict
+        # When the PPP preprocessor is folded in, `_preprocess` produces uint8
+        # NHWC and short-circuits BEFORE the NHWC/NCHW float logic — so the
+        # `_input_shape` heuristic below is irrelevant for PPP models. Record
+        # the plan so `_preprocess` takes the uint8 fast path.
+        if ppp_meta is not None:
+            slot.config["_ov_ppp"] = ppp_meta
+        else:
+            # Capture the model's declared input layout so `_preprocess` feeds
+            # NHWC vs NCHW correctly (ArcFace/CLIP are NHWC). Best-effort — a
+            # failure leaves `_input_shape` unset → NCHW fallback (unchanged).
+            try:
+                slot.config["_input_shape"] = list(compiled.inputs[0].get_partial_shape())
+            except Exception:
+                pass
 
     elif _runtime == "onnxruntime":
         ort = _runtime_lib
@@ -759,6 +898,22 @@ def _unload_model(slot: ModelSlot) -> None:
 # Avoids the CPython id() reuse problem that caused stale detections.
 _bench_preprocess_cache: dict[tuple[int, int], tuple[dict, float, tuple[int, int]]] = {}
 
+# Verifiable proof that the bench preprocess cache is actually HIT on the
+# sustained-throughput BATCH path. A tagged frame that misses runs the FULL
+# decode-independent preprocess (letterbox + float/255 + transpose); a hit
+# returns the pre-built input tensor untouched — pure inference, no CPU
+# preprocess. The historical bug was that MSG_INFER_BATCH wrapped each item
+# fresh WITHOUT the tag, so the cache was NEVER hit on the batch path and every
+# inference re-ran the full preprocess. The sustained benchmark now asserts
+# hits ≫ misses (surfaced in the `status` command) so that regression is caught.
+_bench_cache_hits: int = 0
+_bench_cache_misses: int = 0
+
+
+def _bench_cache_stats() -> dict:
+    """(hits, misses) for the bench preprocess cache — surfaced in `status`."""
+    return {"hits": _bench_cache_hits, "misses": _bench_cache_misses}
+
 def _is_channels_last(input_shape) -> bool:
     """True when a 4-D model input is NHWC ([N,H,W,C]) rather than NCHW
     ([N,C,H,W]). Heuristic: the LAST dim is a channel count (1 or 3) and the
@@ -819,6 +974,30 @@ def _to_model_tensor(arr: "np.ndarray", channels_last: bool) -> "np.ndarray":
     return arr.transpose(2, 0, 1)[np.newaxis].astype(np.float32)
 
 
+_IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
+_IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)
+
+
+def _apply_normalization(arr: "np.ndarray", config: dict) -> "np.ndarray":
+    """Apply per-model input normalization to an HWC RGB float array in [0,1].
+
+    Only ``inputNormalization == 'imagenet'`` changes anything: it subtracts the
+    ImageNet per-channel mean and divides by the std (RGB), matching the
+    EfficientNet-Lite0 / MobileNetV3 animal + vehicle classifiers whose
+    ``labels.json`` declares ``normalize: imagenet``. Absent / ``'none'`` /
+    ``'zero-one'`` keep the plain ``/255`` array byte-identical to the historical
+    unconditional path (detectors, CLIP/ArcFace, and the AIY bird classifier
+    which bakes its own scale). Runs on the HWC array BEFORE the optional
+    NHWC/NCHW transpose in :func:`_to_model_tensor`, so it is layout-agnostic
+    (channel axis is always last here). Pure numpy.
+    """
+    if config.get("inputNormalization") != "imagenet":
+        return arr
+    if arr.ndim != 3 or arr.shape[-1] != 3:
+        return arr
+    return (arr - _IMAGENET_MEAN) / _IMAGENET_STD
+
+
 def _preprocess_ctc_gray(img: "Image.Image", input_w: int, input_h: int) -> "np.ndarray":
     """EasyOCR-accurate preprocess for a fixed-width grayscale CTC recognizer.
 
@@ -849,16 +1028,152 @@ def _preprocess_ctc_gray(img: "Image.Image", input_w: int, input_h: int) -> "np.
     return arr[np.newaxis, np.newaxis].astype(np.float32)  # (1, 1, H, W)
 
 
+# ---------------------------------------------------------------------------
+# Fix 2 — OpenVINO PrePostProcessor (preprocess folded into the compiled graph)
+# ---------------------------------------------------------------------------
+
+# Default ON for OpenVINO; set CAMSTACK_OV_PPP=0 to fall back to the pure-Python
+# float preprocess (the historical path) if a model/postprocessor misbehaves.
+_OV_PPP_ENABLED: bool = os.environ.get("CAMSTACK_OV_PPP", "1") not in ("0", "false", "no")
+
+
+def _ov_dim_to_int(dim: Any) -> "int | None":
+    """Best-effort static-length extraction from an OpenVINO Dimension (or a
+    plain int). Dynamic / unknown dims return None. Pure — no numpy."""
+    try:
+        return int(dim)
+    except (TypeError, ValueError):
+        pass
+    try:
+        if getattr(dim, "is_static", False):
+            return int(dim.get_length())
+    except (TypeError, ValueError, AttributeError):
+        pass
+    return None
+
+
+def _ov_ppp_plan(
+    dims: "list[int | None] | None",
+    input_channels: int,
+    preprocess_mode: str,
+    normalization: "str | None",
+    enabled: bool,
+) -> "dict | None":
+    """Pure decision: should preprocessing be folded into the OpenVINO graph?
+
+    `dims` is the model's declared input partial-shape as a list of int|None in
+    NCHW order. We only fold for standard 3-channel NCHW detectors/classifiers
+    with a STATIC spatial size — grayscale CTC recognizers (channels==1) and
+    NHWC-native embedders (ArcFace/CLIP, channel dim last) keep the pure-Python
+    path. Returns a `ppp_meta` dict describing the Python-side uint8 producer,
+    or None when PPP is not applicable. No OpenVINO / numpy imports —
+    unit-testable in isolation."""
+    if not enabled:
+        return None
+    if input_channels == 1:
+        return None
+    if dims is None or len(dims) != 4:
+        return None
+    _n, c, h, w = dims
+    # NCHW with a concrete 3-channel dim. A last-dim of 3 (NHWC-native) leaves
+    # c as a spatial value → rejected here, so embedders stay on the float path.
+    if c != 3:
+        return None
+    if h is None or w is None or h <= 0 or w <= 0:
+        return None
+    return {
+        "inputW": int(w),
+        "inputH": int(h),
+        "size": int(max(w, h)),
+        "letterbox": preprocess_mode == "letterbox",
+        "normalization": normalization or "none",
+    }
+
+
+def _preprocess_ov_uint8(
+    img: "Image.Image", plan: dict,
+) -> tuple["np.ndarray", float, tuple[int, int]]:
+    """Produce the uint8 NHWC ``[1,H,W,3]`` input for a PPP-folded OV model.
+
+    Letterbox (aspect-preserving pad) or plain resize stays in cheap uint8 PIL
+    space; the compiled OV graph does the float convert + /255 + optional
+    mean/std + NHWC→NCHW transpose on the inference device. Returns
+    ``(tensor, scale, pad)`` — the scale/pad still drive the postprocessor's box
+    mapping exactly like the letterbox float path. Pure numpy/PIL."""
+    if plan["letterbox"]:
+        canvas, scale_val, pad = letterbox_image(img, plan["size"])
+    else:
+        w, h = plan["inputW"], plan["inputH"]
+        if img.size == (w, h):
+            canvas, scale_val, pad = img, 1.0, (0, 0)
+        else:
+            canvas, scale_val, pad = img.resize((w, h), Image.BILINEAR), 1.0, (0, 0)
+    pix = np.asarray(canvas, dtype=np.uint8)
+    if pix.ndim == 2:
+        pix = np.stack([pix] * 3, axis=-1)
+    return pix[np.newaxis], scale_val, pad
+
+
+def _build_ov_ppp_model(core: Any, path: str, config: dict) -> "tuple[Any, dict] | tuple[None, None]":
+    """Read the OpenVINO IR and, when eligible, attach a PrePostProcessor that
+    accepts a uint8 NHWC tensor and folds convert→scale(→mean/std)→layout into
+    the compiled graph. Returns ``(model, ppp_meta)`` to compile, or
+    ``(None, None)`` when PPP is not applied (caller compiles the path directly,
+    byte-identical to the historical float path). Raises on hard OV errors so
+    the caller's try/except falls back safely."""
+    from openvino import Layout, Type
+    from openvino.preprocess import PrePostProcessor
+
+    model = core.read_model(path)
+    inp = model.input()
+    try:
+        pshape = inp.get_partial_shape()
+        dims = [_ov_dim_to_int(pshape[i]) for i in range(len(pshape))]
+    except Exception:
+        dims = None
+
+    _iw, _ih, input_channels = _resolve_input_dims(config)
+    plan = _ov_ppp_plan(
+        dims,
+        input_channels,
+        config.get("preprocessMode", "letterbox"),
+        config.get("inputNormalization"),
+        _OV_PPP_ENABLED,
+    )
+    if plan is None:
+        return None, None
+
+    ppp = PrePostProcessor(model)
+    # Tensor as the caller feeds it: uint8, NHWC, already at the model's spatial
+    # size (Python letterboxes/resizes to size in uint8) → OV needs no resize.
+    ppp.input().tensor().set_element_type(Type.u8).set_layout(Layout("NHWC"))
+    ppp.input().model().set_layout(Layout("NCHW"))
+    steps = ppp.input().preprocess().convert_element_type(Type.f32)
+    if plan["normalization"] == "imagenet":
+        # ((x/255) - mean) / std, applied in add-order (scale divides, mean
+        # subtracts). RGB channel order matches the fed PIL tensor.
+        steps.scale([255.0, 255.0, 255.0])
+        steps.mean([float(v) for v in _IMAGENET_MEAN])
+        steps.scale([float(v) for v in _IMAGENET_STD])
+    else:
+        steps.scale([255.0, 255.0, 255.0])
+    built = ppp.build()
+    return built, plan
+
+
 def _preprocess(img: Image.Image, config: dict) -> tuple[dict, float, tuple[int, int]]:
     input_size = config.get("inputSize", 640)
     input_w, input_h, input_channels = _resolve_input_dims(config)
     # Bench frames have _bench_frame_id tag → use preprocess cache
     bench_fid = getattr(img, '_bench_frame_id', None)
     if bench_fid is not None:
+        global _bench_cache_hits, _bench_cache_misses
         cache_key = (bench_fid, input_size)
         cached = _bench_preprocess_cache.get(cache_key)
         if cached is not None:
+            _bench_cache_hits += 1
             return cached
+        _bench_cache_misses += 1
     preprocess_mode = config.get("preprocessMode", "letterbox")
     input_dict: dict = {}
     if _runtime == "coreml":
@@ -894,6 +1209,9 @@ def _preprocess(img: Image.Image, config: dict) -> tuple[dict, float, tuple[int,
                     arr = resize_image(img, input_w, input_h)
                     scale_val = 1.0
                     pad = (0, 0)
+                # Per-model input normalization (imagenet mean/std for the
+                # animal + vehicle classifiers; no-op for everything else).
+                arr = _apply_normalization(arr, config)
                 # NHWC vs NCHW from the model's declared last dim (1/3 ⇒
                 # channels-last), exactly as before for 3-channel models.
                 _, _, _, w_or_c = input_shape
@@ -931,6 +1249,21 @@ def _preprocess(img: Image.Image, config: dict) -> tuple[dict, float, tuple[int,
     else:
         # OpenVINO / ONNX — always need a float tensor.
         input_name = config.get("_input_name", "images")
+        # Fix 2 — OpenVINO PrePostProcessor fast path. When the model was
+        # compiled with a folded uint8→float/scale/layout preprocessor
+        # (`_ov_ppp` plan present), skip ALL numpy float work here: produce a
+        # uint8 NHWC tensor (aspect-preserving letterbox stays in cheap PIL
+        # space) and let the compiled OV graph do convert/scale/transpose on
+        # the inference device. Removes the per-frame /255 + transpose that
+        # dominated the non-inference CPU cost in production.
+        ov_ppp = config.get("_ov_ppp") if _runtime == "openvino" else None
+        if ov_ppp is not None and input_channels != 1:
+            input_arr, scale_val, pad = _preprocess_ov_uint8(img, ov_ppp)
+            input_dict[input_name] = input_arr
+            result = (input_dict, scale_val, pad)
+            if bench_fid is not None:
+                _bench_preprocess_cache[(bench_fid, input_size)] = result
+            return result
         if input_channels == 1:
             # Grayscale CTC recognizer (EasyOCR plate-OCR): aspect-preserving
             # resize + edge-pad + [-1,1] norm → [1,1,H,W]. NOT the squared
@@ -946,6 +1279,9 @@ def _preprocess(img: Image.Image, config: dict) -> tuple[dict, float, tuple[int,
                 arr = resize_image(img, input_w, input_h)
                 scale_val = 1.0
                 pad = (0, 0)
+            # Per-model input normalization (imagenet mean/std for the animal +
+            # vehicle classifiers; no-op for detectors, CLIP, bird, etc.).
+            arr = _apply_normalization(arr, config)
             # Feed the layout the model actually declares. NHWC models (ArcFace
             # face-embedding [N,112,112,3], CLIP) must NOT be NCHW-transposed —
             # a strict backend (NPU) throws on the wrong shape and the embedding
@@ -1394,7 +1730,7 @@ def _handle_command(models: list[ModelSlot], cmd: dict) -> dict:
                 "loaded": slot.loaded,
                 "postprocessor": slot.config.get("postprocessor") if slot.loaded else None,
             })
-        return {"cmd": "status", "models": status}
+        return {"cmd": "status", "models": status, "benchCache": _bench_cache_stats()}
 
     return {"cmd": action or "unknown", "status": "error", "error": f"Unknown command: {action}"}
 
@@ -1742,13 +2078,21 @@ async def _run() -> None:
             _dispatch_inference(req_id, img, model_idx)
 
         elif msg_type == MSG_INFER_BATCH:
-            # Header: [1B model_idx][1B count]
-            if len(payload) < 2:
+            # Header: [1B model_idx][1B count][4B frame_id]
+            # frame_id (0 = untagged, the live/default case). A NONZERO frame_id
+            # tags every wrapped item with `_bench_frame_id` so `_preprocess`
+            # uses the bench cache — the whole batch is the SAME pinned frame in
+            # the sustained-throughput benchmark, so decode+preprocess runs ONCE
+            # (first miss) and every later inference is a pure-inference cache
+            # hit. Historically the batch path shipped no frame_id and re-ran the
+            # full preprocess on every inference (the ~78fps ceiling).
+            if len(payload) < 6:
                 await writer.send(req_id, {"error": "truncated infer_batch header"})
                 continue
             model_idx = payload[0]
             count = payload[1]
-            offset = 2
+            batch_frame_id = struct.unpack("<I", payload[2:6])[0]
+            offset = 6
             items: list[Image.Image] = []
             parse_err: Optional[str] = None
             for _ in range(count):
@@ -1766,7 +2110,10 @@ async def _run() -> None:
                 raw = payload[offset:offset + size]
                 offset += size
                 try:
-                    items.append(wrap_raw(raw, width, height, fmt))
+                    item_img = wrap_raw(raw, width, height, fmt)
+                    if batch_frame_id != 0:
+                        item_img._bench_frame_id = batch_frame_id
+                    items.append(item_img)
                 except Exception as exc:
                     parse_err = f"raw wrap failed: {exc}"
                     break

package/python/postprocessors/softmax.py

@@ -16,16 +16,28 @@ def postprocess_softmax(
     scale: float,
     pad: tuple[int, int],
 ) -> dict:
-    """Softmax + argmax + label lookup."""
+    """Softmax + argmax + label lookup.
+
+    When ``config['outputProbabilities']`` is set the model already applies
+    softmax IN-GRAPH (e.g. the Google AIY Birds classifier) — its raw output is
+    a probability distribution, so we consume it directly. Re-softmaxing an
+    already-normalised probability vector is NOT a no-op: it collapses the
+    distribution toward uniform (the top-1 score craters far below its true
+    value), which silently defeats every downstream confidence gate.
+    """
     labels = config.get("labels", [])
 
     # Get first output tensor
     raw = np.array(list(predictions.values())[0]).flatten().astype(np.float32)
 
-    # Stable softmax
-    shifted = raw - np.max(raw)
-    exps = np.exp(shifted)
-    probs = exps / np.sum(exps)
+    if config.get("outputProbabilities"):
+        # Model output is already a softmax distribution — do NOT re-softmax.
+        probs = raw
+    else:
+        # Stable softmax over raw logits.
+        shifted = raw - np.max(raw)
+        exps = np.exp(shifted)
+        probs = exps / np.sum(exps)
 
     # Return top-K classifications — normalization (top-1 + alternates) happens in TypeScript
     top_k = min(5, len(probs))

package/python/postprocessors/test_softmax.py

@@ -0,0 +1,52 @@
+"""Tests for the softmax classifier postprocessor.
+
+Covers the raw-logits (default) path and the `outputProbabilities` path used by
+the Google AIY bird classifier, whose output is ALREADY a softmax distribution
+and must NOT be re-softmaxed (re-softmaxing flattens the scores toward uniform
+and silently defeats every downstream confidence gate).
+"""
+import math
+
+import numpy as np
+
+from softmax import postprocess_softmax
+
+
+def _run(raw, config):
+    predictions = {"logits": np.array(raw, dtype=np.float32)}
+    return postprocess_softmax(predictions, config, 100, 100, 1.0, (0, 0))
+
+
+def test_raw_logits_are_softmaxed():
+    # Strongly-peaked logits → a near-1.0 top-1 probability after softmax.
+    out = _run([10.0, 0.0, 0.0], {"labels": ["a", "b", "c"]})
+    top = out["classifications"][0]
+    assert top["class"] == "a"
+    assert top["score"] > 0.99
+
+
+def test_output_probabilities_are_consumed_verbatim():
+    # Already a probability distribution — consumed directly, NOT re-softmaxed.
+    probs = [0.9, 0.07, 0.03]
+    out = _run(probs, {"labels": ["a", "b", "c"], "outputProbabilities": True})
+    top = out["classifications"][0]
+    assert top["class"] == "a"
+    # The true 0.9 survives verbatim. Re-softmaxing this vector would collapse
+    # the top-1 to ~0.49 — well below a 0.6 gate — which is the bug we prevent.
+    assert math.isclose(top["score"], 0.9, abs_tol=1e-3)
+
+
+def test_re_softmaxing_probabilities_would_crater_the_score():
+    # Guard the rationale: prove the WRONG path (default softmax on an already-
+    # softmaxed vector) really does destroy the top-1 confidence.
+    probs = [0.9, 0.07, 0.03]
+    wrong = _run(probs, {"labels": ["a", "b", "c"]})  # no outputProbabilities
+    assert wrong["classifications"][0]["score"] < 0.6
+
+
+def test_argmax_is_preserved_either_way():
+    probs = [0.05, 0.8, 0.15]
+    a = _run(probs, {"labels": ["a", "b", "c"], "outputProbabilities": True})
+    b = _run(probs, {"labels": ["a", "b", "c"]})
+    assert a["classifications"][0]["class"] == "b"
+    assert b["classifications"][0]["class"] == "b"