@trillboards/edge-sdk 0.2.89 → 0.2.91

package/deploy/detector/trillboards_detector/session.py

@@ -39,6 +39,14 @@ import numpy as np
 import onnxruntime as ort
 from PIL import Image
 
+from .decode import (
+    ANCHOR_4_PLUS_NC,
+    END2END_N_6,
+    PREPROCESSING_BY_LAYOUT,
+    RFDETR_DETS_LABELS,
+    YOLO_DECODERS,
+    LetterboxGeometry,
+)
 from .detections import Detection, DetectionBatch
 from .errors import ModelContractError
 from .labels import ModelVocabulary
@@ -54,13 +62,37 @@ IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32).reshape(3, 1, 1
 NUM_SELECT = 300
 
 
+#: Ultralytics' letterbox fill. `LetterBox.__init__(..., fill=114)` — a mid-grey
+#: the model saw in training as padding and not as content.
+LETTERBOX_FILL = (114, 114, 114)
+
+
 @dataclass
 class GraphSpec:
-    """What the loaded artifact actually is, read off the graph, never assumed."""
+    """What the loaded artifact actually is, CHECKED against what it declared.
+
+    "Read off the graph, never assumed" was the whole rule while one layout
+    shipped. It cannot stay the whole rule now: `[1, 300, 6]` and `[1, 6, 8400]`
+    are both rank-3 tensors with a 6 in them, so a reader that decides from the
+    shape gets one of them backwards and reads x1,y1,x2,y2,score,cls as four box
+    channels and two classes — with no exception, no NaN and a plausible table.
+    So the LAYOUT is DECLARED by the artifact's variant contract and the graph is
+    checked against it here; everything else (resolution, output names, the class
+    axis) is still read off the graph and still refused on disagreement.
+    """
 
     input_name: str
-    box_output: str
-    logit_output: str
+    #: Exactly the outputs `detect()` asks ORT for, in the order the decode wants
+    #: them. Two names for RF-DETR (`dets`, `labels`), one for either YOLO head.
+    output_names: "tuple[str, ...]"
+    #: The DECLARED layout this graph was checked against, republished here so
+    #: `/health` can say which decode is running. A number nobody can attribute
+    #: to a decode is the `agent_version` failure one runbook over.
+    output_layout: str
+    #: Which preprocessing this layout's artifact was exported with. Reported for
+    #: the same reason: the two are not interchangeable and the difference is
+    #: invisible to every shape check.
+    preprocessing: str
     height: int
     width: int
     num_queries: int
@@ -89,6 +121,79 @@ def preprocess(jpeg: bytes, width: int, height: int) -> tuple[np.ndarray, int, i
     return chw[None, ...], src_w, src_h
 
 
+def preprocess_letterbox(jpeg: bytes, width: int, height: int):
+    """JPEG bytes -> NCHW float32 the Ultralytics way, plus the mapping used.
+
+    A DIFFERENT TRANSFORM FROM `preprocess`, NOT A TIDIER ONE. RF-DETR STRETCHES
+    to a square and normalises by ImageNet statistics; an Ultralytics export is
+    scaled to FIT, padded 114-grey and divided by 255 with no mean or standard
+    deviation at all. Using either one on the other's artifact changes the aspect
+    of every object in the frame or shifts every input channel by ~2 standard
+    deviations, and neither failure raises anything: the tensor is the right
+    shape, the scores are still in (0,1), and the table still looks like
+    detections. Which one ran is on `/health` as `preprocessing` for exactly that
+    reason.
+
+    Transcribed from `ultralytics/data/augment.py::LetterBox.__call__`:
+
+      gain    `r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])`
+      resize  `new_unpad = round(shape[1] * r), round(shape[0] * r)`
+      pad     centred, `fill=114`
+
+    IT MATCHES `bench/run_candidate.py::letterbox` BYTE FOR BYTE, and that is the
+    load-bearing property rather than matching Ultralytics' cv2 exactly. The
+    harness is what SCORES a candidate, so the service has to reproduce the arm
+    the score was measured on — the same argument this module's header makes for
+    transcribing `rfdetr`'s own resize. Measured on 2026-08-25: Pillow's BILINEAR
+    low-passes on downscale and cv2's INTER_LINEAR does not, and on a 1200x900
+    frame that moved one class's confidence from 0.18 to 0.51. So the two are NOT
+    interchangeable, and a future move to cv2 semantics is a re-score of every
+    published card, never a preprocessing tidy-up.
+
+    Returns the geometry rather than recomputing it at decode time: a box is
+    un-mapped with the same gain and padding that put the pixels there.
+    """
+    with Image.open(io.BytesIO(jpeg)) as im:
+        rgb = im.convert("RGB")
+        src_w, src_h = rgb.size
+        gain = min(width / src_w, height / src_h)
+        new_w, new_h = int(round(src_w * gain)), int(round(src_h * gain))
+        resized = rgb.resize((new_w, new_h), resample=Image.BILINEAR)
+        canvas = Image.new("RGB", (width, height), LETTERBOX_FILL)
+        # `(W - new_w) // 2` is Ultralytics' `round(dw - 0.1)` for every input:
+        # dw is either an integer or a half-integer, and both forms floor it.
+        # `ops.scale_boxes` recomputes the second form, which is why decoding
+        # against it agrees to 0.0 px — pinned by `test_decode.py`.
+        pad_x, pad_y = (width - new_w) // 2, (height - new_h) // 2
+        canvas.paste(resized, (pad_x, pad_y))
+        arr = np.asarray(canvas, dtype=np.float32)
+
+    blob = (arr / np.float32(255.0)).transpose(2, 0, 1)[None, ...]
+    return blob, LetterboxGeometry(gain, pad_x, pad_y, src_w, src_h)
+
+
+def preprocess_for_layout(layout: str, jpeg: bytes, width: int, height: int):
+    """Pick the transform this artifact was exported with. NOT a formatting step.
+
+    ITS OWN FUNCTION SO IT CAN BE TESTED WITHOUT A MODEL. Inlined in
+    `OnnxDetector.detect` it was reachable only through a real ORT session, which
+    means only on a machine holding a 128 MiB artifact — and a mutation test
+    proved the gap: swapping the two preprocessors left all 178 unit tests green
+    and was caught only by the bench. A dispatch nothing can test is a dispatch
+    that will one day be wrong in the field with every surface green, which is
+    the failure this whole service is built against.
+
+    Returns `(blob, geometry, frame_w, frame_h)`. `geometry` is None for the
+    stretch path, where there is no padding to undo and boxes come back
+    normalised; the YOLO decodes need it and `postprocess` does not.
+    """
+    if layout == RFDETR_DETS_LABELS:
+        blob, frame_w, frame_h = preprocess(jpeg, width, height)
+        return blob, None, frame_w, frame_h
+    blob, geometry = preprocess_letterbox(jpeg, width, height)
+    return blob, geometry, geometry.frame_w, geometry.frame_h
+
+
 def postprocess(
     boxes_cxcywh: np.ndarray,
     logits: np.ndarray,
@@ -270,25 +375,50 @@ class OnnxDetector:
         min_confidence: float,
         wanted: "set[str] | None" = None,
     ) -> DetectionBatch:
+        """One frame in, this model's detections out, in the frame's own pixels.
+
+        THE ARM IS PICKED ONCE, AT LOAD, FROM THE DECLARATION — never per frame
+        and never from the tensor. `self.spec.output_layout` was checked against
+        the graph by `_read_graph_spec`, so by the time a frame arrives there is
+        no decision left to get wrong.
+        """
         started = time.perf_counter()
-        blob, frame_w, frame_h = preprocess(jpeg, self.spec.width, self.spec.height)
+        layout = self.spec.output_layout
+        blob, geometry, frame_w, frame_h = preprocess_for_layout(
+            layout, jpeg, self.spec.width, self.spec.height
+        )
 
         with self._lock:
             infer_started = time.perf_counter()
-            outputs = self._session.run(
-                [self.spec.box_output, self.spec.logit_output], {self.spec.input_name: blob}
-            )
+            outputs = self._session.run(list(self.spec.output_names), {self.spec.input_name: blob})
             infer_ms = (time.perf_counter() - infer_started) * 1000.0
 
-        batch = postprocess(
-            outputs[0],
-            outputs[1],
-            frame_w,
-            frame_h,
-            min_confidence,
-            wanted,
-            vocabulary=self.vocabulary,
-        )
+        if layout == RFDETR_DETS_LABELS:
+            batch = postprocess(
+                outputs[0],
+                outputs[1],
+                frame_w,
+                frame_h,
+                min_confidence,
+                wanted,
+                vocabulary=self.vocabulary,
+            )
+        else:
+            contract = self.vocabulary.onnx_variant
+            scores, class_idx, x0, y0, x1, y1 = YOLO_DECODERS[layout](
+                outputs[0],
+                geometry,
+                min_confidence,
+                box_scale_to_pixels=contract.box_scale_to_pixels,
+                # The tensor ORT RETURNED, checked against the vocabulary — not
+                # the shape the graph DECLARED, which `_read_graph_spec` already
+                # checked at load. They are the same claim only while no axis is
+                # dynamic, and the artifact library is not ours to promise that
+                # about.
+                expected_classes=self.vocabulary.logit_slots,
+            )
+            batch = _to_wire(scores, class_idx, x0, y0, x1, y1, wanted, self.vocabulary)
+
         batch.inference_ms = infer_ms
         batch.total_ms = (time.perf_counter() - started) * 1000.0
         return batch
@@ -337,6 +467,47 @@ def _read_graph_spec(
         )
 
     outputs = session.get_outputs()
+    layout = vocabulary.onnx_variant.output_layout
+    if layout == RFDETR_DETS_LABELS:
+        output_names, num_queries, num_classes = _check_rfdetr_outputs(outputs, vocabulary)
+    elif layout == ANCHOR_4_PLUS_NC:
+        output_names, num_queries, num_classes = _check_anchor_outputs(outputs, vocabulary)
+    elif layout == END2END_N_6:
+        output_names, num_queries, num_classes = _check_end2end_outputs(outputs, vocabulary)
+    else:  # pragma: no cover - `VariantContract` refuses an unknown layout first
+        raise ModelContractError(
+            f"{vocabulary.model_name} declares output layout {layout!r}, which this build "
+            "cannot decode. Refusing rather than returning an empty frame."
+        )
+
+    return GraphSpec(
+        input_name=inputs[0].name,
+        output_names=output_names,
+        output_layout=layout,
+        preprocessing=PREPROCESSING_BY_LAYOUT[layout],
+        height=int(height),
+        width=int(width),
+        num_queries=num_queries,
+        num_classes=num_classes,
+        providers=list(session.get_providers()),
+    )
+
+
+def _sole_output(outputs, layout: str):
+    """The single output a YOLO graph must have, or a refusal naming both.
+
+    A second output on a graph declared single is not a graph we can decode by
+    taking the first one: whichever tensor `output0` happens to be, the other one
+    is a fact about this artifact nobody wrote down.
+    """
+    if len(outputs) != 1:
+        raise ModelContractError(
+            f"{layout} expects exactly one graph output, found {[o.name for o in outputs]}"
+        )
+    return outputs[0]
+
+
+def _check_rfdetr_outputs(outputs, vocabulary: ModelVocabulary):
     by_name = {o.name: o for o in outputs}
     if "dets" not in by_name or "labels" not in by_name:
         raise ModelContractError(
@@ -356,17 +527,50 @@ def _read_graph_spec(
             f"{vocabulary.model_name} describes {vocabulary.logit_slots}. Refusing rather "
             "than labelling boxes by an index that vocabulary does not name — see labels.py."
         )
+    queries = int(logit_shape[1]) if isinstance(logit_shape[1], int) else -1
+    return ("dets", "labels"), queries, int(num_classes)
 
-    return GraphSpec(
-        input_name=inputs[0].name,
-        box_output="dets",
-        logit_output="labels",
-        height=int(height),
-        width=int(width),
-        num_queries=int(logit_shape[1]) if isinstance(logit_shape[1], int) else -1,
-        num_classes=int(num_classes),
-        providers=list(session.get_providers()),
-    )
+
+def _check_anchor_outputs(outputs, vocabulary: ModelVocabulary):
+    """`[1, 4+nc, A]` — and the class axis is the whole check.
+
+    The 4 box channels come FIRST and the class scores after them, so an axis of
+    the wrong width does not fail, it SHIFTS: every class name lands on its
+    neighbour's scores and every box keeps a channel that belonged to a class.
+    `run_candidate.py` refuses the same mismatch in the same words — "refusing
+    rather than labelling a channel by position".
+    """
+    output = _sole_output(outputs, ANCHOR_4_PLUS_NC)
+    shape = output.shape
+    if len(shape) != 3:
+        raise ModelContractError(f"{ANCHOR_4_PLUS_NC} is not (1, 4+nc, anchors): {shape}")
+    expected = 4 + vocabulary.logit_slots
+    declared = shape[1]
+    if isinstance(declared, int) and declared != expected:
+        raise ModelContractError(
+            f"output axis is {declared} wide; 4 box channels + {vocabulary.logit_slots} named "
+            f"classes needs {expected}. Refusing rather than labelling a channel by position."
+        )
+    anchors = int(shape[2]) if isinstance(shape[2], int) else -1
+    return (output.name,), anchors, int(vocabulary.logit_slots)
+
+
+def _check_end2end_outputs(outputs, vocabulary: ModelVocabulary):
+    """`[1, N, 6]` — the six is the contract and the N is not.
+
+    The class index rides IN a column here rather than being the axis, so there
+    is nothing on the graph to check the vocabulary's width against; a row naming
+    a slot this model does not have is caught per-detection by `_to_wire`, which
+    drops it and counts it as `unnamed_class_dropped`.
+    """
+    output = _sole_output(outputs, END2END_N_6)
+    shape = output.shape
+    if len(shape) != 3 or shape[2] != 6:
+        raise ModelContractError(
+            f"{END2END_N_6} is not (1, N, 6) — x1,y1,x2,y2,score,cls: {shape}"
+        )
+    rows = int(shape[1]) if isinstance(shape[1], int) else -1
+    return (output.name,), rows, int(vocabulary.logit_slots)
 
 
 #: Re-exported so `from .session import DetectionBatch` keeps working. ONE
@@ -376,8 +580,11 @@ __all__ = [
     "Detection",
     "DetectionBatch",
     "GraphSpec",
+    "LETTERBOX_FILL",
     "ModelContractError",
     "OnnxDetector",
     "postprocess",
     "preprocess",
+    "preprocess_for_layout",
+    "preprocess_letterbox",
 ]

package/dist/EdgeAgent.js

@@ -44,7 +44,7 @@ const edge_ads_1 = require("@trillboards/edge-ads");
 const CommandRouter_1 = require("./CommandRouter");
 const config_1 = require("./config");
 const dataDir_1 = require("./dataDir");
-const SDK_VERSION = '0.2.89';
+const SDK_VERSION = '0.2.91';
 class EdgeAgent extends events_1.EventEmitter {
     constructor(config, deps) {
         super();

package/dist/diagnose.d.ts

@@ -31,7 +31,7 @@ export interface DiagnosticReport {
         skipped: number;
     };
 }
-declare const SDK_VERSION = "0.2.89";
+declare const SDK_VERSION = "0.2.91";
 declare function checkNodeVersion(): CheckResult;
 declare function checkPlatform(): CheckResult;
 declare function checkMemory(): CheckResult;

package/dist/diagnose.js

@@ -63,7 +63,7 @@ const localNetworkAccess_1 = require("./localNetworkAccess");
 // ---------------------------------------------------------------------------
 // SDK version — mirrors package.json; kept in sync by CI
 // ---------------------------------------------------------------------------
-const SDK_VERSION = '0.2.89';
+const SDK_VERSION = '0.2.91';
 exports._SDK_VERSION = SDK_VERSION;
 // ---------------------------------------------------------------------------
 // ANSI helpers (no external deps)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trillboards/edge-sdk",
-  "version": "0.2.89",
+  "version": "0.2.91",
   "description": "Composable cross-platform Edge AI SDK for audience sensing on DOOH/CTV devices",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",