@trillboards/edge-sdk 0.2.85 → 0.2.91

package/deploy/detector/tests/test_server.py

@@ -24,7 +24,14 @@ import urllib.request
 from dataclasses import dataclass
 
 from trillboards_detector.errors import DetectorNotReady, UnknownClasses
-from trillboards_detector.labels import RFDETR_MEDIUM, ClassIndex, ModelVocabulary
+from trillboards_detector.labels import (
+    ANCHOR_4_PLUS_NC,
+    BOX_UNITS_PIXELS,
+    RFDETR_MEDIUM,
+    ClassIndex,
+    ModelVocabulary,
+    VariantContract,
+)
 from trillboards_detector.server import DetectorServer, MAX_BODY_BYTES
 
 JPEG = bytes([0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46])
@@ -494,6 +501,7 @@ class TestClassFilter(ServerCase):
             slot_names={0: "glove", 1: "hairnet"},
             logit_slots=2,
             approx_ram_bytes=400 * 1024 * 1024,
+            onnx_variant=VariantContract(ANCHOR_4_PLUS_NC, BOX_UNITS_PIXELS, 1.0),
         )
         loader = FakeLoader(vocabularies=(RFDETR_MEDIUM, ppe))
         url = self.serve(loader)

package/deploy/detector/tests/test_session.py

@@ -12,17 +12,34 @@ needs no third-party package at all and is the half that must always run.
 
 from __future__ import annotations
 
+import ast
+import pathlib
 import unittest
 
 try:
     import numpy as np
 
-    from trillboards_detector.labels import RFDETR_MEDIUM
+    from trillboards_detector.labels import (
+        ANCHOR_4_PLUS_NC,
+        BOX_UNITS_NORMALIZED,
+        BOX_UNITS_PIXELS,
+        END2END_N_6,
+        RFDETR_MEDIUM,
+        ModelVocabulary,
+        VariantContract,
+    )
+    from trillboards_detector.decode import (
+        LetterboxGeometry,
+        decode_anchor_4_plus_nc,
+    )
     from trillboards_detector.session import (
         ModelContractError,
         _read_graph_spec,
         _sigmoid,
+        _to_wire,
         postprocess,
+        preprocess_for_layout,
+        preprocess_letterbox,
     )
 
     HAVE_NUMPY = True
@@ -38,6 +55,18 @@ def _logits(pairs, num_queries=3, num_classes=91):
     return out
 
 
+def _solid_jpeg(width, height, colour=(200, 50, 50)):
+    """A real JPEG of a solid colour. Small enough to be free, real enough to
+    decode — the same reasoning as `loader._warmup_jpeg`."""
+    import io as _io
+
+    from PIL import Image as _Image
+
+    buf = _io.BytesIO()
+    _Image.new("RGB", (width, height), colour).save(buf, format="JPEG", quality=95)
+    return buf.getvalue()
+
+
 def _boxes(rows, num_queries=3):
     out = np.zeros((1, num_queries, 4), dtype=np.float32)
     for q, box in rows.items():
@@ -256,6 +285,378 @@ class TestGraphContract(unittest.TestCase):
         with self.assertRaisesRegex(ModelContractError, "one graph input"):
             _read_graph_spec(_FakeSession(two, self.GOOD_OUT), RFDETR_MEDIUM)
 
+    def test_the_shipped_graph_reports_its_layout_and_preprocessing(self):
+        """`/health` has to be able to say which decode ran. Two artifacts of
+        different layouts produce equally plausible tables."""
+        spec = _read_graph_spec(_FakeSession(self.GOOD_IN, self.GOOD_OUT), RFDETR_MEDIUM)
+        self.assertEqual(spec.output_layout, "rfdetr_dets_labels")
+        self.assertEqual(spec.preprocessing, "imagenet-normalised-stretch")
+        self.assertEqual(spec.output_names, ("dets", "labels"))
+
+
+def _yolo_vocabulary(layout, classes=2, units=None, scale=1.0):
+    """A specialist vocabulary for the graph-contract cases.
+
+    `units` defaults to None and is resolved in the BODY, not in the signature.
+    A default argument is evaluated at import time, so naming `BOX_UNITS_PIXELS`
+    there raises `NameError` on a bare interpreter where the guarded import at
+    the top of this file never ran — which is every CI runner without
+    onnxruntime, and would take the whole module down as an ERROR rather than a
+    skip. Caught by running the suite with numpy, ORT and Pillow made
+    unimportable.
+    """
+    return ModelVocabulary(
+        model_name="ppe",
+        slot_names={i: name for i, name in enumerate(["glove", "hairnet", "apron"][:classes])},
+        logit_slots=classes,
+        approx_ram_bytes=400 * 1024 * 1024,
+        onnx_variant=VariantContract(layout, units or BOX_UNITS_PIXELS, scale),
+    )
+
+
+@unittest.skipUnless(HAVE_NUMPY, "numpy is not installed")
+class TestYoloGraphContract(unittest.TestCase):
+    """The declared layout has to survive contact with the graph, or refuse.
+
+    This is where a mis-declared artifact is caught. `[1, 300, 6]` decoded as
+    `[1, 4+nc, A]` reads x1,y1,x2,y2,score,cls as four box channels and two
+    classes; the reverse reads 8,400 anchors as 8,400 detections. Neither raises
+    anywhere downstream — both produce a full table — so the refusal has to
+    happen here, at load, where it takes the model UNAVAILABLE and the request
+    becomes a 503 rather than a plausible answer.
+    """
+
+    IN = [_FakeIo("images", [1, 3, 640, 640])]
+
+    def test_accepts_an_anchor_head_whose_axis_matches_the_vocabulary(self):
+        out = [_FakeIo("output0", [1, 6, 8400])]
+        spec = _read_graph_spec(_FakeSession(self.IN, out), _yolo_vocabulary(ANCHOR_4_PLUS_NC))
+        self.assertEqual(spec.output_layout, ANCHOR_4_PLUS_NC)
+        self.assertEqual(spec.output_names, ("output0",))
+        self.assertEqual(spec.preprocessing, "ultralytics-letterbox-114-centred")
+        self.assertEqual((spec.width, spec.height), (640, 640))
+        self.assertEqual(spec.num_classes, 2)
+        self.assertEqual(spec.num_queries, 8400)
+
+    def test_refuses_an_anchor_axis_the_vocabulary_does_not_describe(self):
+        """4 box channels + 2 named classes needs 6. A 7-wide axis shifts every
+        class name onto its neighbour's scores."""
+        out = [_FakeIo("output0", [1, 7, 8400])]
+        with self.assertRaisesRegex(ModelContractError, "labelling a channel by position"):
+            _read_graph_spec(_FakeSession(self.IN, out), _yolo_vocabulary(ANCHOR_4_PLUS_NC))
+
+    def test_refuses_an_end2end_graph_declared_as_an_anchor_head(self):
+        out = [_FakeIo("output0", [1, 300, 6])]
+        with self.assertRaisesRegex(ModelContractError, "labelling a channel by position"):
+            _read_graph_spec(_FakeSession(self.IN, out), _yolo_vocabulary(ANCHOR_4_PLUS_NC, 2))
+
+    def test_accepts_an_end2end_graph(self):
+        out = [_FakeIo("output0", [1, 300, 6])]
+        spec = _read_graph_spec(_FakeSession(self.IN, out), _yolo_vocabulary(END2END_N_6))
+        self.assertEqual(spec.output_layout, END2END_N_6)
+        self.assertEqual(spec.num_queries, 300)
+
+    def test_refuses_an_anchor_graph_declared_as_end2end(self):
+        """The exact inversion the declared layout exists to catch: `[1, 6, 8400]`
+        has a 6 in it too, and reading it as rows would return 6 detections whose
+        `cls` column is a box coordinate."""
+        out = [_FakeIo("output0", [1, 6, 8400])]
+        with self.assertRaisesRegex(ModelContractError, r"x1,y1,x2,y2,score,cls"):
+            _read_graph_spec(_FakeSession(self.IN, out), _yolo_vocabulary(END2END_N_6))
+
+    def test_refuses_a_second_output_on_a_single_output_layout(self):
+        out = [_FakeIo("output0", [1, 6, 8400]), _FakeIo("proto", [1, 32, 160, 160])]
+        with self.assertRaisesRegex(ModelContractError, "exactly one graph output"):
+            _read_graph_spec(_FakeSession(self.IN, out), _yolo_vocabulary(ANCHOR_4_PLUS_NC))
+
+    def test_refuses_a_dynamic_resolution_on_a_yolo_head_too(self):
+        dyn = [_FakeIo("images", [1, 3, "height", "width"])]
+        out = [_FakeIo("output0", [1, 6, 8400])]
+        with self.assertRaisesRegex(ModelContractError, "fixed input resolution"):
+            _read_graph_spec(_FakeSession(dyn, out), _yolo_vocabulary(ANCHOR_4_PLUS_NC))
+
+    def test_a_normalized_variant_loads_the_same_graph(self):
+        """The LiteRT twin of these weights is `normalized`/640 and the ONNX one
+        is `pixels`/1. The GRAPH is identical, which is the whole trap: nothing
+        about the shape distinguishes them."""
+        out = [_FakeIo("output0", [1, 6, 8400])]
+        spec = _read_graph_spec(
+            _FakeSession(self.IN, out),
+            _yolo_vocabulary(ANCHOR_4_PLUS_NC, 2, BOX_UNITS_NORMALIZED, 640.0),
+        )
+        self.assertEqual(spec.output_layout, ANCHOR_4_PLUS_NC)
+
+
+@unittest.skipUnless(HAVE_NUMPY, "numpy is not installed")
+class TestLetterboxPreprocessing(unittest.TestCase):
+    """The Ultralytics transform, and the padding convention the decode undoes."""
+
+    def test_aspect_is_preserved_and_the_remainder_is_114_grey(self):
+        blob, geometry = preprocess_letterbox(_solid_jpeg(1200, 900), 640, 640)
+        self.assertEqual(blob.shape, (1, 3, 640, 640))
+        self.assertAlmostEqual(geometry.gain, 640.0 / 1200.0, places=9)
+        self.assertEqual((geometry.pad_x, geometry.pad_y), (0.0, 80.0))
+        self.assertEqual((geometry.frame_w, geometry.frame_h), (1200, 900))
+        # The top row is padding, and 114/255 is what the model saw in training.
+        self.assertTrue(np.allclose(blob[0, :, 0, :], 114.0 / 255.0, atol=1e-6))
+        # The centre is content, not padding.
+        self.assertFalse(np.allclose(blob[0, :, 320, 320], 114.0 / 255.0, atol=1e-3))
+
+    def test_pixels_are_scaled_by_255_and_not_imagenet_normalised(self):
+        """`preprocess` subtracts a mean and divides by a standard deviation and
+        this one must not: on a mid-grey frame that is the difference between
+        ~0.45 and ~-0.15, and no shape or dtype check can see it."""
+        blob, _ = preprocess_letterbox(_solid_jpeg(64, 64, (128, 128, 128)), 640, 640)
+        self.assertGreaterEqual(float(blob.min()), 0.0)
+        self.assertLessEqual(float(blob.max()), 1.0)
+        self.assertAlmostEqual(float(blob[0, 0, 320, 320]), 128.0 / 255.0, places=2)
+
+    def test_the_padding_convention_matches_ultralytics_scale_boxes_for_every_input(self):
+        """`(W - new_w) // 2` here versus `round(dw - 0.1)` in
+        `ops.scale_boxes`. They agree for every width because dw is either an
+        integer or a half-integer, and both forms floor it — asserted rather than
+        argued, since a one-pixel disagreement would shift every box by one pixel
+        and nothing would report it."""
+        for width in range(0, 641):
+            for remainder in (width, 640 - width):
+                dw = remainder / 2.0
+                self.assertEqual(remainder // 2, round(dw - 0.1), f"remainder={remainder}")
+
+    def test_a_square_frame_needs_no_padding(self):
+        _, geometry = preprocess_letterbox(_solid_jpeg(320, 320), 640, 640)
+        self.assertEqual((geometry.pad_x, geometry.pad_y), (0.0, 0.0))
+        self.assertAlmostEqual(geometry.gain, 2.0, places=9)
+
+
+@unittest.skipUnless(HAVE_NUMPY, "numpy is not installed")
+class TestPreprocessorDispatch(unittest.TestCase):
+    """The layout picks the transform, and nothing else may.
+
+    Handing an Ultralytics export the RF-DETR stretch produces a tensor of the
+    right shape and dtype whose every channel is shifted by roughly two standard
+    deviations, on an image whose aspect ratio has been changed. It raises
+    nothing. A mutation test found this dispatch reachable only through a real
+    ORT session — all 178 unit tests stayed green while the two preprocessors
+    were swapped — which is why it is a function with its own test rather than
+    four lines inside `detect`.
+    """
+
+    def test_a_yolo_layout_gets_the_letterbox_and_keeps_the_frame_size(self):
+        for layout in (ANCHOR_4_PLUS_NC, END2END_N_6):
+            blob, geometry, frame_w, frame_h = preprocess_for_layout(
+                layout, _solid_jpeg(1200, 900), 640, 640
+            )
+            self.assertIsNotNone(geometry, layout)
+            self.assertEqual((frame_w, frame_h), (1200, 900))
+            # Padding is present and it is 114-grey, not black and not content.
+            self.assertTrue(np.allclose(blob[0, :, 0, :], 114.0 / 255.0, atol=1e-6), layout)
+            # /255 only: an ImageNet-normalised tensor goes negative.
+            self.assertGreaterEqual(float(blob.min()), 0.0, layout)
+
+    def test_the_rfdetr_layout_gets_the_stretch_and_no_geometry(self):
+        blob, geometry, frame_w, frame_h = preprocess_for_layout(
+            "rfdetr_dets_labels", _solid_jpeg(1200, 900), 576, 576
+        )
+        self.assertIsNone(geometry)
+        self.assertEqual((frame_w, frame_h), (1200, 900))
+        self.assertEqual(blob.shape, (1, 3, 576, 576))
+        # ImageNet normalisation puts a mid-tone below zero; the letterbox path
+        # cannot. That sign is the cheapest thing that tells the two apart.
+        self.assertLess(float(blob.min()), 0.0)
+        # A STRETCH leaves no padding: the top row is content, not 114-grey.
+        self.assertFalse(np.allclose(blob[0, :, 0, :], 114.0 / 255.0, atol=1e-3))
+
+
+@unittest.skipUnless(HAVE_NUMPY, "numpy is not installed")
+class TestWrongBoxUnitsAtTheWire(unittest.TestCase):
+    """What a WRONG `box_units` declaration actually does to a caller.
+
+    IT DEPENDS ON THE FRAME, NOT ON THE DIRECTION, and that is the whole finding.
+    Measured on `ppe_keremberke_v8n` (ONNX), 200 frames of `safety-gloves-v1`,
+    with the declaration as the only variable and the frame's aspect ratio as the
+    only other:
+
+        frame as submitted   declared   on wire   degenerate_dropped   widths
+        letterboxed (pad>0)  correct        848                    0   33.5-250.5 px
+        letterboxed (pad>0)  WRONG            0                  848   -
+        square (pad==0)      correct        707                    0   18.2-122.0 px
+        square (pad==0)      WRONG          707                    0   0.028-0.191 px
+
+    On the SQUARE frames the wrong declaration is perfectly silent: same count,
+    **zero** label mismatches over all 707, `max |confidence delta| = 0.0`
+    EXACTLY, and a width ratio of 639.9987-640.0012 — boxes exactly 640x too
+    small and nothing else different. That is the claim `labels.VariantContract`
+    makes, and it holds.
+
+    On the LETTERBOXED frames the same mistake loses everything. `_to_frame_pixels`
+    subtracts the padding BEFORE it clips, so boxes 640x too small land at a
+    negative offset, clip to zero extent, and `_to_wire` drops all of them. That
+    is why a naive check can report a mis-declaration as loud: measure it on
+    padded frames and it always is.
+
+    Both regimes are reachable in production — the connector submits whatever the
+    camera gives it — so neither is "the" behaviour, and a mis-declaration cannot
+    be relied on to announce itself. It is a declaration checked against a
+    measurement (`bench/run_candidate.py`) for exactly that reason.
+
+    The fixtures below reproduce each regime with a hand-built tensor: an
+    identity geometry is the square case, a padded geometry the letterboxed one.
+    """
+
+    @staticmethod
+    def _vocabulary():
+        return ModelVocabulary(
+            model_name="probe",
+            slot_names={0: "glove", 1: "hairnet"},
+            logit_slots=2,
+            approx_ram_bytes=1,
+            onnx_variant=VariantContract(ANCHOR_4_PLUS_NC, BOX_UNITS_PIXELS, 1.0),
+        )
+
+    @staticmethod
+    def _tensor(cx, cy, w, h):
+        """One anchor, one class, whatever units the caller means them to be."""
+        grid = np.zeros((1, 6, 1), dtype=np.float32)
+        grid[0, 0, 0], grid[0, 1, 0], grid[0, 2, 0], grid[0, 3, 0] = cx, cy, w, h
+        grid[0, 4, 0] = 0.9
+        return grid
+
+    def _wire(self, raw, scale, geometry=None):
+        if geometry is None:  # the SQUARE case: nothing was padded away
+            geometry = LetterboxGeometry(1.0, 0.0, 0.0, 640, 640)
+        scores, classes, x0, y0, x1, y1 = decode_anchor_4_plus_nc(
+            raw, geometry, 0.25, box_scale_to_pixels=scale, expected_classes=2
+        )
+        return _to_wire(scores, classes, x0, y0, x1, y1, None, self._vocabulary())
+
+    def test_on_a_square_frame_a_wrong_declaration_is_completely_silent(self):
+        """The 640x error survives to the wire with everything else intact.
+
+        Right count, right label, byte-identical confidence, and a box two
+        orders of magnitude too small. Nothing in the payload says it is wrong,
+        which is why there is no default `box_units`.
+        """
+        normalized = self._tensor(0.5, 0.5, 0.1, 0.1)
+        good = self._wire(normalized, 640.0)
+        self.assertEqual(len(good.detections), 1)
+        self.assertAlmostEqual(good.detections[0].width, 64.0, places=3)
+
+        bad = self._wire(normalized, 1.0)
+        self.assertEqual(len(bad.detections), 1)
+        self.assertEqual(bad.degenerate_dropped, 0)
+        self.assertEqual(bad.detections[0].label, good.detections[0].label)
+        self.assertEqual(bad.detections[0].confidence, good.detections[0].confidence)
+        self.assertAlmostEqual(
+            good.detections[0].width / bad.detections[0].width, 640.0, places=2
+        )
+
+    def test_on_a_letterboxed_frame_the_same_mistake_loses_every_detection(self):
+        """Padding is subtracted before the clip, so the sub-pixel box clamps away.
+
+        Same tensor, same wrong declaration as the test above; only the geometry
+        differs. This is the regime a padded camera frame is in, and it is why
+        "we would have noticed" is not an argument.
+        """
+        padded = LetterboxGeometry(0.5, 0.0, 80.0, 1280, 960)
+        normalized = self._tensor(0.5, 0.5, 0.1, 0.1)
+
+        good = self._wire(normalized, 640.0, padded)
+        self.assertEqual(len(good.detections), 1)
+
+        bad = self._wire(normalized, 1.0, padded)
+        self.assertEqual(bad.detections, [])
+        self.assertEqual(bad.degenerate_dropped, 1)
+
+    def test_a_pixel_artifact_declared_normalized_loses_every_detection(self):
+        """The other direction, on a square frame: 640x too BIG clamps flush."""
+        pixels = self._tensor(320.0, 320.0, 64.0, 64.0)
+        good = self._wire(pixels, 1.0)
+        self.assertEqual(len(good.detections), 1)
+        self.assertAlmostEqual(good.detections[0].width, 64.0, places=3)
+        self.assertEqual(good.degenerate_dropped, 0)
+
+        bad = self._wire(pixels, 640.0)
+        self.assertEqual(bad.detections, [])
+        self.assertEqual(bad.degenerate_dropped, 1)
+
+
+class TestEveryVocabularyConstructionDeclaresItsVariant(unittest.TestCase):
+    """Every `ModelVocabulary(...)` in the tree must supply `onnx_variant`.
+
+    STDLIB ONLY, ON PURPOSE. The thing this guards lives in `bench/`, which
+    imports onnxruntime at module scope and therefore cannot be imported by the
+    `detector-service` CI lane at all — so a runtime test would skip in exactly
+    the environment that has to catch this, and pass vacuously everywhere else.
+    `ast` needs nothing installed, so this runs on the bare interpreter.
+
+    It exists because making `onnx_variant` REQUIRED broke three call sites in
+    `bench/run_candidate.py` — including `probe_loadability`, which decides
+    whether a candidate is publishable, and the `ShippedRunner` construction on
+    the scoring path — and every one of them failed with a `TypeError` at
+    runtime that no test in this suite could reach. A missing required argument
+    is a static fact; check it statically.
+    """
+
+    #: `labels.py` DEFINES the dataclass, so its own signature is not a call.
+    _EXEMPT = {"labels.py"}
+
+    def _roots(self):
+        """The detector package, plus the repo's own callers WHEN THEY EXIST.
+
+        The first version of this test scanned only the three directories under
+        `deploy/detector` and passed while `scripts/sensing/onnx_artifact_probe.py`
+        held a fourth broken construction — the guard's blind spot was the reason
+        the bug survived the guard. Anything in the repo that imports the
+        detector package is a caller and belongs in scope.
+
+        `deploy/` ships inside the npm tarball, so on an installed connector the
+        repo roots are simply absent; they are added only if present rather than
+        asserted, and the detector roots always are.
+        """
+        here = pathlib.Path(__file__).resolve().parent.parent
+        roots = [here / "trillboards_detector", here / "bench", here / "tests"]
+        for parent in here.parents:
+            if (parent / ".git").exists():
+                roots.append(parent / "scripts" / "sensing")
+                break
+        return roots
+
+    def test_no_construction_omits_the_variant_contract(self):
+        offenders = []
+        for root in self._roots():
+            if not root.is_dir():
+                continue
+            for path in sorted(root.rglob("*.py")):
+                if path.name in self._EXEMPT:
+                    continue
+                try:
+                    tree = ast.parse(path.read_text(encoding="utf-8"))
+                except SyntaxError as exc:  # a file that cannot parse is its own failure
+                    offenders.append(f"{path.name}: unparseable ({exc})")
+                    continue
+                for node in ast.walk(tree):
+                    if not isinstance(node, ast.Call):
+                        continue
+                    func = node.func
+                    name = getattr(func, "id", None) or getattr(func, "attr", None)
+                    if name != "ModelVocabulary":
+                        continue
+                    kwargs = {kw.arg for kw in node.keywords if kw.arg}
+                    has_kwsplat = any(kw.arg is None for kw in node.keywords)
+                    # 5 positional args reaches `onnx_variant`; `**kwargs` is opaque
+                    # and is trusted rather than guessed at.
+                    if "onnx_variant" in kwargs or len(node.args) >= 5 or has_kwsplat:
+                        continue
+                    offenders.append(f"{path.name}:{node.lineno}")
+        self.assertEqual(
+            offenders,
+            [],
+            "ModelVocabulary requires onnx_variant — a variant whose output layout is "
+            "inferred is a variant whose boxes are plausible and wrong. Omitting it is a "
+            "TypeError at runtime, and these call sites are not reachable from this "
+            f"suite: {offenders}",
+        )
+
 
 if __name__ == "__main__":
     unittest.main()