@trillboards/edge-sdk 0.2.92 → 0.2.94

package/deploy/detector/tests/test_litert_twin_parity_contract.py

@@ -0,0 +1,165 @@
+"""The committed twin-parity measurement must stay a MEASUREMENT.
+
+`bench/litert-twin-parity.json` is the only place the box-scale-per-layout fact
+survives into CI: the verifier that produced it needs both runtimes and ~1 GB of
+wheels, so it is a documented manual run. That makes the JSON the artifact a
+future edit can quietly falsify — flip one `pass` to true, drop the failing row,
+or widen a tolerance, and nothing downstream notices.
+
+STDLIB ONLY, and unittest rather than pytest, because the `detector-service`
+step in `ci.yml` runs `python3 -m unittest discover` with no pip install. An
+earlier draft of this file imported the verifier (and therefore numpy) at module
+scope and would have errored the whole lane before its existing tests ran, while
+also contributing no collected cases of its own — a check that appears to run
+and does not, which is precisely the defect the measurement below is about.
+
+These are pure-logic checks over committed bytes. They do not re-measure; they
+assert the recorded numbers are internally consistent with the code that claims
+to have produced them, which is the part a hand-edit breaks.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import unittest
+from pathlib import Path
+
+BENCH = Path(__file__).resolve().parent.parent / "bench"
+CONTRACT = BENCH / "litert-twin-parity.json"
+
+sys.path.insert(0, str(BENCH))
+
+from twin_parity_contract import (  # noqa: E402
+    ANCHOR,
+    BOX_TOL_PX,
+    CLS_TOL,
+    COMPARABLE_LAYOUTS,
+    DETS_LABELS,
+    END2END,
+    ParityError,
+    box_scale_for,
+    classify_layout,
+    verdict,
+)
+
+#: The artifact whose row-level parity could NOT be established, named so the
+#: assertion below cannot pass by the row being deleted. See the test.
+UNVERIFIED_END2END_ARTIFACT = "queue_supermarket.onnx"
+
+
+class TwinParityContract(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.contract = json.loads(CONTRACT.read_text())
+        cls.twins = cls.contract["twins"]
+
+    def test_tolerances_match_the_verifier(self):
+        """A widened tolerance in the JSON that the code does not share is a lie."""
+        self.assertEqual(self.contract["tolerances"]["box_residual_px"], BOX_TOL_PX)
+        self.assertEqual(self.contract["tolerances"]["cls_residual"], CLS_TOL)
+
+    def test_every_row_used_the_scale_its_layout_dictates(self):
+        for row in self.twins:
+            with self.subTest(onnx=row["onnx"]):
+                expected = box_scale_for(row["layout"], row["imgsz"])
+                self.assertEqual(
+                    row["box_scale_applied"], expected,
+                    f"{row['onnx']} recorded scale x{row['box_scale_applied']} but layout "
+                    f"{row['layout']} dictates x{expected}",
+                )
+
+    def test_recorded_verdicts_follow_from_recorded_residuals(self):
+        """`pass` must be derivable from the numbers beside it, not asserted."""
+        for row in self.twins:
+            with self.subTest(onnx=row["onnx"]):
+                ok, why = verdict(row)
+                self.assertEqual(
+                    ok, row["pass"],
+                    f"{row['onnx']}: recorded pass={row['pass']} but verdict says {ok} ({why})",
+                )
+
+    def test_the_anchor_scale_is_load_bearing_on_every_anchor_row(self):
+        """An anchor twin whose RAW boxes already match is not normalised.
+
+        This is the 640x bug's signature. If it ever holds, the row passed for
+        the wrong reason and the layout was misread.
+        """
+        anchors = [r for r in self.twins if r["layout"] == ANCHOR]
+        self.assertTrue(anchors, "the contract has no anchor-layout rows to check")
+        for row in anchors:
+            with self.subTest(onnx=row["onnx"]):
+                self.assertGreater(
+                    row["box_residual_unscaled"], BOX_TOL_PX,
+                    f"{row['onnx']}: unscaled residual {row['box_residual_unscaled']} is already "
+                    "inside tolerance, so multiplying by imgsz changed nothing",
+                )
+
+    def test_the_unverified_end2end_row_is_present_and_still_failing(self):
+        """We do NOT get to publish a twin we could not compare.
+
+        `queue_supermarket` returns zero detections above 0.25 on 40 real
+        production frames, so its row-level parity is unknown and it is recorded
+        failing on purpose.
+
+        The PRESENCE assertion is the load-bearing half. Checking only that
+        every `end2end_n_6` row still fails passes vacuously the moment somebody
+        deletes the row — which is exactly the hand-edit this file exists to
+        prevent, and the same "absence read as success" shape as an empty corpus
+        scoring `overallPass: true`. So the row is named.
+        """
+        by_name = {r["onnx"]: r for r in self.twins}
+        self.assertIn(
+            UNVERIFIED_END2END_ARTIFACT, by_name,
+            f"{UNVERIFIED_END2END_ARTIFACT} is missing from the contract. It is the only "
+            f"{END2END} twin and its parity is unverified; removing the row does not make "
+            "it verified. If it was genuinely re-measured on a corpus it fires on, update "
+            "this test with that corpus named.",
+        )
+        row = by_name[UNVERIFIED_END2END_ARTIFACT]
+        self.assertEqual(row["layout"], END2END)
+        self.assertFalse(
+            row["pass"],
+            f"{UNVERIFIED_END2END_ARTIFACT} is recorded passing — see the message above",
+        )
+
+    def test_layout_is_classified_from_the_graph_not_the_values(self):
+        """Shape and output names decide the layout. Nothing else may.
+
+        Dispatching on `shape[1] >= 5` alone routes a [1, 300, 6] end2end tensor
+        into the anchor branch and produces a confident, meaningless class
+        residual. A layout guess that reads the numbers it is interpreting
+        always finds a self-consistent story.
+        """
+        self.assertEqual(classify_layout((1, 300, 6), ["output0"]), END2END)
+        self.assertEqual(classify_layout((1, 14, 8400), ["output0"]), ANCHOR)
+        self.assertEqual(classify_layout((1, 6, 8400), ["output0"]), ANCHOR)
+        self.assertEqual(classify_layout((1, 300, 4), ["dets", "labels"]), DETS_LABELS)
+        with self.assertRaises(ParityError):
+            classify_layout((1, 8400), ["output0"])
+
+    def test_no_layout_gets_a_scale_by_default(self):
+        """An unknown layout must refuse, never fall through to 1.0.
+
+        A silent wrong-layout decode is worse than a refusal: it returns boxes,
+        and boxes that are plausible and wrong are what this file is about.
+        """
+        with self.assertRaises(ParityError):
+            box_scale_for("some_future_layout", 640)
+
+    def test_rfdetr_is_classified_but_not_advertised_as_comparable(self):
+        """Naming a layout is not the same as being able to compare it.
+
+        `classify_layout` recognises RF-DETR's two-output form and
+        `box_scale_for` knows its scale is 1.0, but this bench cannot split a
+        two-output graph into box and class halves, and RF-DETR ships no LiteRT
+        twin. So it must be absent from the comparable set — otherwise the
+        verifier advertises a capability that fails at the point of use.
+        """
+        self.assertNotIn(DETS_LABELS, COMPARABLE_LAYOUTS)
+        self.assertEqual(COMPARABLE_LAYOUTS, frozenset({ANCHOR, END2END}))
+        self.assertEqual(box_scale_for(DETS_LABELS, 640), 1.0)
+
+
+if __name__ == "__main__":
+    unittest.main()