@trillboards/edge-sdk 0.2.85 → 0.2.91

package/deploy/detector/bench/verify_detector_decode.py

@@ -0,0 +1,574 @@
+#!/usr/bin/env python3
+"""verify_detector_decode — are the YOLO decodes the same instrument Ultralytics is?
+
+`verify_against_reference.py` does this for the RF-DETR path against the `rfdetr`
+package. This file does it for the two Ultralytics layouts against `ultralytics`
+itself, and it compares BOXES rather than presence, because boxes are where every
+one of this decode's failure modes lives: a transposed axis, a doubled sigmoid,
+suppression skipped or run twice, and a coordinate space read in the wrong units
+all leave the class names and the frame count untouched.
+
+THREE ARMS, EACH ISOLATING ONE VARIABLE
+---------------------------------------
+`--arm decode` (the strict one). Our decode and Ultralytics'
+`utils/nms.non_max_suppression` + `utils/ops.scale_boxes` run on THE SAME RAW
+TENSOR. Preprocessing, the model and the frame are all held identical, so the
+only thing that can differ is the arithmetic under test. Agreement is asserted in
+pixels, on every detection, at 0.0.
+
+`--arm pipeline`. The whole service path — JPEG in, `OnnxDetector.detect` out —
+against `YOLO(...).predict()`. The reference is handed OUR letterboxed 640x640
+frame, so its own `LetterBox` is the identity and the resize kernel drops out.
+Without that step the two arms disagree for a reason that is not a decode bug:
+Pillow's BILINEAR low-passes on downscale and Ultralytics' cv2 INTER_LINEAR does
+not, and on a 1200x900 frame that moved one class from 0.18 to 0.51. That is §5
+rule 1 one level down — an arm at one resize kernel compared against an arm at
+another is not a comparison — and it is the same trap as comparing against the
+`.pt`, which letterboxes to 640x480 while the ONNX graph is fixed 640x640.
+
+`--arm litert`. The SAME weights through both runtimes, decoded by THIS decode,
+with each side's DECLARED `box_scale_to_pixels` applied. This is the arm the
+`box_units` column exists for: 16 of 17 committed twins emit ONNX boxes in pixels
+and LiteRT boxes in 0..1, every shape and dtype check passes on the pair, and a
+consumer reusing the ONNX decode draws every box 640x too small in silence. It
+needs a manifest naming each twin's declared units (`--litert-manifest`), because
+the point is to check a DECLARATION against a measurement — deriving the units
+from the tensor here would make the arm prove nothing.
+
+Needs `ultralytics` (and `ai_edge_litert` for the litert arm), neither of which is
+a dependency of this package: `bench/` is excluded from the wheel, and a
+measurement-only dependency must never reach a connector's runtime. A documented
+manual run, never CI.
+
+    python3 bench/verify_detector_decode.py --arm decode \
+      --models-dir /tmp/onnx --frames-dir /tmp/frames [--limit 8]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))  # noqa: E402
+
+from trillboards_detector.decode import (  # noqa: E402
+    ANCHOR_4_PLUS_NC,
+    END2END_N_6,
+    NMS_IOU_THRESHOLD,
+    YOLO_DECODERS,
+)
+from trillboards_detector.labels import (  # noqa: E402
+    BOX_UNITS_NORMALIZED,
+    BOX_UNITS_PIXELS,
+    ModelVocabulary,
+    VariantContract,
+)
+from trillboards_detector.session import (  # noqa: E402
+    LETTERBOX_FILL,
+    OnnxDetector,
+    preprocess_letterbox,
+)
+
+THRESHOLD = 0.25
+
+#: A score tolerance, ENFORCED, and separate from the box tolerance because the
+#: two catch different faults. `d_score` used to be computed and only reported,
+#: so a decode whose boxes and counts were right and whose CONFIDENCES were
+#: wrong printed AGREE — which is exactly the sigmoid-applied-twice fault this
+#: file's header names, and it would have invalidated every threshold and score
+#: card downstream while the strict arm called it clean. The default is 0.0: the
+#: `decode` arm feeds both sides the same tensor, so any difference at all is
+#: arithmetic, not noise.
+DEFAULT_SCORE_TOLERANCE = 0.0
+
+
+def _graph_layout(session) -> str:
+    """The layout of an artifact under test, for a BENCH and nowhere else.
+
+    The service never does this — its layout is declared and checked. A bench
+    sweeping a directory of candidates has no registry row to read, and inventing
+    one by hand for 96 files would be a hand-list nobody could trust. So the
+    shape is read HERE, in the file whose whole job is to check the decodes, and
+    the service's refusal to do the same is the property being protected.
+    """
+    outputs = session.get_outputs()
+    if len(outputs) == 2:
+        return "rfdetr_dets_labels"
+    shape = outputs[0].shape
+    if len(shape) == 3 and shape[2] == 6:
+        return END2END_N_6
+    if len(shape) == 3:
+        return ANCHOR_4_PLUS_NC
+    raise SystemExit(f"cannot classify {[o.shape for o in outputs]}")
+
+
+def _names(model_path: Path) -> "dict[int, str]":
+    """Ultralytics writes its class list into the ONNX metadata; read it there."""
+    import ast
+
+    import onnxruntime as ort
+
+    opts = ort.SessionOptions()
+    opts.log_severity_level = 3
+    session = ort.InferenceSession(str(model_path), sess_options=opts, providers=["CPUExecutionProvider"])
+    raw = session.get_modelmeta().custom_metadata_map.get("names")
+    if not raw:
+        raise SystemExit(f"{model_path.name} carries no `names` metadata; a class list cannot be guessed")
+    return {int(k): str(v) for k, v in ast.literal_eval(raw).items()}
+
+
+def _canonical(name: str) -> str:
+    return name.strip().lower().replace("_", " ").replace("-", " ")
+
+
+def _vocabulary(model_path: Path, layout: str, units=BOX_UNITS_PIXELS, scale=1.0) -> ModelVocabulary:
+    names = _names(model_path)
+    slots: "dict[int, str]" = {}
+    for slot, raw in names.items():
+        folded = _canonical(raw)
+        # A bench vocabulary must still be a legal one: two slots sharing a name
+        # would make a class filter ambiguous, and `ModelVocabulary` refuses it.
+        while folded in slots.values():
+            folded = f"{folded} {slot}"
+        slots[slot] = folded
+    return ModelVocabulary(
+        model_name="bench",
+        slot_names=slots,
+        logit_slots=len(slots),
+        approx_ram_bytes=1,
+        onnx_variant=VariantContract(layout, units, scale),
+    )
+
+
+def _sorted_index(scores, classes):
+    import numpy as np
+
+    return np.lexsort((classes, -scores))
+
+
+def _compare(mine, theirs, tolerance_px: float, score_tolerance: float, label: str, problems: list):
+    """Count, class, box AND score. All four are enforced; none is only reported.
+
+    THE SCORE CHECK IS NOT OPTIONAL. Boxes and counts can agree exactly while
+    confidences do not — a doubled sigmoid maps a monotone score to a monotone
+    score, so on a frame whose ranking does not change it moves every confidence
+    and nothing else. Computing `d_score` and not asserting on it is how that
+    fault passes the strict arm, and every threshold and score card downstream
+    is a function of the confidences rather than of the boxes.
+    """
+    import numpy as np
+
+    (mb, ms, mc), (rb, rs, rc) = mine, theirs
+    if mb.shape[0] != rb.shape[0]:
+        problems.append(f"{label}: detection COUNT differs — ours {mb.shape[0]}, reference {rb.shape[0]}")
+        return 0, 0.0, 0.0
+    if mb.shape[0] == 0:
+        return 0, 0.0, 0.0
+    a, b = _sorted_index(ms, mc), _sorted_index(rs, rc)
+    if not np.array_equal(mc[a], rc[b]):
+        problems.append(f"{label}: CLASS assignment differs")
+        return mb.shape[0], 0.0, 0.0
+    d_box = float(np.abs(mb[a] - rb[b]).max())
+    d_score = float(np.abs(ms[a] - rs[b]).max())
+    if d_box > tolerance_px:
+        problems.append(f"{label}: box differs by {d_box:.5f} px (tolerance {tolerance_px})")
+    if d_score > score_tolerance:
+        problems.append(
+            f"{label}: CONFIDENCE differs by {d_score:.3g} (tolerance {score_tolerance})"
+        )
+    return mb.shape[0], d_box, d_score
+
+
+def _reference_nms(raw, layout, geometry, width, height):
+    """Ultralytics' own suppression and box rescaling, on our raw tensor."""
+    import torch
+    from ultralytics.utils import nms as unms
+    from ultralytics.utils import ops
+
+    out = unms.non_max_suppression(
+        torch.from_numpy(raw),
+        THRESHOLD,
+        NMS_IOU_THRESHOLD,
+        None,
+        False,
+        max_det=300,
+        nc=0,
+        end2end=layout == END2END_N_6,
+    )[0]
+    boxes = out[:, :4].clone()
+    if boxes.shape[0]:
+        boxes = ops.scale_boxes((height, width), boxes, (geometry.frame_h, geometry.frame_w))
+    return boxes.numpy(), out[:, 4].numpy(), out[:, 5].numpy().astype("int64")
+
+
+def arm_decode(models, frames, tolerance_px, score_tolerance, problems):
+    """Same raw tensor, two decodes. The strict control."""
+    import onnxruntime as ort
+
+    compared = frames_seen = 0
+    worst_box = worst_score = 0.0
+    layouts: "dict[str, int]" = {}
+    # A MODEL THIS ARM NEVER COMPARED IS NOT A MODEL THIS ARM CHECKED, and the
+    # two ways that happens are different facts: `skipped` is a graph whose
+    # layout this arm does not handle (an RF-DETR export swept up by a
+    # directory glob), `unexercised` is one that handled fine and produced no
+    # above-threshold detection on these frames. Counting either into `models`
+    # would let "96 models agree" cover artifacts whose decode never ran — the
+    # same vacuous-row failure the litert arm already names.
+    skipped: "list[str]" = []
+    unexercised: "list[str]" = []
+    for model_path in models:
+        opts = ort.SessionOptions()
+        opts.log_severity_level = 3
+        session = ort.InferenceSession(
+            str(model_path), sess_options=opts, providers=["CPUExecutionProvider"]
+        )
+        layout = _graph_layout(session)
+        if layout not in YOLO_DECODERS:
+            skipped.append(f"{model_path.name} ({layout})")
+            continue
+        layouts[layout] = layouts.get(layout, 0) + 1
+        model_dets = 0
+        graph_in = session.get_inputs()[0]
+        _, _, height, width = graph_in.shape
+        output_name = session.get_outputs()[0].name
+        for frame in frames:
+            blob, geometry = preprocess_letterbox(frame.read_bytes(), width, height)
+            raw = session.run([output_name], {graph_in.name: blob})[0]
+            expected_classes = len(_names(model_path))
+            scores, classes, x0, y0, x1, y1 = YOLO_DECODERS[layout](
+                raw, geometry, THRESHOLD,
+                box_scale_to_pixels=1.0,
+                expected_classes=expected_classes,
+            )
+            import numpy as np
+
+            ours = (np.stack([x0, y0, x1, y1], axis=1) if scores.shape[0] else np.zeros((0, 4)), scores, classes)
+            n, d_box, d_score = _compare(
+                ours,
+                _reference_nms(raw, layout, geometry, width, height),
+                tolerance_px,
+                score_tolerance,
+                f"{model_path.name} {frame.name}",
+                problems,
+            )
+            compared += n
+            model_dets += n
+            frames_seen += 1
+            worst_box, worst_score = max(worst_box, d_box), max(worst_score, d_score)
+        if model_dets == 0:
+            unexercised.append(model_path.name)
+    return {
+        "arm": "decode",
+        "models_in_directories": len(models),
+        "models_compared": sum(layouts.values()) - len(unexercised),
+        "layouts": layouts,
+        # NAMED, never merely counted. Each of these produced no evidence and
+        # saying which is the whole point of reporting them at all.
+        "models_skipped_wrong_layout": skipped,
+        "models_with_no_above_threshold_detection": unexercised,
+        "frames_decoded": frames_seen,
+        "detections_compared": compared,
+        "max_box_delta_px": worst_box,
+        "max_score_delta": worst_score,
+    }
+
+
+def arm_pipeline(models, frames, tolerance_px, score_tolerance, problems):
+    """The whole service path against `YOLO.predict`, resize kernel held constant."""
+    import numpy as np
+    import onnxruntime as ort
+    from PIL import Image
+    from ultralytics import YOLO
+
+    compared = decisions = frames_seen = 0
+    worst_box = worst_score = 0.0
+    skipped: "list[str]" = []
+    unexercised: "list[str]" = []
+    for model_path in models:
+        opts = ort.SessionOptions()
+        opts.log_severity_level = 3
+        probe = ort.InferenceSession(
+            str(model_path), sess_options=opts, providers=["CPUExecutionProvider"]
+        )
+        layout = _graph_layout(probe)
+        if layout not in YOLO_DECODERS:
+            skipped.append(f"{model_path.name} ({layout})")
+            continue
+        model_dets = 0
+        _, _, height, width = probe.get_inputs()[0].shape
+        del probe
+
+        vocabulary = _vocabulary(model_path, layout)
+        detector = OnnxDetector(model_path, vocabulary)
+        slot_of = {name: slot for slot, name in vocabulary.slot_names.items()}
+        reference = YOLO(str(model_path), task="detect")
+
+        for frame in frames:
+            # Letterbox ONCE, to uint8, and feed both arms those bytes: the
+            # reference's own LetterBox is then 640 -> 640, i.e. the identity.
+            with Image.open(frame) as im:
+                rgb = im.convert("RGB")
+                src_w, src_h = rgb.size
+                gain = min(width / src_w, height / src_h)
+                new_w, new_h = int(round(src_w * gain)), int(round(src_h * gain))
+                canvas = Image.new("RGB", (width, height), LETTERBOX_FILL)
+                canvas.paste(rgb.resize((new_w, new_h), resample=Image.BILINEAR),
+                             ((width - new_w) // 2, (height - new_h) // 2))
+
+            import io as _io
+
+            # The service takes JPEG BYTES, so the rig has to hand it bytes — and
+            # then hand the REFERENCE the pixels that came back out of that same
+            # JPEG, not the canvas that went in. JPEG is lossy even at quality
+            # 100 with no chroma subsampling, and the ±1 it costs is enough to
+            # move a borderline detection: this rig reported box deltas of
+            # hundreds of pixels and count mismatches until the round trip was
+            # closed, all of it the instrument. That is the third variable in a
+            # comparison meant to hold two.
+            buf = _io.BytesIO()
+            canvas.save(buf, format="JPEG", quality=100, subsampling=0)
+            jpeg = buf.getvalue()
+            batch = detector.detect(jpeg, THRESHOLD, None)
+
+            with Image.open(_io.BytesIO(jpeg)) as decoded:
+                bgr = np.asarray(decoded.convert("RGB"), dtype=np.uint8)[..., ::-1].copy()
+            result = reference.predict(
+                bgr, conf=THRESHOLD, iou=NMS_IOU_THRESHOLD, imgsz=width,
+                rect=False, verbose=False, device="cpu",
+            )[0]
+
+            ours = (
+                np.asarray(
+                    [[d.x, d.y, d.x + d.width, d.y + d.height] for d in batch.detections],
+                    dtype=np.float32,
+                ).reshape(-1, 4),
+                np.asarray([d.confidence for d in batch.detections], dtype=np.float32),
+                np.asarray([slot_of[d.label] for d in batch.detections], dtype="int64"),
+            )
+            theirs = (
+                result.boxes.xyxy.numpy(),
+                result.boxes.conf.numpy(),
+                result.boxes.cls.numpy().astype("int64"),
+            )
+            n, d_box, d_score = _compare(
+                ours, theirs, tolerance_px, score_tolerance,
+                f"{model_path.name} {frame.name}", problems
+            )
+            compared += n
+            model_dets += n
+            frames_seen += 1
+            worst_box, worst_score = max(worst_box, d_box), max(worst_score, d_score)
+
+            for slot in vocabulary.slot_names:
+                mine = float(ours[1][ours[2] == slot].max()) if (ours[2] == slot).any() else 0.0
+                ref = float(theirs[1][theirs[2] == slot].max()) if (theirs[2] == slot).any() else 0.0
+                decisions += 1
+                if (mine >= THRESHOLD) != (ref >= THRESHOLD):
+                    problems.append(
+                        f"{model_path.name} {frame.name} slot {slot}: presence DECISION differs — "
+                        f"ours {mine:.4f}, reference {ref:.4f}"
+                    )
+        if model_dets == 0:
+            unexercised.append(model_path.name)
+    return {
+        "arm": "pipeline",
+        "models_in_directories": len(models),
+        "models_compared": len(models) - len(skipped) - len(unexercised),
+        "models_skipped_wrong_layout": skipped,
+        "models_with_no_above_threshold_detection": unexercised,
+        "frames_decoded": frames_seen,
+        "presence_decisions": decisions,
+        "detections_compared": compared,
+        "max_box_delta_px": worst_box,
+        "max_score_delta": worst_score,
+    }
+
+
+def arm_litert(manifest_path: Path, models_dirs, frames, tolerance_px, score_tolerance, problems):
+    """The same weights through both runtimes, in PIXELS, per the declaration."""
+    import numpy as np
+    import onnxruntime as ort
+    from ai_edge_litert.interpreter import Interpreter
+
+    manifest = json.loads(manifest_path.read_text())
+    litert_dir = manifest_path.parent
+    compared = twins = frames_seen = 0
+    worst_box = worst_score = 0.0
+    rows: "list[str]" = []
+    vacuous: "list[str]" = []
+
+    for entry in manifest:
+        alias = entry["alias"]
+        onnx_path = next(
+            (d / f"{alias}.onnx" for d in models_dirs if (d / f"{alias}.onnx").exists()), None
+        )
+        if onnx_path is None:
+            problems.append(f"{alias}: no ONNX twin found under {[str(d) for d in models_dirs]}")
+            continue
+        opts = ort.SessionOptions()
+        opts.log_severity_level = 3
+        session = ort.InferenceSession(
+            str(onnx_path), sess_options=opts, providers=["CPUExecutionProvider"]
+        )
+        layout = _graph_layout(session)
+        if layout not in YOLO_DECODERS:
+            continue
+        graph_in = session.get_inputs()[0]
+        _, _, height, width = graph_in.shape
+        output_name = session.get_outputs()[0].name
+
+        interpreter = Interpreter(model_path=str(litert_dir / entry["litert"]))
+        interpreter.allocate_tensors()
+        lit_in = interpreter.get_input_details()[0]
+        lit_out = interpreter.get_output_details()[0]
+        channels_first = list(lit_in["shape"])[1] == 3
+
+        # DECLARED, from the manifest row — the box-side copy of each variant's
+        # `model_platform_variants` row. Deriving either from its tensor here
+        # would make this arm a self-consistency check.
+        onnx_units = BOX_UNITS_PIXELS if entry["box_units_onnx"].startswith("pixels") else BOX_UNITS_NORMALIZED
+        lit_units = BOX_UNITS_PIXELS if entry["box_units_litert"].startswith("pixels") else BOX_UNITS_NORMALIZED
+        declared_scale = float(entry["litert_box_scale_to_pixels"])
+        onnx_scale = 1.0 if onnx_units == BOX_UNITS_PIXELS else declared_scale
+        lit_scale = 1.0 if lit_units == BOX_UNITS_PIXELS else declared_scale
+        # Refused, not guessed, exactly as `VariantContract` would.
+        VariantContract(layout, onnx_units, onnx_scale)
+        VariantContract(layout, lit_units, lit_scale)
+
+        twins += 1
+        twin_box = twin_score = 0.0
+        twin_dets = 0
+        transposes = 0
+        for frame in frames:
+            blob, geometry = preprocess_letterbox(frame.read_bytes(), width, height)
+            raw_onnx = session.run([output_name], {graph_in.name: blob})[0]
+            feed = blob if channels_first else np.transpose(blob, (0, 2, 3, 1)).copy()
+            interpreter.set_tensor(lit_in["index"], feed.astype(lit_in["dtype"]))
+            interpreter.invoke()
+            raw_lit = interpreter.get_tensor(lit_out["index"])
+            expected = 4 + len(_names(onnx_path))
+            if layout == ANCHOR_4_PLUS_NC and raw_lit.ndim == 3 and raw_lit.shape[1] != expected \
+                    and raw_lit.shape[2] == expected:
+                # COUNTED, because a transpose nobody can see is a second way to
+                # make two runtimes agree for the wrong reason. Measured across
+                # all 17 committed twins on 2026-08-25: it fires ZERO times —
+                # every LiteRT export here is NCHW in and channels-first out. The
+                # branch stays because a future export need not be, and the count
+                # is printed so its first firing is visible.
+                transposes += 1
+                raw_lit = np.transpose(raw_lit, (0, 2, 1)).copy()
+
+            decode = YOLO_DECODERS[layout]
+            s_o, c_o, *box_o = decode(
+                raw_onnx, geometry, THRESHOLD,
+                box_scale_to_pixels=onnx_scale, expected_classes=expected - 4,
+            )
+            s_l, c_l, *box_l = decode(
+                raw_lit, geometry, THRESHOLD,
+                box_scale_to_pixels=lit_scale, expected_classes=expected - 4,
+            )
+            ours = (np.stack(box_o, axis=1) if s_o.shape[0] else np.zeros((0, 4)), s_o, c_o)
+            theirs = (np.stack(box_l, axis=1) if s_l.shape[0] else np.zeros((0, 4)), s_l, c_l)
+            n, d_box, d_score = _compare(
+                ours, theirs, tolerance_px, score_tolerance, f"{alias} {frame.name}", problems
+            )
+            compared += n
+            twin_dets += n
+            frames_seen += 1
+            twin_box, twin_score = max(twin_box, d_box), max(twin_score, d_score)
+        worst_box, worst_score = max(worst_box, twin_box), max(worst_score, twin_score)
+        if twin_dets == 0:
+            # A ROW THAT COMPARED NOTHING IS NOT AGREEMENT. Every arm here scores
+            # only ABOVE-THRESHOLD detections, so a model that fires on none of
+            # these frames produces a 0.00000 px row that reads exactly like a
+            # perfect one. Measured: `forklift_akdve_11n` fires on none of a
+            # 16-frame retail set, and `queue_supermarket` fires on 34 of them
+            # while firing on NONE of a bus-interior set — so the corpus, not the
+            # twin, decides which rows this arm can speak about.
+            vacuous.append(alias)
+        rows.append(
+            f"  {alias:28s} onnx={onnx_units:10s} litert={lit_units:10s} "
+            f"scale={declared_scale:g}  dets={twin_dets:4d}  max|dbox|={twin_box:.5f}px  "
+            f"max|dscore|={twin_score:.3g}"
+            + ("   <-- NO above-threshold detections: this row is VACUOUS" if twin_dets == 0 else "")
+            + (f"   ({transposes} litert transposes)" if transposes else "")
+        )
+    for row in rows:
+        print(row)
+    return {
+        "arm": "litert",
+        "twins": twins,
+        "twins_with_evidence": twins - len(vacuous),
+        # NAMED, not counted. These twins were not checked by this run and saying
+        # so is the whole point; a summary that reported 17/17 agreeing would be
+        # a claim about models that produced nothing to compare.
+        "twins_with_no_above_threshold_detection": vacuous,
+        "frames_decoded": frames_seen,
+        "detections_compared": compared,
+        "max_box_delta_px": worst_box,
+        "max_score_delta": worst_score,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--arm", choices=("decode", "pipeline", "litert"), default="decode")
+    parser.add_argument("--models-dir", action="append", default=[], required=True)
+    parser.add_argument("--frames-dir", required=True)
+    parser.add_argument("--limit", type=int, default=8, help="frames per model")
+    parser.add_argument("--models", type=int, default=0, help="0 = every model in the directories")
+    parser.add_argument(
+        "--tolerance-px", type=float, default=0.0,
+        help="0.0 for the decode arm: same tensor in, the arithmetic must agree exactly.",
+    )
+    parser.add_argument(
+        "--score-tolerance", type=float, default=DEFAULT_SCORE_TOLERANCE,
+        help="ENFORCED, not merely reported. A doubled sigmoid moves every confidence "
+             "without moving a box, so a box-only tolerance passes it.",
+    )
+    parser.add_argument("--litert-manifest", default=None)
+    args = parser.parse_args()
+
+    dirs = [Path(d) for d in args.models_dir]
+    models = sorted(p for d in dirs for p in d.glob("*.onnx"))
+    if args.models:
+        step = max(1, len(models) // args.models)
+        models = models[::step][: args.models]
+    frames = sorted(Path(args.frames_dir).glob("*.jpg"))[: args.limit]
+    if not models:
+        raise SystemExit(f"no .onnx under {args.models_dir}")
+    if not frames:
+        raise SystemExit(f"no frames in {args.frames_dir}")
+
+    problems: "list[str]" = []
+    if args.arm == "decode":
+        summary = arm_decode(models, frames, args.tolerance_px, args.score_tolerance, problems)
+    elif args.arm == "pipeline":
+        summary = arm_pipeline(models, frames, args.tolerance_px, args.score_tolerance, problems)
+    else:
+        if not args.litert_manifest:
+            raise SystemExit("--arm litert needs --litert-manifest naming each twin's DECLARED units")
+        summary = arm_litert(
+            Path(args.litert_manifest), dirs, frames, args.tolerance_px, args.score_tolerance, problems
+        )
+
+    print(json.dumps(summary, indent=2))
+    if problems:
+        print(f"DISAGREEMENT — {len(problems)} problem(s); the decode is not the reference instrument:")
+        for problem in problems[:40]:
+            print(f"  - {problem}")
+        if len(problems) > 40:
+            print(f"  ... and {len(problems) - 40} more")
+        return 1
+    print(
+        f"AGREE: every detection matches within {args.tolerance_px} px and "
+        f"{args.score_tolerance} confidence."
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())