@trillboards/edge-sdk 0.2.85 → 0.2.91

package/deploy/detector/bench/verify_yolo_decode.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""verify_yolo_decode — is `run_candidate.py`'s YOLO decode the same instrument
+Ultralytics is?
+
+This is `verify_against_reference.py` one level up. That file checks the RF-DETR
+ONNX export against the `rfdetr` package; this one checks the harness's OWN
+decode of an Ultralytics (1, 4+C, 8400) head against `ultralytics` itself, on the
+same pixels. It exists because §5 rule 5 of `cctv-detector-tier.md` is that every
+wrong number in that day's work came from the instrument rather than the
+reasoning, and the YOLO decode is the newest, least-exercised code in the scoring
+path. A decode that transposes an axis, or applies a sigmoid twice, or reads the
+box channels as class scores, produces a complete and entirely plausible table.
+
+WHAT IS COMPARED, AND WHY NOT BOXES. The harness scores PRESENCE — the maximum
+confidence per class per frame — because there is no box-level ground truth
+committed anywhere in this repo. So the agreement that matters is on that
+quantity, not on box coordinates the harness never reads. Comparing something the
+scorer does not use would prove the export is fine and leave the actual scoring
+path unchecked.
+
+THE REFERENCE IS THE SAME ONNX FILE, LOADED BY ULTRALYTICS — not the `.pt` it
+was exported from, and that choice cost a wrong run to find. Ultralytics'
+predictor letterboxes with `auto = same_shapes and args.rect and format == "pt"`,
+so a `.pt` reference on our 640x480 frames pads to **640x480** while the ONNX
+graph has a fixed **640x640** input. The two arms then disagree by up to 0.25
+confidence in BOTH directions and it looks like a decode bug. It is §5 rule 1
+again: an arm at one imgsz compared against an arm at another is not a
+comparison. Loading the ONNX makes `format != "pt"`, so `auto=False` and both
+sides letterbox to the graph's own input size; `rect=False` is passed as well so
+the geometry does not depend on that default. Pass `--reference <best.pt>`
+explicitly to compare against the checkpoint, and expect it to disagree for this
+reason rather than for a real one.
+
+That leaves exactly one variable — our preprocessing and decode against theirs,
+on identical bytes and identical geometry — which is what this file is for.
+
+TOLERANCE. Pillow BILINEAR and Ultralytics' cv2 INTER_LINEAR agree closely and
+not bit-for-bit, so a small confidence tolerance is expected and a large one is a
+decode bug. The DECISION tolerance is zero: at the threshold, both sides must
+agree on whether the class fires, on every frame.
+
+    python3 bench/verify_yolo_decode.py --onnx best.onnx \
+      --frames-dir /tmp/eval134 --limit 20 [--threshold 0.25] [--conf-tol 0.05]
+
+Needs `ultralytics` — a documented manual run, never CI.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))  # noqa: E402
+
+from run_candidate import HarnessYoloRunner, Frame, introspect_in_subprocess  # noqa: E402
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--onnx", required=True)
+    parser.add_argument(
+        "--reference", default=None,
+        help="what ultralytics should load as the reference. Defaults to --onnx, which is the "
+             "only setting that holds input geometry constant; see the file header.",
+    )
+    parser.add_argument("--frames-dir", required=True)
+    parser.add_argument("--limit", type=int, default=20)
+    parser.add_argument("--threshold", type=float, default=0.25)
+    parser.add_argument("--conf-tol", type=float, default=0.05)
+    args = parser.parse_args()
+
+    from ultralytics import YOLO  # noqa: PLC0415 - heavy, and only this file needs it
+
+    introspection = introspect_in_subprocess(Path(args.onnx))
+    names = {int(k): v for k, v in introspection["names"].items()}
+    ours = HarnessYoloRunner(Path(args.onnx), names)
+    reference = YOLO(args.reference or args.onnx)
+
+    frame_paths = sorted(Path(args.frames_dir).glob("*.jpg"))[: args.limit]
+    if not frame_paths:
+        raise SystemExit(f"no frames in {args.frames_dir}")
+
+    problems: "list[str]" = []
+    decisions = 0
+    for path in frame_paths:
+        frame = Frame(path)
+        # `HarnessYoloRunner.run` returns (per_class, infer_ms, box_max); the
+        # third arrived with the box-units verdict and this call still unpacked
+        # two, so this file could not execute at all.
+        mine, _, _ = ours.run(frame, 0.0)
+        # imgsz is pinned to the graph's own input size and verbose is off; a
+        # default imgsz here would compare an arm at 640 against one at 1280,
+        # which §5 rule 1 says is not a comparison.
+        result = reference.predict(
+            str(path), imgsz=ours.width, conf=0.001, rect=False, verbose=False, device="cpu"
+        )[0]
+        theirs: "dict[str, float]" = {}
+        for cls, conf in zip(result.boxes.cls.tolist(), result.boxes.conf.tolist()):
+            name = names[int(cls)]
+            theirs[name] = max(theirs.get(name, 0.0), float(conf))
+
+        for name in names.values():
+            a = mine.get(name, [0.0])[0] if name in mine else 0.0
+            b = theirs.get(name, 0.0)
+            decisions += 1
+            if (a >= args.threshold) != (b >= args.threshold):
+                problems.append(
+                    f"{path.name} {name}: DECISION differs at {args.threshold} — ours {a:.4f}, reference {b:.4f}"
+                )
+            elif abs(a - b) > args.conf_tol and max(a, b) >= args.threshold:
+                problems.append(
+                    f"{path.name} {name}: confidence {a:.4f} vs {b:.4f} (tol {args.conf_tol})"
+                )
+
+    print(
+        f"reference {Path(args.reference or args.onnx).name}  frames {len(frame_paths)}  "
+        f"classes {len(names)}  imgsz {ours.width}x{ours.height}  presence decisions compared {decisions}"
+    )
+    if problems:
+        print("DISAGREEMENT — the decode in run_candidate.py is not the reference instrument:")
+        for p in problems[:40]:
+            print(f"  - {p}")
+        if len(problems) > 40:
+            print(f"  ... and {len(problems) - 40} more")
+        return 1
+    print(
+        f"AGREE: every presence decision matches at threshold {args.threshold}, and every "
+        f"firing confidence is within {args.conf_tol}."
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())