@trillboards/edge-sdk 0.2.84 → 0.2.89

package/deploy/detector/bench/run_candidate.py

@@ -0,0 +1,933 @@
+#!/usr/bin/env python3
+"""run_candidate — RUN a candidate detector over a labelled frame set and record
+what only the running process can know: per-frame per-class confidences,
+forward-pass latency, resident footprint after a real probe, and whether the
+SHIPPED service would load this graph at all.
+
+It does not score anything. Scoring lives in
+`trillboard-api/scripts/sensing/score-detector-candidate.js`, and the split is
+deliberate — it is the same split `scripts/vision/detector-arm-eval.mjs` already
+makes ("Producing --detections is model-specific and deliberately out of scope"),
+and it is what let one eval set score five model families without a rewrite.
+What is NEW here is that four of the things a publish decision needs cannot be
+recovered from a detections file after the fact:
+
+  * latency p50/p95 — a number without its box is not transferable, and the box
+    is only knowable where the forward pass happened.
+  * resident footprint — RSS measured AFTER a probe inference. A session that has
+    not run a forward pass has not allocated its arenas; measuring before the
+    probe reports the loader, not the model.
+  * the artifact's sha256 — the publish rule is "the artifact, its checksum and
+    its score land together", so the checksum has to be of the bytes that were
+    ACTUALLY loaded, not of a path someone typed into a migration later.
+  * shipped-runtime loadability — see below. This is the yolov8_nano check made
+    mechanical.
+
+THE LOADABILITY CHECK IS WHY THIS IS A FILE AND NOT A NOTEBOOK.
+`trillboards_detector.session._read_graph_spec` refuses any graph whose outputs
+are not named `dets`/`labels`, so an Ultralytics export — `output0`, shaped
+(1, 4+C, 8400) — cannot be loaded by the service that would have to run it. A
+candidate can therefore score perfectly and still be unshippable, and that is
+EXACTLY the yolov8_nano failure: advertised to ~97 of 114 profiles, loadable
+nowhere, `vehicle_count` ~0% fleet-wide for six months. So the check runs against
+the real shipped contract (imported, never transcribed — a copy drifts) and its
+refusal string is recorded in the run file, where the scorer turns it into a
+publish verdict.
+
+TWO EXECUTION PATHS, and which one ran is recorded.
+
+  shipped   The graph passes `_read_graph_spec`, so inference goes through
+            `OnnxDetector` — the same class the connector runs. This is the only
+            path whose numbers describe production.
+  harness   The graph does not pass, so this file decodes it itself. The numbers
+            describe the MODEL, not the product, and `shipped_service_loadable`
+            is false in the output so nothing downstream can forget that.
+
+PRESENCE ONLY, AND THAT IS A LIMIT OF THE LABELS, NOT OF THE RIG. There is no
+box-level ground truth committed anywhere in this repo (grep every eval set for
+`bbox|xmin|x1|polygon`: zero), so localisation is unscorable and this file does
+not pretend otherwise — it emits the max confidence per class per frame and no
+boxes. Count is unscorable for a second, independent reason: the only count
+column in the alert-evidence eval set is `model_person_count`, which is the
+shipped VLM's own output and rides there as the CONTROL ARM. Scoring a candidate
+against it would be scoring a model against a model. Because only per-class
+maxima are needed, NMS is not run and cannot bias the result: suppression moves
+which box carries a score, never the maximum over boxes.
+
+Usage:
+
+    python3 bench/run_candidate.py \
+      --onnx /path/to/model.onnx \
+      --candidate ppe_construction \
+      --manifest trillboard-api/scripts/ai/eval-baselines/alert-evidence-frame-eval-set-2026-08-21.json \
+      --frames-dir /tmp/eval134 \
+      --out /tmp/run-ppe-construction.json
+
+Requires this package's own pinned runtime (onnxruntime 1.29.0 / numpy / pillow).
+Run it from the detector's uv tool env, not the system interpreter.
+"""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import hashlib
+import json
+import os
+import platform
+import statistics
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))  # noqa: E402
+
+from trillboards_detector.errors import ModelContractError  # noqa: E402
+from trillboards_detector.labels import (  # noqa: E402
+    BUILTIN_VOCABULARIES,
+    ModelVocabulary,
+    normalize_class_name,
+)
+from trillboards_detector.registry import read_process_rss  # noqa: E402
+from trillboards_detector.session import OnnxDetector, _read_graph_spec  # noqa: E402
+
+#: Emitted into the run file so the scorer can refuse a shape it does not know.
+SCHEMA = "detector-candidate-run/2"
+#: v2 added the `variant` block — format, output layout, and MEASURED box units.
+#: The version was bumped rather than the field quietly added because a v1 run
+#: carries no coordinate space at all, and a scorer that treated its absence as
+#: "fine" would publish exactly the variant this field exists to catch.
+
+#: HOW A BOX-UNIT VERDICT IS REACHED, and why a threshold rather than a flag.
+#:
+#: LiteRT and ONNX emit box coordinates in DIFFERENT UNITS FOR THE SAME WEIGHTS.
+#: Ultralytics' TFLite export path wraps a `task=detect`, `end2end=False` model in
+#: `_NormalizeCoords`, dividing the four box channels by (W, H). Measured here on
+#: all 17 committed LiteRT twins: 16 of 17 have ONNX boxes in pixels 0..640 and
+#: LiteRT boxes in 0..1, agreeing to 0.00266 px after multiplying by 640, while
+#: their CLASS channels agree to 1.2e-07. `queue_supermarket`, the one end2end
+#: twin, agrees on both sides — end2end heads emit pixels everywhere.
+#:
+#: Every check a publish gate would otherwise run PASSES on that pair: same
+#: output shape, same dtype, loads, runs, class scores identical. An Android
+#: post-processor reusing the ONNX decode draws every box 640x too small and
+#: nothing reports an error. So the units are MEASURED from a forward pass, per
+#: artifact, and recorded — never carried across from the sibling format.
+#:
+#: The discriminator is a ratio, max|box channel| / imgsz, and it is decisive by
+#: two and a half orders of magnitude. Measured over 119 ONNX artifacts the ratio
+#: spans 0.988..1.167; a normalised head sits at ~1/640 = 0.0016. The dead zone
+#: below is EMPTY across every artifact in the tree, and an artifact landing in it
+#: is refused rather than guessed at — that is the whole point of the field.
+BOX_UNITS_PIXEL_RATIO_MIN = 0.2
+BOX_UNITS_NORMALIZED_RATIO_MAX = 0.02
+
+#: Warm-up forward passes before any timing is kept. ORT's first run pays for
+#: arena allocation and kernel selection; folding that into a p50 over 134 frames
+#: moves the median a little and the p95 a lot.
+DEFAULT_WARMUP = 3
+
+
+def read_load_average() -> "dict | None":
+    """The machine's run-queue depth, because a ms figure without it is not one.
+
+    This repo's rule is that every latency number carries its hardware. That is
+    necessary and it is NOT sufficient on a shared box: measured here, two
+    back-to-back runs of the same artifact on the same machine gave p50 68.3 ms
+    and 276.2 ms — 4x — at load average 9.58 on 16 cores, with nothing about the
+    model, the frames or the code changing between them. A reader given only
+    "68 ms, M4 Max" cannot tell which of those two runs they are holding.
+
+    So the run queue is recorded beside the timing and `contention_ratio` is
+    load1 / cores. Above ~0.7 the p95 is measuring other people's work.
+    """
+    try:
+        one, five, fifteen = os.getloadavg()
+    except (OSError, AttributeError):  # pragma: no cover - Windows has no getloadavg
+        return None
+    cores = os.cpu_count() or 1
+    return {
+        "load_1m": round(one, 2), "load_5m": round(five, 2), "load_15m": round(fifteen, 2),
+        "cores": cores, "contention_ratio": round(one / cores, 3),
+    }
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def describe_box() -> str:
+    """Name the machine, because every latency figure in our runbooks carries one.
+
+    `cctv-detector-tier.md` §6 is explicit that the ACCURACY findings are
+    hardware-independent and the latency ones are not, and the whole "INT8 is
+    slower than fp32" result is an Apple-arm64 fact nobody has reproduced on the
+    AMD Zen4 box carrying live customers. A ms figure whose box is unstated reads
+    as if it transfers.
+    """
+    bits = [platform.system(), platform.machine()]
+    cores = os.cpu_count()
+    if cores:
+        bits.append(f"{cores} cores")
+    if platform.system() == "Darwin":
+        try:
+            brand = subprocess.run(
+                ["sysctl", "-n", "machdep.cpu.brand_string"],
+                capture_output=True, text=True, timeout=5, check=False,
+            ).stdout.strip()
+            if brand:
+                bits.insert(1, brand)
+        except (OSError, subprocess.SubprocessError):  # pragma: no cover
+            pass
+    return " ".join(b for b in bits if b)
+
+
+def _parse_names_literal(raw, model_path: Path) -> "dict[int, str]":
+    """§5 rule 2 of `cctv-detector-tier.md`: NEVER guess a class index.
+
+    A fallback to index 0 once scored BUSES for a whole run and the table looked
+    entirely plausible. Ultralytics writes `names` into the ONNX metadata as a
+    Python dict literal; if it is absent there is no way to name a slot, and the
+    honest move is to stop rather than to count from zero.
+    """
+    if not raw:
+        raise ModelContractError(
+            f"{model_path.name} carries no `names` in its ONNX metadata. Refusing to score it: "
+            "a class index this file cannot name is a class it would have to guess, and a "
+            "guessed index once scored buses for a whole run against person labels."
+        )
+    parsed = ast.literal_eval(raw)
+    if not isinstance(parsed, dict) or not parsed:
+        raise ModelContractError(f"{model_path.name}: `names` metadata is not a non-empty dict")
+    return {int(k): str(v) for k, v in parsed.items()}
+
+
+def read_declared_class_names(model_path: Path) -> "dict[int, str]":
+    """Class names the artifact declares ABOUT ITSELF. Both formats are self-describing.
+
+    ONNX carries them in `custom_metadata_map['names']` as a Python dict literal.
+    A LiteRT `.tflite` from Ultralytics is a ZIP holding `metadata.json`, whose
+    `names` is a real JSON object — a raw byte search of the file finds nothing
+    because the archive is compressed, which is a good way to conclude the format
+    is not self-describing and reach for a cross-format import that should never
+    exist. Names must come from the artifact: four models were caught tonight
+    lying by class list alone, including a "gun and knife" detector that is stock
+    YOLOv8n with all 80 COCO classes.
+    """
+    if model_path.suffix.lower() == ".tflite":
+        import zipfile  # noqa: PLC0415
+
+        if not zipfile.is_zipfile(model_path):
+            raise ModelContractError(
+                f"{model_path.name} carries no embedded metadata archive, so it cannot name its "
+                "own classes and this file will not name them for it"
+            )
+        with zipfile.ZipFile(model_path) as archive:
+            if "metadata.json" not in archive.namelist():
+                raise ModelContractError(f"{model_path.name}: no metadata.json in the artifact")
+            meta = json.loads(archive.read("metadata.json").decode("utf-8"))
+        names = meta.get("names")
+        if not isinstance(names, dict) or not names:
+            raise ModelContractError(f"{model_path.name}: metadata.json declares no `names`")
+        return {int(k): str(v) for k, v in names.items()}
+
+    import onnxruntime as ort  # noqa: PLC0415
+
+    opts = ort.SessionOptions()
+    opts.log_severity_level = 3
+    session = ort.InferenceSession(
+        str(model_path), sess_options=opts, providers=["CPUExecutionProvider"]
+    )
+    meta = session.get_modelmeta().custom_metadata_map or {}
+    return _parse_names_literal(meta.get("names"), model_path)
+
+
+def classify_output_layout(output_shapes: "list[list]") -> str:
+    """Which decode does this graph need?
+
+    A publish gate that records only "onnx, 12,245,970 bytes" tells a consumer
+    nothing about how to read the tensor, and two incompatible layouts are in the
+    tree right now: 112 of 119 artifacts are the anchor head `[1, 4+nc, A]`, and
+    **7 are `[1, N, 6]`** — NMS-free end2end rows of `x1,y1,x2,y2,score,cls`. A
+    parser that assumes one reads the other as garbage with no error, so the
+    layout is a declared property of the variant, derived from the graph.
+    """
+    if len(output_shapes) == 2:
+        return "rfdetr_dets_labels"
+    if len(output_shapes) != 1:
+        return f"unknown_{len(output_shapes)}_outputs"
+    shape = output_shapes[0]
+    if len(shape) == 3 and shape[2] == 6:
+        return "end2end_n_6"
+    if len(shape) == 3:
+        return "anchor_4_plus_nc"
+    return f"unknown_rank_{len(shape)}"
+
+
+def classify_box_units(box_abs_max: float, imgsz: int) -> dict:
+    """pixels, normalized, or REFUSE — never a guess.
+
+    Returns the verdict plus the evidence, because a bare enum in a score card is
+    a claim and the ratio is what makes it checkable.
+    """
+    if imgsz <= 0 or box_abs_max is None:
+        return {"units": None, "ratio": None, "scale_to_pixels": None,
+                "refusal": "no box channels were observed, so the coordinate space is unmeasured"}
+    ratio = box_abs_max / imgsz
+    if ratio >= BOX_UNITS_PIXEL_RATIO_MIN:
+        return {"units": "pixels", "ratio": ratio, "scale_to_pixels": 1.0, "refusal": None}
+    if ratio <= BOX_UNITS_NORMALIZED_RATIO_MAX:
+        return {"units": "normalized", "ratio": ratio, "scale_to_pixels": float(imgsz), "refusal": None}
+    return {
+        "units": None, "ratio": ratio, "scale_to_pixels": None,
+        "refusal": (
+            f"max|box|/imgsz = {ratio:.4f} falls between {BOX_UNITS_NORMALIZED_RATIO_MAX} and "
+            f"{BOX_UNITS_PIXEL_RATIO_MIN}, where neither pixels nor normalised is credible. "
+            "Measured over 119 artifacts this band is empty; refusing rather than picking one, "
+            "because the wrong pick is silent and scales every box by 640."
+        ),
+    }
+
+
+def letterbox(image, width: int, height: int):
+    """Ultralytics' resize: scale to fit, pad the remainder 114-grey, centred.
+
+    The shipped RF-DETR path STRETCHES (`session.preprocess`, transcribed from
+    `rfdetr/detr.py`). Ultralytics does not, and using the wrong one for a YOLO
+    export is not a rounding difference — it changes the aspect of every object in
+    the frame. The two preprocessors are kept apart and each is named in the run
+    file, so an arm can never be compared against one preprocessed differently.
+    """
+    import numpy as np  # noqa: PLC0415
+    from PIL import Image  # noqa: PLC0415
+
+    src_w, src_h = image.size
+    scale = min(width / src_w, height / src_h)
+    new_w, new_h = int(round(src_w * scale)), int(round(src_h * scale))
+    resized = image.resize((new_w, new_h), resample=Image.BILINEAR)
+    canvas = Image.new("RGB", (width, height), (114, 114, 114))
+    canvas.paste(resized, ((width - new_w) // 2, (height - new_h) // 2))
+    arr = np.asarray(canvas, dtype=np.float32) / np.float32(255.0)
+    return arr.transpose(2, 0, 1)[None, ...]
+
+
+def _decode_yolo(raw, layout: str, slot_names: "dict[int, str]", conf_floor: float):
+    """Per-class maxima and the largest box magnitude, for either YOLO layout.
+
+    Presence only: the maximum confidence per class. NMS is not run and cannot
+    change the answer — suppression moves which box carries a score, never the
+    maximum over boxes — and running one would add a setting nobody could
+    reconstruct from the run file later.
+    """
+    per_class: "dict[str, list[float]]" = {}
+    if layout == "end2end_n_6":
+        rows = raw[0]                       # (N, 6) = x1,y1,x2,y2,score,cls
+        box_max = float(abs(rows[:, :4]).max()) if rows.size else 0.0
+        for x1, y1, x2, y2, score, cls in rows:
+            name = slot_names.get(int(cls))
+            if name is None or float(score) < conf_floor:
+                continue
+            per_class.setdefault(name, []).append(float(score))
+        for values in per_class.values():
+            values.sort(reverse=True)
+        return per_class, box_max
+    grid = raw[0]                           # (4 + nc, anchors), scores already sigmoid
+    box_max = float(abs(grid[:4, :]).max()) if grid.size else 0.0
+    maxima = grid[4:, :].max(axis=1)
+    for idx, name in slot_names.items():
+        if idx >= maxima.shape[0]:
+            continue
+        value = float(maxima[idx])
+        if value >= conf_floor:
+            per_class[name] = [value]
+    return per_class, box_max
+
+
+class HarnessYoloRunner:
+    """Decode an Ultralytics head, for graphs the shipped service refuses.
+
+    Handles BOTH committed layouts — the anchor head `[1, 4+nc, A]` (112 of 119
+    artifacts) and the NMS-free `[1, N, 6]` end2end head (7 of them) — because a
+    parser that assumes one reads the other as garbage without erroring.
+    """
+
+    path_kind = "harness-yolo"
+    preprocessing = "ultralytics-letterbox-114-centred"
+    variant_format = "onnx"
+
+    def __init__(self, model_path: Path, slot_names: "dict[int, str]"):
+        # Names are handed IN, never re-read here: reading them needs a session,
+        # and a second session in the measuring process is exactly what
+        # `introspect_in_subprocess` exists to prevent.
+        import numpy as np  # noqa: PLC0415
+        import onnxruntime as ort  # noqa: PLC0415
+
+        self._np = np
+        opts = ort.SessionOptions()
+        opts.log_severity_level = 3
+        self.session = ort.InferenceSession(
+            str(model_path), sess_options=opts, providers=["CPUExecutionProvider"]
+        )
+        graph_in = self.session.get_inputs()[0]
+        shape = graph_in.shape
+        if len(shape) != 4 or shape[1] != 3:
+            raise ModelContractError(f"expected an NCHW 3-channel input, found {shape}")
+        if not isinstance(shape[2], int) or not isinstance(shape[3], int):
+            raise ModelContractError(
+                f"expected a fixed input resolution, found {shape[2]}x{shape[3]}; imgsz is one "
+                "of the two numbers a detector arm is meaningless without"
+            )
+        self.input_name = graph_in.name
+        self.height, self.width = int(shape[2]), int(shape[3])
+        outputs = self.session.get_outputs()
+        self.output_shapes = [list(o.shape) for o in outputs]
+        self.layout = classify_output_layout(self.output_shapes)
+        if self.layout.startswith("unknown"):
+            raise ModelContractError(
+                f"cannot classify this graph's output layout ({self.output_shapes}); refusing "
+                "rather than decoding a tensor by guessing what its axes mean"
+            )
+        self.output_name = outputs[0].name
+        self.slot_names = dict(slot_names)
+        if self.layout == "anchor_4_plus_nc":
+            expected = 4 + len(self.slot_names)
+            declared = self.output_shapes[0][1] if len(self.output_shapes[0]) == 3 else None
+            if isinstance(declared, int) and declared != expected:
+                raise ModelContractError(
+                    f"output axis is {declared} wide; 4 box channels + {len(self.slot_names)} named "
+                    f"classes needs {expected}. Refusing rather than labelling a channel by position."
+                )
+        self.providers = list(self.session.get_providers())
+
+    def class_names(self) -> "dict[int, str]":
+        return dict(self.slot_names)
+
+    def run(self, frame, conf_floor: float):
+        blob = letterbox(frame, self.width, self.height)
+        started = time.perf_counter()
+        (raw,) = self.session.run([self.output_name], {self.input_name: blob})
+        infer_ms = (time.perf_counter() - started) * 1000.0
+        per_class, box_max = _decode_yolo(raw, self.layout, self.slot_names, conf_floor)
+        return per_class, infer_ms, box_max
+
+
+class LitertRunner:
+    """The SAME weights through LiteRT — a DIFFERENT artifact, scored separately.
+
+    This class exists because the two formats are not interchangeable and the
+    difference is invisible to every other check. Scoring the ONNX and copying the
+    verdict onto the `.tflite` row is precisely the mistake the box-units field is
+    for: the class channels agree to 1.2e-07 while the boxes are 640x apart.
+
+    `ai_edge_litert` is imported lazily and is deliberately NOT in the detector
+    package's pinned dependencies — `bench/` is excluded from the wheel
+    (`[tool.hatch.build.targets.wheel] packages = ["trillboards_detector"]`), so a
+    measurement-only dependency must never reach the connector's runtime.
+    """
+
+    path_kind = "harness-litert"
+    preprocessing = "ultralytics-letterbox-114-centred"
+    variant_format = "litert"
+
+    def __init__(self, model_path: Path, slot_names: "dict[int, str]"):
+        import numpy as np  # noqa: PLC0415
+
+        try:
+            from ai_edge_litert.interpreter import Interpreter  # noqa: PLC0415
+        except ImportError as exc:  # pragma: no cover - environment, not logic
+            raise SystemExit(
+                "scoring a .tflite needs `ai_edge_litert` (uv pip install ai-edge-litert). It is "
+                "intentionally absent from the detector package's own dependencies: bench/ is not "
+                "shipped in the wheel and a measurement dependency must not reach the connector."
+            ) from exc
+
+        self._np = np
+        self.interpreter = Interpreter(model_path=str(model_path))
+        self.interpreter.allocate_tensors()
+        self._in = self.interpreter.get_input_details()[0]
+        outs = self.interpreter.get_output_details()
+        self.output_shapes = [list(o["shape"]) for o in outs]
+        self._out = outs[0]
+        shape = list(self._in["shape"])
+        if len(shape) != 4:
+            raise ModelContractError(f"expected a rank-4 input, found {shape}")
+        # LiteRT exports may be NCHW or NHWC and the difference is silent, so the
+        # layout is read off the graph rather than assumed.
+        if shape[1] == 3:
+            self.channels_first = True
+            self.height, self.width = int(shape[2]), int(shape[3])
+        elif shape[3] == 3:
+            self.channels_first = False
+            self.height, self.width = int(shape[1]), int(shape[2])
+        else:
+            raise ModelContractError(f"cannot find a 3-channel axis in input shape {shape}")
+        self.layout = classify_output_layout(self.output_shapes)
+        self.slot_names = dict(slot_names)
+        self.providers = ["LiteRT-XNNPACK-CPU"]
+
+    def class_names(self) -> "dict[int, str]":
+        return dict(self.slot_names)
+
+    def run(self, frame, conf_floor: float):
+        np = self._np
+        blob = letterbox(frame, self.width, self.height)
+        if not self.channels_first:
+            blob = np.transpose(blob, (0, 2, 3, 1)).copy()
+        started = time.perf_counter()
+        self.interpreter.set_tensor(self._in["index"], blob)
+        self.interpreter.invoke()
+        raw = self.interpreter.get_tensor(self._out["index"])
+        infer_ms = (time.perf_counter() - started) * 1000.0
+        # A LiteRT export can transpose the anchor head relative to ONNX; orient by
+        # the named class count rather than by position.
+        if self.layout == "anchor_4_plus_nc" and raw.ndim == 3:
+            expected = 4 + len(self.slot_names)
+            if raw.shape[1] != expected and raw.shape[2] == expected:
+                raw = np.transpose(raw, (0, 2, 1)).copy()
+        per_class, box_max = _decode_yolo(raw, self.layout, self.slot_names, conf_floor)
+        return per_class, infer_ms, box_max
+
+
+class ShippedRunner:
+    """Inference through `OnnxDetector` — the class the connector actually runs."""
+
+    path_kind = "shipped"
+    preprocessing = "session.preprocess (imagenet-normalised stretch)"
+    variant_format = "onnx"
+    layout = "rfdetr_dets_labels"
+
+    def __init__(self, model_path: Path, vocabulary: ModelVocabulary):
+        self.detector = OnnxDetector(model_path, vocabulary)
+        self.vocabulary = vocabulary
+        self.width = self.detector.spec.width
+        self.height = self.detector.spec.height
+        self.providers = list(self.detector.spec.providers)
+
+    def class_names(self) -> "dict[int, str]":
+        return dict(self.vocabulary.slot_names)
+
+    def run(self, frame, conf_floor: float):
+        batch = self.detector.detect(frame.raw_bytes, conf_floor, None)
+        per_class: "dict[str, list[float]]" = {}
+        box_max = 0.0
+        for det in batch.detections:
+            per_class.setdefault(det.label, []).append(float(det.confidence))
+            # `postprocess` scales by frame_w/frame_h, so the wire carries PIXELS
+            # of the submitted frame. Measured from the emitted boxes rather than
+            # asserted from that fact, so the record is evidence either way.
+            box_max = max(box_max, float(det.x + det.width), float(det.y + det.height))
+        for values in per_class.values():
+            values.sort(reverse=True)
+        return per_class, float(batch.inference_ms), box_max
+
+
+class Frame:
+    """A JPEG on disk, decoded LAZILY and releasable.
+
+    THE LAZINESS IS THE MEASUREMENT, not an optimisation. RSS is sampled between
+    the probe inference and the scored pass, so anything resident at that moment
+    is counted as the model's footprint. Eagerly building a list of all 134
+    decoded RGB images puts ~123 MB of pixels (640x480x3 x 134) plus their JPEG
+    bytes inside that number, and under macOS's `rusage-maxrss` high-water source
+    the contamination cannot be subtracted afterwards — the peak is already set.
+
+    So exactly ONE frame is resident when RSS is read, and the scored pass builds
+    and releases the rest one at a time. This is the third instance in this file
+    of the same failure: the harness measuring itself instead of the model (see
+    also `introspect_in_subprocess`, and the probe inference itself).
+    """
+
+    def __init__(self, path: Path):
+        self.path = path
+        self._raw: "bytes | None" = None
+        self._image = None
+
+    @property
+    def raw_bytes(self) -> bytes:
+        if self._raw is None:
+            self._raw = self.path.read_bytes()
+        return self._raw
+
+    @property
+    def image(self):
+        if self._image is None:
+            from PIL import Image  # noqa: PLC0415
+
+            with Image.open(self.path) as handle:
+                self._image = handle.convert("RGB")
+        return self._image
+
+    @property
+    def size(self):
+        return self.image.size
+
+    def resize(self, *args, **kwargs):  # Pillow duck-typing for `letterbox`
+        return self.image.resize(*args, **kwargs)
+
+    def release(self) -> None:
+        """Drop the decoded pixels and the JPEG bytes. Called after every frame."""
+        self._raw = None
+        self._image = None
+
+
+def synthesise_vocabulary(name: str, class_names: "dict[int, str]") -> ModelVocabulary:
+    """A ModelVocabulary over a model's OWN names, canonicalised the service's way.
+
+    Synthesised rather than committed so the loadability check is a FAIR one: it
+    must not fail merely because the committed vocabulary describes a different
+    model. What survives is the part that is genuinely about the graph — output
+    names, rank, and the class axis agreeing with the vocabulary handed in.
+    """
+    width = max(class_names) + 1 if class_names else 0
+    slots: "dict[int, str]" = {}
+    for idx, raw in sorted(class_names.items()):
+        canonical = normalize_class_name(raw)
+        if canonical in slots.values():
+            # Two slots folding to one canonical name is a real property of some
+            # candidates (`CellPhone` + `phone`). Keep the first rather than
+            # raising: this vocabulary exists only to make the graph check fair.
+            continue
+        slots[idx] = canonical
+    return ModelVocabulary(
+        model_name=name, slot_names=slots, logit_slots=width, approx_ram_bytes=1
+    )
+
+
+def probe_loadability(model_path: Path, class_names: "dict[int, str]") -> dict:
+    """Would the SHIPPED service load this artifact? Ask the shipped contract."""
+    import onnxruntime as ort  # noqa: PLC0415
+
+    if model_path.suffix.lower() != ".onnx":
+        # Attempted rather than asserted, so the recorded refusal is the real one.
+        try:
+            ort.InferenceSession(str(model_path), providers=["CPUExecutionProvider"])
+        except Exception as exc:  # noqa: BLE001 - any refusal is the answer
+            return {
+                "shipped_service_loadable": False,
+                "refusal": (
+                    f"the detector service builds an onnxruntime InferenceSession and this is a "
+                    f"{model_path.suffix} artifact: {str(exc).splitlines()[0][:200]}"
+                ),
+            }
+
+    opts = ort.SessionOptions()
+    opts.log_severity_level = 3
+    session = ort.InferenceSession(
+        str(model_path), sess_options=opts, providers=["CPUExecutionProvider"]
+    )
+    try:
+        vocabulary = synthesise_vocabulary("probe-only", class_names)
+    except ValueError as exc:
+        return {"shipped_service_loadable": False, "refusal": f"vocabulary: {exc}"}
+    try:
+        _read_graph_spec(session, vocabulary)
+    except ModelContractError as exc:
+        return {"shipped_service_loadable": False, "refusal": str(exc)}
+    return {"shipped_service_loadable": True, "refusal": None}
+
+
+def introspect_in_subprocess(model_path: Path) -> dict:
+    """Read class names and shipped-runtime loadability WITHOUT touching our RSS.
+
+    Both answers need an ORT session, and a session maps the weights. Building
+    one in THIS process before the baseline is taken poisons the footprint
+    measurement in a way that is invisible in the output: on macOS the RSS source
+    is `rusage-maxrss`, a process high-water mark, so a transient session sets the
+    peak and the real session then appears to cost nothing. The delta reads small
+    and plausible and describes neither session.
+
+    So the introspection runs in a child, and the parent creates exactly one
+    session in its whole life. This is the same class of error as measuring RSS
+    before the probe inference, and it is worth a fork to remove.
+    """
+    result = subprocess.run(
+        [sys.executable, str(Path(__file__).resolve()), "--introspect", "--onnx", str(model_path)],
+        capture_output=True, text=True, check=False,
+    )
+    if result.returncode != 0:
+        raise SystemExit(
+            f"introspection of {model_path.name} failed ({result.returncode}):\n{result.stderr.strip()}"
+        )
+    return json.loads(result.stdout)
+
+
+def percentile(values: "list[float]", q: float):
+    if not values:
+        return None
+    ordered = sorted(values)
+    if len(ordered) == 1:
+        return ordered[0]
+    pos = (len(ordered) - 1) * q
+    low = int(pos)
+    high = min(low + 1, len(ordered) - 1)
+    return ordered[low] + (ordered[high] - ordered[low]) * (pos - low)
+
+
+def load_manifest_frames(manifest_path: Path) -> "list[str]":
+    manifest = json.loads(manifest_path.read_text())
+    keys = [f["s3_key"] for ep in manifest.get("episodes", []) for f in ep.get("frames", [])]
+    if not keys:
+        raise SystemExit(f"{manifest_path}: no frames — nothing to run")
+    return keys
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    parser.add_argument("--onnx", required=True)
+    parser.add_argument("--candidate", help="the model_id it would publish as")
+    parser.add_argument("--manifest")
+    parser.add_argument("--frames-dir")
+    parser.add_argument("--out")
+    parser.add_argument(
+        "--introspect", action="store_true",
+        help="internal: print class names + loadability as JSON and exit. Run in a child so "
+             "the measuring process never builds a throwaway session (see "
+             "introspect_in_subprocess).",
+    )
+    parser.add_argument("--conf-floor", type=float, default=0.05)
+    parser.add_argument("--warmup", type=int, default=DEFAULT_WARMUP)
+    parser.add_argument("--device", default=None, help="override the auto-detected box name")
+    parser.add_argument("--note", default=None)
+    args = parser.parse_args()
+
+    model_path = Path(args.onnx).resolve()
+    if not model_path.is_file():
+        raise SystemExit(f"no artifact at {model_path}")
+
+    if args.introspect:
+        try:
+            names = read_declared_class_names(model_path)
+        except ModelContractError as exc:
+            names = {}
+            declared_refusal = str(exc)
+        else:
+            declared_refusal = None
+        loadability = probe_loadability(model_path, names) if names else None
+        print(json.dumps({
+            "names": {str(k): v for k, v in names.items()},
+            "declared_names_refusal": declared_refusal,
+            "loadability": loadability,
+        }))
+        return 0
+
+    for required in ("candidate", "manifest", "frames_dir", "out"):
+        if not getattr(args, required):
+            parser.error(f"--{required.replace('_', '-')} is required")
+
+    manifest_path = Path(args.manifest).resolve()
+    frames_dir = Path(args.frames_dir).resolve()
+
+    keys = load_manifest_frames(manifest_path)
+    missing = [k for k in keys if not (frames_dir / os.path.basename(k)).is_file()]
+    if missing:
+        raise SystemExit(
+            f"REFUSING: {len(missing)} of {len(keys)} labelled frames are not in {frames_dir}. "
+            "A frame the detector never saw is not a negative, it is a hole, and it would "
+            "silently become a false negative."
+        )
+
+    # RSS baseline BEFORE the session exists. Everything after this is the model.
+    rss_before = read_process_rss()
+
+    introspection = introspect_in_subprocess(model_path)
+    declared_names = {int(k): v for k, v in introspection["names"].items()}
+    loadability = introspection["loadability"]
+    runner = None
+    builtin_used = None
+
+    if not declared_names:
+        # No self-declared names. The only honest source left is a COMMITTED
+        # vocabulary, and only if the graph matches it exactly.
+        for vocabulary in BUILTIN_VOCABULARIES:
+            try:
+                runner = ShippedRunner(model_path, vocabulary)
+                builtin_used = vocabulary.model_name
+                loadability = {"shipped_service_loadable": True, "refusal": None}
+                break
+            except ModelContractError as exc:
+                loadability = {"shipped_service_loadable": False, "refusal": str(exc)}
+        if runner is None:
+            raise SystemExit(
+                f"REFUSING: {model_path.name} declares no class names of its own and matches no "
+                f"committed vocabulary. Last refusal: {loadability and loadability['refusal']}"
+            )
+    elif model_path.suffix.lower() == ".tflite":
+        runner = LitertRunner(model_path, declared_names)
+    elif loadability["shipped_service_loadable"]:
+        runner = ShippedRunner(model_path, synthesise_vocabulary(args.candidate, declared_names))
+    else:
+        runner = HarnessYoloRunner(model_path, declared_names)
+
+    rss_after_session = read_process_rss()
+
+    # THE PROBE. A session that has not run a forward pass has not allocated its
+    # arenas — measuring RSS here rather than above is the difference between the
+    # loader's footprint and the model's.
+    #
+    # EXACTLY ONE FRAME IS RESIDENT WHEN RSS IS READ. Retaining the whole
+    # evaluation set here would fold ~123 MB of decoded pixels into the reported
+    # footprint, and a high-water RSS source makes that unsubtractable later.
+    warmup_n = max(0, int(args.warmup))
+    probe_frame = Frame(frames_dir / os.path.basename(keys[0]))
+    for _ in range(max(1, warmup_n)):
+        runner.run(probe_frame, args.conf_floor)
+    rss_after_probe = read_process_rss()
+    probe_frame.release()
+
+    load_before = read_load_average()
+    detections: "dict[str, dict[str, list[float]]]" = {}
+    infer_ms: "list[float]" = []
+    wall_ms: "list[float]" = []
+    box_abs_max = 0.0
+    for key in keys:
+        frame = Frame(frames_dir / os.path.basename(key))
+        started = time.perf_counter()
+        per_class, ms, box_max = runner.run(frame, args.conf_floor)
+        wall_ms.append((time.perf_counter() - started) * 1000.0)
+        infer_ms.append(ms)
+        # The coordinate space is read off REAL frames, over the whole set, not
+        # off one synthetic tensor: a single frame with no large box would drag
+        # the ratio toward the dead zone and turn a decisive fact into a refusal.
+        box_abs_max = max(box_abs_max, box_max)
+        detections[key] = per_class
+        frame.release()
+
+    load_after = read_load_average()
+    units = classify_box_units(box_abs_max, max(runner.width, runner.height))
+    if units["refusal"]:
+        raise SystemExit(f"REFUSING: {units['refusal']}")
+
+    same_source = bool(rss_before and rss_after_probe and rss_before[1] == rss_after_probe[1])
+    rss = {
+        "source": rss_before[1] if rss_before else None,
+        "before_session_bytes": rss_before[0] if rss_before else None,
+        "after_session_bytes": rss_after_session[0] if rss_after_session else None,
+        "after_probe_bytes": rss_after_probe[0] if rss_after_probe else None,
+        "delta_bytes": (rss_after_probe[0] - rss_before[0]) if same_source else None,
+        "probe_ran": True,
+        "probe_inferences": max(1, warmup_n),
+        "frames_resident_at_measurement": 1,
+        "measured_after_probe": True,
+        "source_note": (
+            "rusage-maxrss is a process HIGH-WATER MARK, so a delta is a LOWER BOUND on this "
+            "model's footprint and legitimately reads small once an earlier peak is behind it. "
+            "proc-statm and psapi-workingset are current and exact."
+        ),
+    }
+
+    payload = {
+        "schema": SCHEMA,
+        "candidate": args.candidate,
+        "produced_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "produced_by": "trillboards-edge-sdk/.../deploy/detector/bench/run_candidate.py",
+        "artifact": {
+            "path": str(model_path),
+            "sha256": sha256_file(model_path),
+            "bytes": model_path.stat().st_size,
+        },
+        "eval_set_manifest": {
+            "path": str(manifest_path),
+            "sha256": sha256_file(manifest_path),
+            "frames": len(keys),
+        },
+        "execution_path": runner.path_kind,
+        "preprocessing": runner.preprocessing,
+        "variant": {
+            "format": runner.variant_format,
+            "output_layout": runner.layout,
+            "end2end": runner.layout == "end2end_n_6",
+            "box_units": units["units"],
+            "box_scale_to_pixels": units["scale_to_pixels"],
+            "box_units_evidence": (
+                f"max|box channel| = {box_abs_max:.4f} over {len(keys)} real frames; "
+                f"/ imgsz {max(runner.width, runner.height)} = {units['ratio']:.4f}. "
+                f"pixels at >= {BOX_UNITS_PIXEL_RATIO_MIN}, normalized at <= "
+                f"{BOX_UNITS_NORMALIZED_RATIO_MAX}, refused in between."
+            ),
+            "box_units_ratio": units["ratio"],
+            "why_this_field_exists": (
+                "LiteRT and ONNX emit box coordinates in DIFFERENT UNITS for the same weights "
+                "(Ultralytics wraps a non-end2end detect export in _NormalizeCoords), and every "
+                "other check passes on the pair: same shape, same dtype, class scores agreeing to "
+                "1.2e-07. A consumer reusing the ONNX decode draws boxes 640x too small, silently."
+            ),
+        },
+        "builtin_vocabulary_used": builtin_used,
+        "runtime_contract": loadability,
+        "imgsz": [runner.width, runner.height],
+        "conf_floor": args.conf_floor,
+        "classes": [
+            {"index": idx, "name": name} for idx, name in sorted(runner.class_names().items())
+        ],
+        "device": args.device or describe_box(),
+        "providers": runner.providers,
+        "latency_ms": {
+            "inference_p50": percentile(infer_ms, 0.5),
+            "inference_p95": percentile(infer_ms, 0.95),
+            "inference_mean": statistics.fmean(infer_ms) if infer_ms else None,
+            "end_to_end_p50": percentile(wall_ms, 0.5),
+            "end_to_end_p95": percentile(wall_ms, 0.95),
+            "n": len(infer_ms),
+            "warmup_excluded": warmup_n,
+            "load_average_before": load_before,
+            "load_average_after": load_after,
+            "contention_note": (
+                "load_average is the machine's run queue during the measurement. Measured on this "
+                "box, two back-to-back runs of one artifact gave p50 68.3 ms and 276.2 ms at load "
+                "9.58 on 16 cores. A ms figure carries its hardware AND its contention, or a "
+                "reader cannot tell which of those two runs they are holding."
+            ),
+            "what_is_timed": (
+                "inference_* is the ORT forward pass only, matching the service's own "
+                "`inference_ms`. end_to_end_* adds decode, preprocess and postprocess, which is "
+                "what a caller waits for."
+            ),
+        },
+        "rss": rss,
+        "detections": detections,
+        "note": args.note,
+    }
+
+    out_path = Path(args.out)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text(json.dumps(payload, indent=2) + "\n")
+
+    print(f"{args.candidate}  |  {runner.path_kind} path  |  {payload['device']}")
+    print(
+        f"  artifact   {model_path.name}  {payload['artifact']['bytes']:,} B  "
+        f"sha256 {payload['artifact']['sha256'][:16]}..."
+    )
+    print(f"  imgsz      {runner.width}x{runner.height}  conf floor {args.conf_floor}")
+    print(f"  classes    {[c['name'] for c in payload['classes']]}")
+    v = payload["variant"]
+    print(f"  variant    format {v['format']}  layout {v['output_layout']}  "
+          f"box units {v['box_units']} (x{v['box_scale_to_pixels']:g} to pixels, ratio {v['box_units_ratio']:.4f})")
+    print(f"  shipped runtime loadable: {loadability['shipped_service_loadable']}")
+    if loadability["refusal"]:
+        print(f"    refusal: {loadability['refusal']}")
+    lat = payload["latency_ms"]
+    load = lat.get("load_average_before")
+    load_bit = (f"  load {load['load_1m']}/{load['cores']} cores (ratio {load['contention_ratio']})"
+                if load else "  load UNKNOWN")
+    print(f"  inference  p50 {lat['inference_p50']:.1f} ms  p95 {lat['inference_p95']:.1f} ms  n={lat['n']}{load_bit}")
+    if load and load["contention_ratio"] > 0.7:
+        print("    ^ CONTENDED BOX: this p95 is partly other processes. Re-measure on an idle machine "
+              "before quoting these figures.")
+    if rss["delta_bytes"] is not None:
+        print(
+            f"  RSS delta  {rss['delta_bytes'] / (1024 * 1024):.0f} MiB after "
+            f"{max(1, warmup_n)} probe inference(s), 1 frame resident  [{rss['source']}]"
+        )
+    else:
+        print(f"  RSS delta  UNAVAILABLE  [{rss['source']}]")
+    print(f"  wrote      {out_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())