@trillboards/edge-sdk 0.2.85 → 0.2.89

package/deploy/detector/bench/count_quantization.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+"""The control for `docs/runbooks/cctv-detector-tier.md` §4: never trust an
+int8 export FLAG, count the nodes it actually emitted.
+
+WHY THIS EXISTS. An int8 export flag can fail SILENTLY or LOUDLY depending on
+the library, and the difference is the argument NAME rather than the exporter:
+RF-DETR documents `quantization` as *ignored when format=onnx* and writes an
+all-FLOAT file without erroring, while ultralytics REJECTS that same spelling
+and quantises only under `quantize`. A runbook sentence written against one
+library reads as universal to the next reader — that is how the RF-DETR
+sentence got applied to ultralytics in five separate briefs on 2026-08-24
+before anyone measured it.
+
+What survives both behaviours is this count. It reads only the emitted graph,
+so it is indifferent to which library, which flag spelling, and which release
+produced the file — the property it reports is the one the runbook actually
+claims.
+
+NOT A CI STEP, deliberately, and for the same reason as
+`verify_against_reference.py` beside it: reproducing the quantised arm needs
+ultralytics plus torch (~3 GB) and a checkpoint download. Counting an ONNX file
+you already have needs only `onnx`. It is a documented manual run.
+
+    # Count any ONNX file you already have (needs only `onnx`):
+    python3 bench/count_quantization.py --onnx <exported>.onnx
+
+    # Reproduce the committed control from scratch:
+    uv venv /tmp/ultra --python 3.12
+    uv pip install --python /tmp/ultra/bin/python \\
+        'ultralytics==8.4.128' onnx onnxruntime onnxslim
+    /tmp/ultra/bin/python bench/count_quantization.py --reproduce-ultralytics
+
+`--reproduce-ultralytics` re-derives all three rows of the committed result
+(`bench/quantization-control.json`): the rejected spelling, the fp32 baseline,
+and the genuinely quantised arm.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+
+# Ultralytics colourises the argument name inside its own SyntaxError, so the
+# raw message carries ANSI escapes. Strip them: the committed control is a
+# readable record that must compare equal to a fresh run's output.
+ANSI = re.compile(r"\x1b\[[0-9;]*m")
+
+# The control is pinned to the smallest ultralytics detection checkpoint so the
+# arm is cheap to re-run. It is NOT a recommendation: §3 of the runbook rejects
+# raw COCO YOLO as a gate on accuracy grounds, and rejects INT8 outright (it
+# loses 31.7 points of recall SILENTLY and is slower than fp32). This model is
+# here only because it is the smallest artifact that can demonstrate a REAL
+# quantisation, against which a silent no-op is distinguishable.
+CONTROL_MODEL = "yolo11n.pt"
+CONTROL_ULTRALYTICS_VERSION = "8.4.128"
+
+
+def count_onnx(path: Path) -> dict:
+    """Node and initializer census of one ONNX file.
+
+    The two halves answer different questions and a real quantisation moves
+    BOTH: QDQ nodes prove the graph carries quantise/dequantise steps, and INT8
+    initializers prove the WEIGHTS were actually converted. A file with QDQ
+    nodes but no int8 weights is quantising activations only; a file with
+    neither is what a silently-ignored flag produces.
+
+    NON-FLOAT ALONE IS NOT THE PREDICATE, and this is the trap inside the trap:
+    a perfectly clean fp32 export still carries INT64 SHAPE CONSTANTS — the
+    committed yolo11n fp32 control has 15 of them against 181 FLOAT tensors. A
+    check that only counted "non-FLOAT initializers > 0" would call that file
+    quantised. `quantised` below therefore requires a QuantizeLinear node AND a
+    genuine INT8 weight tensor, and `initializers_by_dtype` is reported in full
+    so the next reader can see which dtypes were actually present rather than
+    trusting a single derived count.
+    """
+    import onnx
+    from onnx import TensorProto
+
+    model = onnx.load(str(path), load_external_data=False)
+    ops: dict[str, int] = {}
+    for node in model.graph.node:
+        ops[node.op_type] = ops.get(node.op_type, 0) + 1
+
+    by_dtype: dict[str, int] = {}
+    for init in model.graph.initializer:
+        name = TensorProto.DataType.Name(init.data_type)
+        by_dtype[name] = by_dtype.get(name, 0) + 1
+
+    total_init = sum(by_dtype.values())
+    non_float = total_init - by_dtype.get("FLOAT", 0)
+    return {
+        "file": path.name,
+        "size_bytes": path.stat().st_size,
+        "size_mb": round(path.stat().st_size / 1e6, 1),
+        "quantize_linear": ops.get("QuantizeLinear", 0),
+        "dequantize_linear": ops.get("DequantizeLinear", 0),
+        "initializers_total": total_init,
+        "initializers_by_dtype": dict(sorted(by_dtype.items())),
+        "initializers_non_float": non_float,
+        "initializers_int8": by_dtype.get("INT8", 0),
+        # The predicate the runbook tells you to trust. Deliberately NOT
+        # `non_float > 0` — see the docstring: INT64 shape constants make that
+        # true on a clean fp32 file.
+        "quantised": ops.get("QuantizeLinear", 0) > 0 and by_dtype.get("INT8", 0) > 0,
+    }
+
+
+def reproduce_ultralytics() -> dict:
+    """Re-derive the committed three-row control."""
+    from ultralytics import YOLO
+
+    rows: dict[str, object] = {
+        "model": CONTROL_MODEL,
+        "ultralytics": CONTROL_ULTRALYTICS_VERSION,
+    }
+
+    # Row 1 — the REJECTED spelling. RF-DETR ignores this silently; ultralytics
+    # raises. Same characters, opposite failure mode, which is the whole point.
+    try:
+        YOLO(CONTROL_MODEL).export(format="onnx", quantization="int8")
+        rows["rejected_spelling"] = {
+            "raised": False,
+            "note": "did NOT raise — ultralytics behaviour changed, update the runbook",
+        }
+    except Exception as exc:  # noqa: BLE001 - the exception IS the measurement
+        rows["rejected_spelling"] = {
+            "raised": True,
+            "type": type(exc).__name__,
+            "message": ANSI.sub("", str(exc)).strip().splitlines()[0][:300],
+        }
+
+    # Row 2 — fp32 baseline. Expect 0 QDQ nodes and 0 INT8 tensors. It is NOT
+    # expected to be free of non-FLOAT initializers: the INT64 shape constants
+    # are normal and are exactly why `quantised` does not key on that count.
+    fp32 = Path(YOLO(CONTROL_MODEL).export(format="onnx"))
+    rows["fp32"] = count_onnx(fp32)
+
+    # Row 3 — the correct spelling, which genuinely quantises. Ultralytics
+    # auto-fetches coco8 to calibrate.
+    int8 = Path(YOLO(CONTROL_MODEL).export(format="onnx", quantize="int8"))
+    rows["int8"] = count_onnx(int8)
+    return rows
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--onnx", help="count one ONNX file already on disk")
+    ap.add_argument(
+        "--reproduce-ultralytics",
+        action="store_true",
+        help="re-derive the committed control (needs ultralytics + torch)",
+    )
+    ap.add_argument("--json-out", help="write the result to this path")
+    args = ap.parse_args()
+
+    if not args.onnx and not args.reproduce_ultralytics:
+        ap.error("pass --onnx <file> or --reproduce-ultralytics")
+
+    result = (
+        reproduce_ultralytics()
+        if args.reproduce_ultralytics
+        else count_onnx(Path(args.onnx))
+    )
+    text = json.dumps(result, indent=2)
+    print(text)
+    if args.json_out:
+        Path(args.json_out).write_text(text + "\n", encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/deploy/detector/bench/quantization-control.json

@@ -0,0 +1,52 @@
+{
+  "_": "Committed control for docs/runbooks/cctv-detector-tier.md \u00a74. Regenerate with: bench/count_quantization.py --reproduce-ultralytics. Sizes are raw bytes; the MB/MiB split is why the first write-up of this control disagreed with itself (10.7 MB is 10.2 MiB).",
+  "measured_at": "2026-08-25",
+  "environment": {
+    "ultralytics": "8.4.128",
+    "onnx": "1.22.0",
+    "onnxslim": "0.1.96",
+    "onnx_opset": 18,
+    "torch": "2.13.0",
+    "python": "3.12.12",
+    "device": "CPU (Apple M4 Max)",
+    "calibration": "coco8.yaml (4 images; ultralytics warns >300 recommended)"
+  },
+  "model": "yolo11n.pt",
+  "rejected_spelling": {
+    "raised": true,
+    "type": "SyntaxError",
+    "message": "'quantization' is not a valid YOLO argument. Similar arguments are i.e. ['quantize']."
+  },
+  "fp32": {
+    "file": "yolo11n.onnx",
+    "size_bytes": 10741399,
+    "size_mb": 10.7,
+    "quantize_linear": 0,
+    "dequantize_linear": 0,
+    "initializers_total": 196,
+    "initializers_by_dtype": {
+      "FLOAT": 181,
+      "INT64": 15
+    },
+    "initializers_non_float": 15,
+    "initializers_int8": 0,
+    "quantised": false
+  },
+  "int8": {
+    "file": "yolo11n_int8.onnx",
+    "size_bytes": 3100688,
+    "size_mb": 3.1,
+    "quantize_linear": 174,
+    "dequantize_linear": 349,
+    "initializers_total": 894,
+    "initializers_by_dtype": {
+      "FLOAT": 355,
+      "INT32": 174,
+      "INT64": 15,
+      "INT8": 350
+    },
+    "initializers_non_float": 539,
+    "initializers_int8": 350,
+    "quantised": true
+  }
+}