@trillboards/edge-sdk 0.2.91 → 0.2.93
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/detector/bench/box_units.py +71 -0
- package/deploy/detector/bench/litert-twin-parity.json +241 -0
- package/deploy/detector/bench/run_candidate.py +7 -42
- package/deploy/detector/bench/twin_parity_contract.py +113 -0
- package/deploy/detector/bench/verify_litert_twin_parity.py +188 -0
- package/deploy/detector/tests/test_box_units.py +119 -0
- package/deploy/detector/tests/test_litert_twin_parity_contract.py +165 -0
- package/dist/EdgeAgent.js +1 -1
- package/dist/diagnose.d.ts +1 -1
- package/dist/diagnose.js +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""What a box coordinate MEANS, decided from a forward pass — STDLIB ONLY.
|
|
2
|
+
|
|
3
|
+
Split out of `run_candidate.py` so the `detector-service` CI lane can reach it.
|
|
4
|
+
That lane runs `python3 -m unittest discover` with NO pip install, and
|
|
5
|
+
`run_candidate.py` imports `trillboards_detector.session` and therefore
|
|
6
|
+
onnxruntime at module scope — so a test importing it does not skip, it errors
|
|
7
|
+
the whole lane during discovery. The decision lives here in stdlib; the forward
|
|
8
|
+
pass that produces `box_abs_max` stays there.
|
|
9
|
+
|
|
10
|
+
Same split, and the same reason, as `bench/twin_parity_contract.py`.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
#: LiteRT boxes in 0..1, agreeing to 0.00266 px after multiplying by 640, while
|
|
17
|
+
#: their CLASS channels agree to 1.2e-07. `queue_supermarket`, the one end2end
|
|
18
|
+
#: twin, agrees on both sides — end2end heads emit pixels everywhere.
|
|
19
|
+
#:
|
|
20
|
+
#: Every check a publish gate would otherwise run PASSES on that pair: same
|
|
21
|
+
#: output shape, same dtype, loads, runs, class scores identical. An Android
|
|
22
|
+
#: post-processor reusing the ONNX decode draws every box 640x too small and
|
|
23
|
+
#: nothing reports an error. So the units are MEASURED from a forward pass, per
|
|
24
|
+
#: artifact, and recorded — never carried across from the sibling format.
|
|
25
|
+
#:
|
|
26
|
+
#: The discriminator is a ratio, max|box channel| / imgsz, and it is decisive by
|
|
27
|
+
#: two and a half orders of magnitude. Measured over 119 ONNX artifacts the ratio
|
|
28
|
+
#: spans 0.988..1.167; a normalised head sits at ~1/640 = 0.0016. The dead zone
|
|
29
|
+
#: below is EMPTY across every artifact in the tree, and an artifact landing in it
|
|
30
|
+
#: is refused rather than guessed at — that is the whole point of the field.
|
|
31
|
+
BOX_UNITS_PIXEL_RATIO_MIN = 0.2
|
|
32
|
+
BOX_UNITS_NORMALIZED_RATIO_MAX = 0.02
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def classify_box_units(box_abs_max: float, imgsz: int) -> dict:
|
|
36
|
+
"""pixels, normalized, or REFUSE — never a guess.
|
|
37
|
+
|
|
38
|
+
Returns the verdict plus the evidence, because a bare enum in a score card is
|
|
39
|
+
a claim and the ratio is what makes it checkable.
|
|
40
|
+
"""
|
|
41
|
+
# `<= 0.0` MATTERS AS MUCH AS `is None`, and it is the case that actually
|
|
42
|
+
# happens. A model whose classification head is dead still emits box rows —
|
|
43
|
+
# `forklift_akdve_11n` has healthy boxes (2.93-636 px) and a max class score
|
|
44
|
+
# of 0.0063 — so nothing is admitted at threshold, `box_abs_max` stays at its
|
|
45
|
+
# 0.0 seed, and the ratio 0.0/imgsz lands under BOX_UNITS_NORMALIZED_RATIO_MAX.
|
|
46
|
+
# The old predicate then returned a CONFIDENT `normalized` with
|
|
47
|
+
# scale_to_pixels=640 from zero observations, and that declaration is exactly
|
|
48
|
+
# what `session.py` now trusts absolutely: the layout and units are DECLARED
|
|
49
|
+
# by the variant contract and never re-derived from the tensor. So a dead
|
|
50
|
+
# artifact published a coordinate-space claim no frame supports.
|
|
51
|
+
#
|
|
52
|
+
# The refusal string below already described this case. It just could not be
|
|
53
|
+
# reached, which is the whole shape of the defect this function exists to
|
|
54
|
+
# prevent — a check that runs, passes, and measures nothing.
|
|
55
|
+
if imgsz <= 0 or box_abs_max is None or box_abs_max <= 0.0:
|
|
56
|
+
return {"units": None, "ratio": None, "scale_to_pixels": None,
|
|
57
|
+
"refusal": "no box channels were observed, so the coordinate space is unmeasured"}
|
|
58
|
+
ratio = box_abs_max / imgsz
|
|
59
|
+
if ratio >= BOX_UNITS_PIXEL_RATIO_MIN:
|
|
60
|
+
return {"units": "pixels", "ratio": ratio, "scale_to_pixels": 1.0, "refusal": None}
|
|
61
|
+
if ratio <= BOX_UNITS_NORMALIZED_RATIO_MAX:
|
|
62
|
+
return {"units": "normalized", "ratio": ratio, "scale_to_pixels": float(imgsz), "refusal": None}
|
|
63
|
+
return {
|
|
64
|
+
"units": None, "ratio": ratio, "scale_to_pixels": None,
|
|
65
|
+
"refusal": (
|
|
66
|
+
f"max|box|/imgsz = {ratio:.4f} falls between {BOX_UNITS_NORMALIZED_RATIO_MAX} and "
|
|
67
|
+
f"{BOX_UNITS_PIXEL_RATIO_MIN}, where neither pixels nor normalised is credible. "
|
|
68
|
+
"Measured over 119 artifacts this band is empty; refusing rather than picking one, "
|
|
69
|
+
"because the wrong pick is silent and scales every box by 640."
|
|
70
|
+
),
|
|
71
|
+
}
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
{
|
|
2
|
+
"_comment": [
|
|
3
|
+
"MEASURED, not asserted. Produced by bench/verify_litert_twin_parity.py on 2026-08-25,",
|
|
4
|
+
"one fp32 input tensor fed to both runtimes (onnxruntime 1.29.0 CPU, ai-edge-litert),",
|
|
5
|
+
"seed 1234, both taking NCHW [1,3,640,640] \u2014 the LiteRT export of an ultralytics detect",
|
|
6
|
+
"model is NOT NHWC.",
|
|
7
|
+
"",
|
|
8
|
+
"THE RESULT: the box scale is a property of the LAYOUT, not of the runtime.",
|
|
9
|
+
" anchor_4_plus_nc LiteRT boxes are NORMALISED; box*imgsz recovers ONNX pixels",
|
|
10
|
+
" to <= 0.00588 px, and the class channels already agree to",
|
|
11
|
+
" <= 4.2e-06 with no scaling at all.",
|
|
12
|
+
" end2end_n_6 the _NormalizeCoords wrapper is NOT applied; both runtimes",
|
|
13
|
+
" emit pixels, so the scale is 1.0.",
|
|
14
|
+
"A decoder that scales every LiteRT model is exactly as wrong as one that scales",
|
|
15
|
+
"none \u2014 it just corrupts the other half of the library, and silently, because",
|
|
16
|
+
"shape, dtype, class count and every score still match.",
|
|
17
|
+
"",
|
|
18
|
+
"queue_supermarket is recorded FAILING on purpose. Its box value RANGE is identical",
|
|
19
|
+
"in both runtimes and already in pixels (litert [-45.104, 652.062], onnx the same) and",
|
|
20
|
+
"its scores agree to 7.2e-08, but row-wise the two disagree and lexsorting by box does",
|
|
21
|
+
"not collapse it. The model returns ZERO detections above 0.25 on 40 real production",
|
|
22
|
+
"frames, so there is no above-threshold sample to compare and row-level parity is",
|
|
23
|
+
"UNVERIFIED. It is not published. Settle it on a corpus the model actually fires on."
|
|
24
|
+
],
|
|
25
|
+
"measured_at": "2026-08-25",
|
|
26
|
+
"tolerances": {
|
|
27
|
+
"box_residual_px": 0.05,
|
|
28
|
+
"cls_residual": 0.0001
|
|
29
|
+
},
|
|
30
|
+
"box_scale_by_layout": {
|
|
31
|
+
"anchor_4_plus_nc": "imgsz",
|
|
32
|
+
"end2end_n_6": 1.0,
|
|
33
|
+
"rfdetr_dets_labels": 1.0
|
|
34
|
+
},
|
|
35
|
+
"twins": [
|
|
36
|
+
{
|
|
37
|
+
"layout": "anchor_4_plus_nc",
|
|
38
|
+
"imgsz": 640,
|
|
39
|
+
"box_scale_applied": 640.0,
|
|
40
|
+
"box_residual_px": 0.000865936279296875,
|
|
41
|
+
"box_residual_unscaled": 641.0272229909897,
|
|
42
|
+
"cls_residual": 1.996755599975586e-06,
|
|
43
|
+
"pass": true,
|
|
44
|
+
"reason": "ok",
|
|
45
|
+
"onnx": "apron_murugu.onnx",
|
|
46
|
+
"litert": "apron_murugu.tflite"
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"layout": "anchor_4_plus_nc",
|
|
50
|
+
"imgsz": 640,
|
|
51
|
+
"box_scale_applied": 640.0,
|
|
52
|
+
"box_residual_px": 0.000476837158203125,
|
|
53
|
+
"box_residual_unscaled": 635.8220509290695,
|
|
54
|
+
"cls_residual": 1.1929387255804613e-07,
|
|
55
|
+
"pass": true,
|
|
56
|
+
"reason": "ok",
|
|
57
|
+
"onnx": "fall_posture_3cls_11n.onnx",
|
|
58
|
+
"litert": "fall_posture_3cls_11n.tflite"
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"layout": "anchor_4_plus_nc",
|
|
62
|
+
"imgsz": 640,
|
|
63
|
+
"box_scale_applied": 640.0,
|
|
64
|
+
"box_residual_px": 0.0030517578125,
|
|
65
|
+
"box_residual_unscaled": 656.5295736789703,
|
|
66
|
+
"cls_residual": 6.020069122314453e-06,
|
|
67
|
+
"pass": true,
|
|
68
|
+
"reason": "ok",
|
|
69
|
+
"onnx": "fire_5cls_fpsuppress_11n.onnx",
|
|
70
|
+
"litert": "fire_5cls_fpsuppress_11n.tflite"
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"layout": "anchor_4_plus_nc",
|
|
74
|
+
"imgsz": 640,
|
|
75
|
+
"box_scale_applied": 640.0,
|
|
76
|
+
"box_residual_px": 0.00335693359375,
|
|
77
|
+
"box_residual_unscaled": 635.6288108825684,
|
|
78
|
+
"cls_residual": 1.237494871020317e-07,
|
|
79
|
+
"pass": true,
|
|
80
|
+
"reason": "ok",
|
|
81
|
+
"onnx": "forklift_akdve_11n.onnx",
|
|
82
|
+
"litert": "forklift_akdve_11n.tflite"
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"layout": "anchor_4_plus_nc",
|
|
86
|
+
"imgsz": 640,
|
|
87
|
+
"box_scale_applied": 640.0,
|
|
88
|
+
"box_residual_px": 0.001369476318359375,
|
|
89
|
+
"box_residual_unscaled": 634.430369079113,
|
|
90
|
+
"cls_residual": 1.3195858628023416e-07,
|
|
91
|
+
"pass": true,
|
|
92
|
+
"reason": "ok",
|
|
93
|
+
"onnx": "glove_madesh.onnx",
|
|
94
|
+
"litert": "glove_madesh.tflite"
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
"layout": "anchor_4_plus_nc",
|
|
98
|
+
"imgsz": 640,
|
|
99
|
+
"box_scale_applied": 640.0,
|
|
100
|
+
"box_residual_px": 0.00102996826171875,
|
|
101
|
+
"box_residual_unscaled": 648.4288479089737,
|
|
102
|
+
"cls_residual": 7.897615432739258e-07,
|
|
103
|
+
"pass": true,
|
|
104
|
+
"reason": "ok",
|
|
105
|
+
"onnx": "glove_sunita.onnx",
|
|
106
|
+
"litert": "glove_sunita.tflite"
|
|
107
|
+
},
|
|
108
|
+
{
|
|
109
|
+
"layout": "anchor_4_plus_nc",
|
|
110
|
+
"imgsz": 640,
|
|
111
|
+
"box_scale_applied": 640.0,
|
|
112
|
+
"box_residual_px": 0.0028209686279296875,
|
|
113
|
+
"box_residual_unscaled": 637.0318885445595,
|
|
114
|
+
"cls_residual": 1.6111880540847778e-07,
|
|
115
|
+
"pass": true,
|
|
116
|
+
"reason": "ok",
|
|
117
|
+
"onnx": "hf_crowd_irail.onnx",
|
|
118
|
+
"litert": "hf_crowd_irail.tflite"
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
"layout": "anchor_4_plus_nc",
|
|
122
|
+
"imgsz": 640,
|
|
123
|
+
"box_scale_applied": 640.0,
|
|
124
|
+
"box_residual_px": 0.001659393310546875,
|
|
125
|
+
"box_residual_unscaled": 639.9944154024124,
|
|
126
|
+
"cls_residual": 1.6391277313232422e-06,
|
|
127
|
+
"pass": true,
|
|
128
|
+
"reason": "ok",
|
|
129
|
+
"onnx": "hf_smoking.onnx",
|
|
130
|
+
"litert": "hf_smoking.tflite"
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
"layout": "anchor_4_plus_nc",
|
|
134
|
+
"imgsz": 640,
|
|
135
|
+
"box_scale_applied": 640.0,
|
|
136
|
+
"box_residual_px": 0.00424957275390625,
|
|
137
|
+
"box_residual_unscaled": 637.0888672471046,
|
|
138
|
+
"cls_residual": 1.2430425044840376e-07,
|
|
139
|
+
"pass": true,
|
|
140
|
+
"reason": "ok",
|
|
141
|
+
"onnx": "hg-v6.onnx",
|
|
142
|
+
"litert": "hg-v6.tflite"
|
|
143
|
+
},
|
|
144
|
+
{
|
|
145
|
+
"layout": "anchor_4_plus_nc",
|
|
146
|
+
"imgsz": 640,
|
|
147
|
+
"box_scale_applied": 640.0,
|
|
148
|
+
"box_residual_px": 0.00177001953125,
|
|
149
|
+
"box_residual_unscaled": 638.4268003106117,
|
|
150
|
+
"cls_residual": 1.3917764363213792e-07,
|
|
151
|
+
"pass": true,
|
|
152
|
+
"reason": "ok",
|
|
153
|
+
"onnx": "ppe_keremberke_v8m.onnx",
|
|
154
|
+
"litert": "ppe_keremberke_v8m.tflite"
|
|
155
|
+
},
|
|
156
|
+
{
|
|
157
|
+
"layout": "anchor_4_plus_nc",
|
|
158
|
+
"imgsz": 640,
|
|
159
|
+
"box_scale_applied": 640.0,
|
|
160
|
+
"box_residual_px": 0.002044677734375,
|
|
161
|
+
"box_residual_unscaled": 663.3222885131836,
|
|
162
|
+
"cls_residual": 1.227854227181524e-07,
|
|
163
|
+
"pass": true,
|
|
164
|
+
"reason": "ok",
|
|
165
|
+
"onnx": "ppe_keremberke_v8n.onnx",
|
|
166
|
+
"litert": "ppe_keremberke_v8n.tflite"
|
|
167
|
+
},
|
|
168
|
+
{
|
|
169
|
+
"layout": "anchor_4_plus_nc",
|
|
170
|
+
"imgsz": 640,
|
|
171
|
+
"box_scale_applied": 640.0,
|
|
172
|
+
"box_residual_px": 0.00238800048828125,
|
|
173
|
+
"box_residual_unscaled": 652.3816465139389,
|
|
174
|
+
"cls_residual": 1.3384212138589646e-07,
|
|
175
|
+
"pass": true,
|
|
176
|
+
"reason": "ok",
|
|
177
|
+
"onnx": "ppe_keremberke_v8s.onnx",
|
|
178
|
+
"litert": "ppe_keremberke_v8s.tflite"
|
|
179
|
+
},
|
|
180
|
+
{
|
|
181
|
+
"layout": "end2end_n_6",
|
|
182
|
+
"imgsz": 640,
|
|
183
|
+
"box_scale_applied": 1.0,
|
|
184
|
+
"box_residual_px": 630.3428831100464,
|
|
185
|
+
"box_residual_unscaled": 630.3428831100464,
|
|
186
|
+
"cls_residual": 8.372444426640868e-08,
|
|
187
|
+
"pass": false,
|
|
188
|
+
"reason": "box residual 630.3 px exceeds 0.05 under layout end2end_n_6 (scale x1.0)",
|
|
189
|
+
"onnx": "queue_supermarket.onnx",
|
|
190
|
+
"litert": "queue_supermarket.tflite"
|
|
191
|
+
},
|
|
192
|
+
{
|
|
193
|
+
"layout": "anchor_4_plus_nc",
|
|
194
|
+
"imgsz": 640,
|
|
195
|
+
"box_scale_applied": 640.0,
|
|
196
|
+
"box_residual_px": 0.000308990478515625,
|
|
197
|
+
"box_residual_unscaled": 647.0199807882309,
|
|
198
|
+
"cls_residual": 1.218754732690286e-07,
|
|
199
|
+
"pass": true,
|
|
200
|
+
"reason": "ok",
|
|
201
|
+
"onnx": "shoplift_pavan.onnx",
|
|
202
|
+
"litert": "shoplift_pavan.tflite"
|
|
203
|
+
},
|
|
204
|
+
{
|
|
205
|
+
"layout": "anchor_4_plus_nc",
|
|
206
|
+
"imgsz": 640,
|
|
207
|
+
"box_scale_applied": 640.0,
|
|
208
|
+
"box_residual_px": 0.000621795654296875,
|
|
209
|
+
"box_residual_unscaled": 634.978705227375,
|
|
210
|
+
"cls_residual": 1.3541824728235952e-07,
|
|
211
|
+
"pass": true,
|
|
212
|
+
"reason": "ok",
|
|
213
|
+
"onnx": "spill_fluid.onnx",
|
|
214
|
+
"litert": "spill_fluid.tflite"
|
|
215
|
+
},
|
|
216
|
+
{
|
|
217
|
+
"layout": "anchor_4_plus_nc",
|
|
218
|
+
"imgsz": 640,
|
|
219
|
+
"box_scale_applied": 640.0,
|
|
220
|
+
"box_residual_px": 0.0025310516357421875,
|
|
221
|
+
"box_residual_unscaled": 633.9265187978745,
|
|
222
|
+
"cls_residual": 1.9371509552001953e-07,
|
|
223
|
+
"pass": true,
|
|
224
|
+
"reason": "ok",
|
|
225
|
+
"onnx": "spill_robot.onnx",
|
|
226
|
+
"litert": "spill_robot.tflite"
|
|
227
|
+
},
|
|
228
|
+
{
|
|
229
|
+
"layout": "anchor_4_plus_nc",
|
|
230
|
+
"imgsz": 640,
|
|
231
|
+
"box_scale_applied": 640.0,
|
|
232
|
+
"box_residual_px": 0.00058746337890625,
|
|
233
|
+
"box_residual_unscaled": 641.6996326446533,
|
|
234
|
+
"cls_residual": 1.0132789611816406e-06,
|
|
235
|
+
"pass": true,
|
|
236
|
+
"reason": "ok",
|
|
237
|
+
"onnx": "weapon_gunknife_11n.onnx",
|
|
238
|
+
"litert": "weapon_gunknife_11n.tflite"
|
|
239
|
+
}
|
|
240
|
+
]
|
|
241
|
+
}
|
|
@@ -82,7 +82,13 @@ import time
|
|
|
82
82
|
from pathlib import Path
|
|
83
83
|
|
|
84
84
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) # noqa: E402
|
|
85
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent)) # noqa: E402 (bench siblings)
|
|
85
86
|
|
|
87
|
+
from box_units import ( # noqa: E402 (sibling module, stdlib-only on purpose)
|
|
88
|
+
BOX_UNITS_NORMALIZED_RATIO_MAX,
|
|
89
|
+
BOX_UNITS_PIXEL_RATIO_MIN,
|
|
90
|
+
classify_box_units,
|
|
91
|
+
)
|
|
86
92
|
from trillboards_detector.errors import ModelContractError # noqa: E402
|
|
87
93
|
from trillboards_detector.labels import ( # noqa: E402
|
|
88
94
|
BOX_UNITS_PIXELS,
|
|
@@ -107,23 +113,7 @@ SCHEMA = "detector-candidate-run/2"
|
|
|
107
113
|
#: Ultralytics' TFLite export path wraps a `task=detect`, `end2end=False` model in
|
|
108
114
|
#: `_NormalizeCoords`, dividing the four box channels by (W, H). Measured here on
|
|
109
115
|
#: all 17 committed LiteRT twins: 16 of 17 have ONNX boxes in pixels 0..640 and
|
|
110
|
-
|
|
111
|
-
#: their CLASS channels agree to 1.2e-07. `queue_supermarket`, the one end2end
|
|
112
|
-
#: twin, agrees on both sides — end2end heads emit pixels everywhere.
|
|
113
|
-
#:
|
|
114
|
-
#: Every check a publish gate would otherwise run PASSES on that pair: same
|
|
115
|
-
#: output shape, same dtype, loads, runs, class scores identical. An Android
|
|
116
|
-
#: post-processor reusing the ONNX decode draws every box 640x too small and
|
|
117
|
-
#: nothing reports an error. So the units are MEASURED from a forward pass, per
|
|
118
|
-
#: artifact, and recorded — never carried across from the sibling format.
|
|
119
|
-
#:
|
|
120
|
-
#: The discriminator is a ratio, max|box channel| / imgsz, and it is decisive by
|
|
121
|
-
#: two and a half orders of magnitude. Measured over 119 ONNX artifacts the ratio
|
|
122
|
-
#: spans 0.988..1.167; a normalised head sits at ~1/640 = 0.0016. The dead zone
|
|
123
|
-
#: below is EMPTY across every artifact in the tree, and an artifact landing in it
|
|
124
|
-
#: is refused rather than guessed at — that is the whole point of the field.
|
|
125
|
-
BOX_UNITS_PIXEL_RATIO_MIN = 0.2
|
|
126
|
-
BOX_UNITS_NORMALIZED_RATIO_MAX = 0.02
|
|
116
|
+
|
|
127
117
|
|
|
128
118
|
#: Warm-up forward passes before any timing is kept. ORT's first run pays for
|
|
129
119
|
#: arena allocation and kernel selection; folding that into a p50 over 134 frames
|
|
@@ -271,31 +261,6 @@ def classify_output_layout(output_shapes: "list[list]") -> str:
|
|
|
271
261
|
return f"unknown_rank_{len(shape)}"
|
|
272
262
|
|
|
273
263
|
|
|
274
|
-
def classify_box_units(box_abs_max: float, imgsz: int) -> dict:
|
|
275
|
-
"""pixels, normalized, or REFUSE — never a guess.
|
|
276
|
-
|
|
277
|
-
Returns the verdict plus the evidence, because a bare enum in a score card is
|
|
278
|
-
a claim and the ratio is what makes it checkable.
|
|
279
|
-
"""
|
|
280
|
-
if imgsz <= 0 or box_abs_max is None:
|
|
281
|
-
return {"units": None, "ratio": None, "scale_to_pixels": None,
|
|
282
|
-
"refusal": "no box channels were observed, so the coordinate space is unmeasured"}
|
|
283
|
-
ratio = box_abs_max / imgsz
|
|
284
|
-
if ratio >= BOX_UNITS_PIXEL_RATIO_MIN:
|
|
285
|
-
return {"units": "pixels", "ratio": ratio, "scale_to_pixels": 1.0, "refusal": None}
|
|
286
|
-
if ratio <= BOX_UNITS_NORMALIZED_RATIO_MAX:
|
|
287
|
-
return {"units": "normalized", "ratio": ratio, "scale_to_pixels": float(imgsz), "refusal": None}
|
|
288
|
-
return {
|
|
289
|
-
"units": None, "ratio": ratio, "scale_to_pixels": None,
|
|
290
|
-
"refusal": (
|
|
291
|
-
f"max|box|/imgsz = {ratio:.4f} falls between {BOX_UNITS_NORMALIZED_RATIO_MAX} and "
|
|
292
|
-
f"{BOX_UNITS_PIXEL_RATIO_MIN}, where neither pixels nor normalised is credible. "
|
|
293
|
-
"Measured over 119 artifacts this band is empty; refusing rather than picking one, "
|
|
294
|
-
"because the wrong pick is silent and scales every box by 640."
|
|
295
|
-
),
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
|
|
299
264
|
def letterbox(image, width: int, height: int):
|
|
300
265
|
"""Ultralytics' resize: scale to fit, pad the remainder 114-grey, centred.
|
|
301
266
|
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""The pure-logic half of the twin-parity check — STDLIB ONLY, deliberately.
|
|
2
|
+
|
|
3
|
+
`verify_litert_twin_parity.py` needs onnxruntime, ai-edge-litert, numpy and
|
|
4
|
+
Pillow, which together are about a gigabyte of wheels and are on no CI runner.
|
|
5
|
+
The `detector-service` step in `ci.yml` runs
|
|
6
|
+
|
|
7
|
+
python3 -m unittest discover -s <this package> -p "test_*.py"
|
|
8
|
+
|
|
9
|
+
with NO pip install, and that constraint is the point rather than an accident:
|
|
10
|
+
`server.py` imports no numpy either, so the whole wire contract runs on a bare
|
|
11
|
+
runner. A test that imported the verifier would drag numpy into discovery and
|
|
12
|
+
error the entire lane before its existing tests ran — which is the same defect
|
|
13
|
+
this file exists to catch, one level up: a check that appears to run and does
|
|
14
|
+
not.
|
|
15
|
+
|
|
16
|
+
So the decisions a reader has to trust — which layout a graph is, what scale
|
|
17
|
+
that layout implies, and whether a recorded verdict follows from its recorded
|
|
18
|
+
residuals — live here, in stdlib, and are exercised in CI. The arithmetic over
|
|
19
|
+
real tensors lives in the verifier and is a documented manual run.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
#: Tolerances. Measured across 16 anchor-layout twins on 2026-08-25: the worst
|
|
25
|
+
#: scaled box residual was 0.00588 px and the worst class residual 4.2e-06.
|
|
26
|
+
#: These ceilings sit ~2 orders above the observed worst, so a genuine export
|
|
27
|
+
#: defect fails while fp32 reassociation noise does not.
|
|
28
|
+
BOX_TOL_PX = 0.05
|
|
29
|
+
CLS_TOL = 1e-04
|
|
30
|
+
|
|
31
|
+
ANCHOR = "anchor_4_plus_nc"
|
|
32
|
+
END2END = "end2end_n_6"
|
|
33
|
+
DETS_LABELS = "rfdetr_dets_labels"
|
|
34
|
+
|
|
35
|
+
#: Which layouts this bench can COMPARE. `DETS_LABELS` is deliberately absent:
|
|
36
|
+
#: RF-DETR has no LiteRT twin and its own control is `verify_against_reference.py`,
|
|
37
|
+
#: so classifying it correctly and then refusing it by name is the honest
|
|
38
|
+
#: behaviour. Advertising a layout the comparison cannot split would be the
|
|
39
|
+
#: `yolov8_nano` shape in a bench script.
|
|
40
|
+
COMPARABLE_LAYOUTS = frozenset({ANCHOR, END2END})
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class ParityError(RuntimeError):
|
|
44
|
+
"""The twin does not agree with its ONNX source, or cannot be compared."""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def classify_layout(shape, output_names) -> str:
|
|
48
|
+
"""Name the layout from the GRAPH, never from the values.
|
|
49
|
+
|
|
50
|
+
`dets`/`labels` is RF-DETR's two-output form. Otherwise a rank-3 tensor is
|
|
51
|
+
`end2end_n_6` when its LAST axis is 6 (N rows of x1,y1,x2,y2,score,class),
|
|
52
|
+
and `anchor_4_plus_nc` when the MIDDLE axis is the channel axis. Those two
|
|
53
|
+
are separated by which axis is the small one — a fact about the graph's
|
|
54
|
+
declared shape, needing no sample to decide.
|
|
55
|
+
|
|
56
|
+
The runtime decoder does not even do this: `session.py` reads the layout
|
|
57
|
+
from the artifact's DECLARED variant contract, because `[1, 300, 6]` and
|
|
58
|
+
`[1, 6, 8400]` are both rank-3 with a 6 in them and a reader that decides
|
|
59
|
+
from shape alone gets one of them backwards with no exception and a
|
|
60
|
+
plausible table. This function is the bench's convenience, and it is
|
|
61
|
+
deliberately stricter than it needs to be so a new head shape refuses
|
|
62
|
+
rather than being absorbed.
|
|
63
|
+
"""
|
|
64
|
+
shape = tuple(int(v) for v in shape)
|
|
65
|
+
lowered = [str(n).lower() for n in output_names]
|
|
66
|
+
if "dets" in lowered and "labels" in lowered:
|
|
67
|
+
return DETS_LABELS
|
|
68
|
+
if len(shape) != 3:
|
|
69
|
+
raise ParityError(f"unknown output rank {len(shape)} for shape {shape}")
|
|
70
|
+
if shape[2] == 6 and shape[1] > 6:
|
|
71
|
+
return END2END
|
|
72
|
+
if shape[1] >= 5 and shape[2] > shape[1]:
|
|
73
|
+
return ANCHOR
|
|
74
|
+
raise ParityError(f"shape {shape} matches no known detector layout")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def box_scale_for(layout: str, imgsz: int) -> float:
|
|
78
|
+
"""The multiplier that puts LiteRT boxes into the ONNX pixel frame.
|
|
79
|
+
|
|
80
|
+
Ultralytics wraps a LiteRT export of a `task=detect` / `end2end=False`
|
|
81
|
+
model in `_NormalizeCoords`, which divides the four BOX channels by (W, H)
|
|
82
|
+
and leaves the class channels alone. An end2end head is `end2end=True`, so
|
|
83
|
+
the wrapper never runs and both runtimes emit pixels.
|
|
84
|
+
|
|
85
|
+
No layout gets a scale by default. A fall-through to 1.0 would be a guess
|
|
86
|
+
about an artifact nobody measured, and the wrong guess is silent.
|
|
87
|
+
"""
|
|
88
|
+
if layout == ANCHOR:
|
|
89
|
+
return float(imgsz)
|
|
90
|
+
if layout in (END2END, DETS_LABELS):
|
|
91
|
+
return 1.0
|
|
92
|
+
raise ParityError(f"no box scale defined for layout {layout!r}")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def verdict(result) -> "tuple[bool, str]":
|
|
96
|
+
"""Does this recorded measurement pass, and if not, which assertion failed."""
|
|
97
|
+
layout = result["layout"]
|
|
98
|
+
# The scale must be LOAD-BEARING on the anchor layout. If scaling and not
|
|
99
|
+
# scaling both pass, the model is not telling us which one is right and the
|
|
100
|
+
# tolerance is doing no work — that is the 640x bug passing its own check.
|
|
101
|
+
if layout == ANCHOR and result["box_residual_unscaled"] <= BOX_TOL_PX:
|
|
102
|
+
return False, (
|
|
103
|
+
"the imgsz scale changed nothing — an anchor-layout twin whose raw boxes "
|
|
104
|
+
"already match its ONNX source is not normalised, so the layout is misread"
|
|
105
|
+
)
|
|
106
|
+
if result["box_residual_px"] > BOX_TOL_PX:
|
|
107
|
+
return False, (
|
|
108
|
+
f"box residual {result['box_residual_px']:.4g} px exceeds {BOX_TOL_PX} "
|
|
109
|
+
f"under layout {layout} (scale x{result['box_scale_applied']})"
|
|
110
|
+
)
|
|
111
|
+
if result["cls_residual"] > CLS_TOL:
|
|
112
|
+
return False, f"class residual {result['cls_residual']:.4g} exceeds {CLS_TOL}"
|
|
113
|
+
return True, "ok"
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""The control for the SECOND runtime: one model, two runtimes, same tensor.
|
|
3
|
+
|
|
4
|
+
WHY THIS EXISTS. Every detector we publish ships twice — an ONNX artifact for
|
|
5
|
+
the venue box and a LiteRT twin for the Android screen device. They are exported
|
|
6
|
+
from one `.pt`, so it is tempting to treat the twin as free. It is not, and the
|
|
7
|
+
way it fails is the way this repo's defects always fail: the check runs, passes,
|
|
8
|
+
and measures the wrong thing.
|
|
9
|
+
|
|
10
|
+
Ultralytics wraps a LiteRT export of a `task=detect` / `end2end=False` model in
|
|
11
|
+
`_NormalizeCoords`, which divides the four box channels by (W, H). The class
|
|
12
|
+
channels are untouched. So a twin whose boxes are 640x wrong still matches its
|
|
13
|
+
ONNX source on output shape, on dtype, on class count, and on every score — the
|
|
14
|
+
four things a parity check naturally asserts. Only the boxes disagree, and only
|
|
15
|
+
if you look at them separately from the scores.
|
|
16
|
+
|
|
17
|
+
The scale is a property of the LAYOUT, not of the runtime:
|
|
18
|
+
|
|
19
|
+
* `anchor_4_plus_nc` [1, 4+nc, 8400] LiteRT emits NORMALISED boxes.
|
|
20
|
+
Multiply by imgsz to recover ONNX px.
|
|
21
|
+
* `end2end_n_6` [1, 300, 6] the wrapper is NOT applied; both
|
|
22
|
+
runtimes already emit pixels.
|
|
23
|
+
|
|
24
|
+
Which means a decoder that scales every LiteRT model is as wrong as one that
|
|
25
|
+
scales none — it just picks a different half of the library to corrupt. See
|
|
26
|
+
`docs/runbooks/cctv-detector-tier.md`.
|
|
27
|
+
|
|
28
|
+
Detect the layout from the GRAPH — output names and rank — never from the
|
|
29
|
+
magnitude of the values being interpreted. A layout guess that reads the numbers
|
|
30
|
+
it is trying to interpret will always find a self-consistent story: dispatching
|
|
31
|
+
on `shape[1] >= 5` routes a [1, 300, 6] end2end tensor into the anchor branch
|
|
32
|
+
and reports a confident, meaningless class residual of 632.
|
|
33
|
+
|
|
34
|
+
NOT A CI STEP, deliberately: it needs both runtimes and the artifact pair, which
|
|
35
|
+
together are ~1 GB of wheels. It is a documented manual run, the same shape as
|
|
36
|
+
`verify_against_reference.py` beside it. What CI holds instead is the measured
|
|
37
|
+
result, in `bench/litert-twin-parity.json`, which the publish path reads.
|
|
38
|
+
|
|
39
|
+
uv venv /tmp/twin --python 3.11
|
|
40
|
+
uv pip install --python /tmp/twin/bin/python onnxruntime ai-edge-litert numpy pillow
|
|
41
|
+
/tmp/twin/bin/python bench/verify_litert_twin_parity.py \\
|
|
42
|
+
--onnx <exported>.onnx --litert <exported>.tflite
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
from __future__ import annotations
|
|
46
|
+
|
|
47
|
+
import argparse
|
|
48
|
+
import json
|
|
49
|
+
import sys
|
|
50
|
+
from pathlib import Path
|
|
51
|
+
|
|
52
|
+
import numpy as np
|
|
53
|
+
|
|
54
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
55
|
+
|
|
56
|
+
from twin_parity_contract import ( # noqa: E402 (same directory)
|
|
57
|
+
ANCHOR,
|
|
58
|
+
BOX_TOL_PX,
|
|
59
|
+
CLS_TOL,
|
|
60
|
+
COMPARABLE_LAYOUTS,
|
|
61
|
+
DETS_LABELS,
|
|
62
|
+
END2END,
|
|
63
|
+
ParityError,
|
|
64
|
+
box_scale_for,
|
|
65
|
+
classify_layout,
|
|
66
|
+
verdict,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def split_channels(arr: np.ndarray, layout: str):
|
|
71
|
+
"""Return (boxes, class_or_score) for one output tensor."""
|
|
72
|
+
if layout == ANCHOR:
|
|
73
|
+
return arr[:, :4, :], arr[:, 4:, :]
|
|
74
|
+
if layout == END2END:
|
|
75
|
+
return arr[..., :4], arr[..., 4:]
|
|
76
|
+
raise ParityError(f"cannot split channels for layout {layout}")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def compare(onnx_out: np.ndarray, litert_out: np.ndarray, layout: str, imgsz: int) -> dict:
|
|
80
|
+
# Refuse BEFORE touching the tensors. `classify_layout` names RF-DETR
|
|
81
|
+
# correctly and `box_scale_for` knows its scale, but this bench cannot split
|
|
82
|
+
# a two-output graph into box and class halves — and RF-DETR has no LiteRT
|
|
83
|
+
# twin to compare against anyway. Its control is `verify_against_reference.py`
|
|
84
|
+
# beside this file. Advertising a layout the comparison cannot handle is the
|
|
85
|
+
# `yolov8_nano` shape in a bench script, so it stops here with the pointer.
|
|
86
|
+
if layout not in COMPARABLE_LAYOUTS:
|
|
87
|
+
raise ParityError(
|
|
88
|
+
f"layout {layout!r} is classified but not comparable by this bench "
|
|
89
|
+
f"(comparable: {sorted(COMPARABLE_LAYOUTS)}). For {DETS_LABELS} use "
|
|
90
|
+
"bench/verify_against_reference.py, which is RF-DETR's control."
|
|
91
|
+
)
|
|
92
|
+
if onnx_out.shape != litert_out.shape:
|
|
93
|
+
raise ParityError(f"shape mismatch onnx={onnx_out.shape} litert={litert_out.shape}")
|
|
94
|
+
a = litert_out.astype(np.float64)
|
|
95
|
+
b = onnx_out.astype(np.float64)
|
|
96
|
+
scale = box_scale_for(layout, imgsz)
|
|
97
|
+
box_l, cls_l = split_channels(a, layout)
|
|
98
|
+
box_o, cls_o = split_channels(b, layout)
|
|
99
|
+
return {
|
|
100
|
+
"layout": layout,
|
|
101
|
+
"imgsz": imgsz,
|
|
102
|
+
"box_scale_applied": scale,
|
|
103
|
+
"box_residual_px": float(np.abs(box_l * scale - box_o).max()),
|
|
104
|
+
"box_residual_unscaled": float(np.abs(box_l - box_o).max()),
|
|
105
|
+
"cls_residual": float(np.abs(cls_l - cls_o).max()),
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def verdict(result: dict) -> tuple[bool, str]:
|
|
110
|
+
# The scale must be LOAD-BEARING on the anchor layout. If scaling and not
|
|
111
|
+
# scaling both pass, the model is not telling us which one is right and the
|
|
112
|
+
# tolerance is doing no work — that is the 640x bug passing its own check.
|
|
113
|
+
if result["layout"] == ANCHOR and result["box_residual_unscaled"] <= BOX_TOL_PX:
|
|
114
|
+
return False, (
|
|
115
|
+
"the imgsz scale changed nothing — an anchor-layout twin whose raw boxes "
|
|
116
|
+
"already match its ONNX source is not normalised, so the layout is misread"
|
|
117
|
+
)
|
|
118
|
+
if result["box_residual_px"] > BOX_TOL_PX:
|
|
119
|
+
return False, (
|
|
120
|
+
f"box residual {result['box_residual_px']:.4g} px exceeds {BOX_TOL_PX} "
|
|
121
|
+
f"under layout {result['layout']} (scale x{result['box_scale_applied']})"
|
|
122
|
+
)
|
|
123
|
+
if result["cls_residual"] > CLS_TOL:
|
|
124
|
+
return False, f"class residual {result['cls_residual']:.4g} exceeds {CLS_TOL}"
|
|
125
|
+
return True, "ok"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def main() -> int:
|
|
129
|
+
ap = argparse.ArgumentParser()
|
|
130
|
+
ap.add_argument("--onnx", required=True)
|
|
131
|
+
ap.add_argument("--litert", required=True)
|
|
132
|
+
ap.add_argument("--image", help="optional JPEG; random input is used when omitted")
|
|
133
|
+
ap.add_argument("--seed", type=int, default=1234)
|
|
134
|
+
ap.add_argument("--json", action="store_true", help="emit the measurement as JSON")
|
|
135
|
+
args = ap.parse_args()
|
|
136
|
+
|
|
137
|
+
import onnxruntime as ort
|
|
138
|
+
from ai_edge_litert.interpreter import Interpreter
|
|
139
|
+
|
|
140
|
+
interp = Interpreter(model_path=args.litert, num_threads=4)
|
|
141
|
+
interp.allocate_tensors()
|
|
142
|
+
in_detail = interp.get_input_details()[0]
|
|
143
|
+
out_details = interp.get_output_details()
|
|
144
|
+
shape = tuple(int(v) for v in in_detail["shape"])
|
|
145
|
+
|
|
146
|
+
# Both runtimes take NCHW here — the LiteRT export of an ultralytics detect
|
|
147
|
+
# model is NOT NHWC. Transposing "because LiteRT is channels-last" silently
|
|
148
|
+
# feeds a rotated image and every residual then looks like export noise.
|
|
149
|
+
if len(shape) != 4 or shape[1] != 3:
|
|
150
|
+
raise ParityError(f"expected an NCHW [1,3,H,W] input, got {shape}")
|
|
151
|
+
imgsz = shape[2]
|
|
152
|
+
|
|
153
|
+
if args.image:
|
|
154
|
+
from PIL import Image
|
|
155
|
+
|
|
156
|
+
pixels = np.asarray(Image.open(args.image).convert("RGB").resize((imgsz, imgsz)))
|
|
157
|
+
x = (pixels.astype(np.float32) / 255.0).transpose(2, 0, 1)[None]
|
|
158
|
+
else:
|
|
159
|
+
x = np.random.default_rng(args.seed).random(shape).astype(np.float32)
|
|
160
|
+
|
|
161
|
+
interp.set_tensor(in_detail["index"], x)
|
|
162
|
+
interp.invoke()
|
|
163
|
+
litert_out = interp.get_tensor(out_details[0]["index"])
|
|
164
|
+
|
|
165
|
+
sess = ort.InferenceSession(args.onnx, providers=["CPUExecutionProvider"])
|
|
166
|
+
onnx_names = [o.name for o in sess.get_outputs()]
|
|
167
|
+
onnx_out = sess.run(None, {sess.get_inputs()[0].name: x})[0]
|
|
168
|
+
|
|
169
|
+
layout = classify_layout(tuple(int(v) for v in onnx_out.shape), onnx_names)
|
|
170
|
+
result = compare(onnx_out, litert_out, layout, imgsz)
|
|
171
|
+
ok, why = verdict(result)
|
|
172
|
+
result["pass"] = ok
|
|
173
|
+
result["reason"] = why
|
|
174
|
+
result["onnx"] = Path(args.onnx).name
|
|
175
|
+
result["litert"] = Path(args.litert).name
|
|
176
|
+
|
|
177
|
+
if args.json:
|
|
178
|
+
print(json.dumps(result, indent=2))
|
|
179
|
+
else:
|
|
180
|
+
print(f"{'PASS' if ok else 'FAIL'} {result['onnx']} layout={layout} {why}")
|
|
181
|
+
print(f" box x{result['box_scale_applied']:g}: {result['box_residual_px']:.4g} px"
|
|
182
|
+
f" (unscaled {result['box_residual_unscaled']:.4g})")
|
|
183
|
+
print(f" cls : {result['cls_residual']:.4g}")
|
|
184
|
+
return 0 if ok else 1
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
if __name__ == "__main__":
|
|
188
|
+
sys.exit(main())
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""A coordinate space is MEASURED or REFUSED. It is never inferred from silence.
|
|
2
|
+
|
|
3
|
+
`session.py` takes the layout and the box units from the artifact's DECLARED
|
|
4
|
+
variant contract and never re-derives them from the tensor — that is deliberate,
|
|
5
|
+
because `[1, 300, 6]` and `[1, 6, 8400]` are both rank-3 with a 6 in them and a
|
|
6
|
+
reader that decides from shape alone gets one of them backwards with no
|
|
7
|
+
exception and a plausible table. The whole weight of that design sits on the
|
|
8
|
+
declaration being true.
|
|
9
|
+
|
|
10
|
+
`classify_box_units` is where the declaration is made, from one forward pass.
|
|
11
|
+
Its refusal band exists because the wrong pick is silent: LiteRT boxes in 0..1
|
|
12
|
+
and ONNX boxes in pixels agree on output shape, dtype, class count and every
|
|
13
|
+
score, and differ only by a factor of imgsz on four channels.
|
|
14
|
+
|
|
15
|
+
The hole these tests close: the guard was `box_abs_max is None`, so a
|
|
16
|
+
`box_abs_max` of exactly `0.0` fell through to `ratio = 0.0`, landed under
|
|
17
|
+
BOX_UNITS_NORMALIZED_RATIO_MAX, and returned a CONFIDENT `normalized` with
|
|
18
|
+
`scale_to_pixels=640` from zero observations. That is not hypothetical: a model
|
|
19
|
+
whose classification head is dead still emits box rows — `forklift_akdve_11n`
|
|
20
|
+
has healthy boxes (2.93-636 px) and a max class score of 0.0063 — so nothing is
|
|
21
|
+
admitted at threshold and `box_abs_max` never moves off its seed. The dead
|
|
22
|
+
artifact then published a coordinate-space claim no frame supports.
|
|
23
|
+
|
|
24
|
+
STDLIB ONLY, and that is why `box_units.py` is a separate module: the
|
|
25
|
+
`detector-service` CI lane runs `python3 -m unittest discover` with no pip
|
|
26
|
+
install, and `run_candidate.py` imports onnxruntime at module scope, so a test
|
|
27
|
+
importing it would error the whole lane during discovery rather than skip.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import pathlib
|
|
33
|
+
import sys
|
|
34
|
+
import unittest
|
|
35
|
+
|
|
36
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "bench"))
|
|
37
|
+
|
|
38
|
+
from box_units import ( # noqa: E402
|
|
39
|
+
BOX_UNITS_NORMALIZED_RATIO_MAX,
|
|
40
|
+
BOX_UNITS_PIXEL_RATIO_MIN,
|
|
41
|
+
classify_box_units,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
IMGSZ = 640
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class ZeroEvidenceIsRefused(unittest.TestCase):
|
|
48
|
+
def test_exactly_zero_refuses(self):
|
|
49
|
+
"""The seed value must not read as a measurement."""
|
|
50
|
+
r = classify_box_units(0.0, IMGSZ)
|
|
51
|
+
self.assertIsNone(r["units"], "0.0 observed box extent was classified as a unit system")
|
|
52
|
+
self.assertIsNone(r["scale_to_pixels"])
|
|
53
|
+
self.assertIn("unmeasured", r["refusal"])
|
|
54
|
+
|
|
55
|
+
def test_negative_refuses(self):
|
|
56
|
+
"""A negative max is impossible from `abs()`, so it means a bug upstream."""
|
|
57
|
+
self.assertIsNone(classify_box_units(-1.0, IMGSZ)["units"])
|
|
58
|
+
|
|
59
|
+
def test_none_still_refuses(self):
|
|
60
|
+
r = classify_box_units(None, IMGSZ)
|
|
61
|
+
self.assertIsNone(r["units"])
|
|
62
|
+
self.assertIn("unmeasured", r["refusal"])
|
|
63
|
+
|
|
64
|
+
def test_zero_imgsz_refuses(self):
|
|
65
|
+
self.assertIsNone(classify_box_units(639.9, 0)["units"])
|
|
66
|
+
|
|
67
|
+
def test_the_refusal_is_reachable_at_all(self):
|
|
68
|
+
"""Guards the shape of the bug, not just this instance.
|
|
69
|
+
|
|
70
|
+
The refusal string named this exact case and could not be reached. A
|
|
71
|
+
message describing a state the code cannot produce is not a safeguard.
|
|
72
|
+
"""
|
|
73
|
+
reasons = {
|
|
74
|
+
classify_box_units(v, IMGSZ)["refusal"]
|
|
75
|
+
for v in (None, 0.0, -0.0, -1.0)
|
|
76
|
+
}
|
|
77
|
+
self.assertEqual(len(reasons), 1, "every zero-evidence input must give the same reason")
|
|
78
|
+
self.assertTrue(all(r for r in reasons))
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class RealMeasurementsStillClassify(unittest.TestCase):
|
|
82
|
+
"""The fix must not widen the refusal into the measured bands."""
|
|
83
|
+
|
|
84
|
+
def test_a_normalised_head_is_still_normalised(self):
|
|
85
|
+
# 1/640 = 0.0016 — a normalised head's ratio, from the runbook.
|
|
86
|
+
r = classify_box_units(1.0, IMGSZ)
|
|
87
|
+
self.assertEqual(r["units"], "normalized")
|
|
88
|
+
self.assertEqual(r["scale_to_pixels"], float(IMGSZ))
|
|
89
|
+
self.assertIsNone(r["refusal"])
|
|
90
|
+
|
|
91
|
+
def test_smallest_credible_normalised_value_survives(self):
|
|
92
|
+
"""The tightest case the fix could have swallowed.
|
|
93
|
+
|
|
94
|
+
A normalised box of 1e-6 of the frame is absurd but non-zero, and the
|
|
95
|
+
distinction the fix draws is between zero and non-zero, not between
|
|
96
|
+
small and large.
|
|
97
|
+
"""
|
|
98
|
+
r = classify_box_units(1e-6, IMGSZ)
|
|
99
|
+
self.assertEqual(r["units"], "normalized")
|
|
100
|
+
|
|
101
|
+
def test_a_pixel_head_is_still_pixels(self):
|
|
102
|
+
r = classify_box_units(639.29, IMGSZ) # the shipped rfdetr_medium score card
|
|
103
|
+
self.assertEqual(r["units"], "pixels")
|
|
104
|
+
self.assertEqual(r["scale_to_pixels"], 1.0)
|
|
105
|
+
|
|
106
|
+
def test_the_dead_zone_still_refuses_with_its_ratio(self):
|
|
107
|
+
mid = (BOX_UNITS_NORMALIZED_RATIO_MAX + BOX_UNITS_PIXEL_RATIO_MIN) / 2
|
|
108
|
+
r = classify_box_units(mid * IMGSZ, IMGSZ)
|
|
109
|
+
self.assertIsNone(r["units"])
|
|
110
|
+
self.assertAlmostEqual(r["ratio"], mid, places=6)
|
|
111
|
+
self.assertIn("falls between", r["refusal"])
|
|
112
|
+
|
|
113
|
+
def test_band_edges_are_inclusive_as_documented(self):
|
|
114
|
+
self.assertEqual(classify_box_units(BOX_UNITS_PIXEL_RATIO_MIN * IMGSZ, IMGSZ)["units"], "pixels")
|
|
115
|
+
self.assertEqual(classify_box_units(BOX_UNITS_NORMALIZED_RATIO_MAX * IMGSZ, IMGSZ)["units"], "normalized")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
if __name__ == "__main__":
|
|
119
|
+
unittest.main()
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""The committed twin-parity measurement must stay a MEASUREMENT.
|
|
2
|
+
|
|
3
|
+
`bench/litert-twin-parity.json` is the only place the box-scale-per-layout fact
|
|
4
|
+
survives into CI: the verifier that produced it needs both runtimes and ~1 GB of
|
|
5
|
+
wheels, so it is a documented manual run. That makes the JSON the artifact a
|
|
6
|
+
future edit can quietly falsify — flip one `pass` to true, drop the failing row,
|
|
7
|
+
or widen a tolerance, and nothing downstream notices.
|
|
8
|
+
|
|
9
|
+
STDLIB ONLY, and unittest rather than pytest, because the `detector-service`
|
|
10
|
+
step in `ci.yml` runs `python3 -m unittest discover` with no pip install. An
|
|
11
|
+
earlier draft of this file imported the verifier (and therefore numpy) at module
|
|
12
|
+
scope and would have errored the whole lane before its existing tests ran, while
|
|
13
|
+
also contributing no collected cases of its own — a check that appears to run
|
|
14
|
+
and does not, which is precisely the defect the measurement below is about.
|
|
15
|
+
|
|
16
|
+
These are pure-logic checks over committed bytes. They do not re-measure; they
|
|
17
|
+
assert the recorded numbers are internally consistent with the code that claims
|
|
18
|
+
to have produced them, which is the part a hand-edit breaks.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import json
|
|
24
|
+
import sys
|
|
25
|
+
import unittest
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
|
|
28
|
+
BENCH = Path(__file__).resolve().parent.parent / "bench"
|
|
29
|
+
CONTRACT = BENCH / "litert-twin-parity.json"
|
|
30
|
+
|
|
31
|
+
sys.path.insert(0, str(BENCH))
|
|
32
|
+
|
|
33
|
+
from twin_parity_contract import ( # noqa: E402
|
|
34
|
+
ANCHOR,
|
|
35
|
+
BOX_TOL_PX,
|
|
36
|
+
CLS_TOL,
|
|
37
|
+
COMPARABLE_LAYOUTS,
|
|
38
|
+
DETS_LABELS,
|
|
39
|
+
END2END,
|
|
40
|
+
ParityError,
|
|
41
|
+
box_scale_for,
|
|
42
|
+
classify_layout,
|
|
43
|
+
verdict,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
#: The artifact whose row-level parity could NOT be established, named so the
|
|
47
|
+
#: assertion below cannot pass by the row being deleted. See the test.
|
|
48
|
+
UNVERIFIED_END2END_ARTIFACT = "queue_supermarket.onnx"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class TwinParityContract(unittest.TestCase):
|
|
52
|
+
@classmethod
|
|
53
|
+
def setUpClass(cls) -> None:
|
|
54
|
+
cls.contract = json.loads(CONTRACT.read_text())
|
|
55
|
+
cls.twins = cls.contract["twins"]
|
|
56
|
+
|
|
57
|
+
def test_tolerances_match_the_verifier(self):
|
|
58
|
+
"""A widened tolerance in the JSON that the code does not share is a lie."""
|
|
59
|
+
self.assertEqual(self.contract["tolerances"]["box_residual_px"], BOX_TOL_PX)
|
|
60
|
+
self.assertEqual(self.contract["tolerances"]["cls_residual"], CLS_TOL)
|
|
61
|
+
|
|
62
|
+
def test_every_row_used_the_scale_its_layout_dictates(self):
|
|
63
|
+
for row in self.twins:
|
|
64
|
+
with self.subTest(onnx=row["onnx"]):
|
|
65
|
+
expected = box_scale_for(row["layout"], row["imgsz"])
|
|
66
|
+
self.assertEqual(
|
|
67
|
+
row["box_scale_applied"], expected,
|
|
68
|
+
f"{row['onnx']} recorded scale x{row['box_scale_applied']} but layout "
|
|
69
|
+
f"{row['layout']} dictates x{expected}",
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
def test_recorded_verdicts_follow_from_recorded_residuals(self):
|
|
73
|
+
"""`pass` must be derivable from the numbers beside it, not asserted."""
|
|
74
|
+
for row in self.twins:
|
|
75
|
+
with self.subTest(onnx=row["onnx"]):
|
|
76
|
+
ok, why = verdict(row)
|
|
77
|
+
self.assertEqual(
|
|
78
|
+
ok, row["pass"],
|
|
79
|
+
f"{row['onnx']}: recorded pass={row['pass']} but verdict says {ok} ({why})",
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
def test_the_anchor_scale_is_load_bearing_on_every_anchor_row(self):
|
|
83
|
+
"""An anchor twin whose RAW boxes already match is not normalised.
|
|
84
|
+
|
|
85
|
+
This is the 640x bug's signature. If it ever holds, the row passed for
|
|
86
|
+
the wrong reason and the layout was misread.
|
|
87
|
+
"""
|
|
88
|
+
anchors = [r for r in self.twins if r["layout"] == ANCHOR]
|
|
89
|
+
self.assertTrue(anchors, "the contract has no anchor-layout rows to check")
|
|
90
|
+
for row in anchors:
|
|
91
|
+
with self.subTest(onnx=row["onnx"]):
|
|
92
|
+
self.assertGreater(
|
|
93
|
+
row["box_residual_unscaled"], BOX_TOL_PX,
|
|
94
|
+
f"{row['onnx']}: unscaled residual {row['box_residual_unscaled']} is already "
|
|
95
|
+
"inside tolerance, so multiplying by imgsz changed nothing",
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
def test_the_unverified_end2end_row_is_present_and_still_failing(self):
|
|
99
|
+
"""We do NOT get to publish a twin we could not compare.
|
|
100
|
+
|
|
101
|
+
`queue_supermarket` returns zero detections above 0.25 on 40 real
|
|
102
|
+
production frames, so its row-level parity is unknown and it is recorded
|
|
103
|
+
failing on purpose.
|
|
104
|
+
|
|
105
|
+
The PRESENCE assertion is the load-bearing half. Checking only that
|
|
106
|
+
every `end2end_n_6` row still fails passes vacuously the moment somebody
|
|
107
|
+
deletes the row — which is exactly the hand-edit this file exists to
|
|
108
|
+
prevent, and the same "absence read as success" shape as an empty corpus
|
|
109
|
+
scoring `overallPass: true`. So the row is named.
|
|
110
|
+
"""
|
|
111
|
+
by_name = {r["onnx"]: r for r in self.twins}
|
|
112
|
+
self.assertIn(
|
|
113
|
+
UNVERIFIED_END2END_ARTIFACT, by_name,
|
|
114
|
+
f"{UNVERIFIED_END2END_ARTIFACT} is missing from the contract. It is the only "
|
|
115
|
+
f"{END2END} twin and its parity is unverified; removing the row does not make "
|
|
116
|
+
"it verified. If it was genuinely re-measured on a corpus it fires on, update "
|
|
117
|
+
"this test with that corpus named.",
|
|
118
|
+
)
|
|
119
|
+
row = by_name[UNVERIFIED_END2END_ARTIFACT]
|
|
120
|
+
self.assertEqual(row["layout"], END2END)
|
|
121
|
+
self.assertFalse(
|
|
122
|
+
row["pass"],
|
|
123
|
+
f"{UNVERIFIED_END2END_ARTIFACT} is recorded passing — see the message above",
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
def test_layout_is_classified_from_the_graph_not_the_values(self):
|
|
127
|
+
"""Shape and output names decide the layout. Nothing else may.
|
|
128
|
+
|
|
129
|
+
Dispatching on `shape[1] >= 5` alone routes a [1, 300, 6] end2end tensor
|
|
130
|
+
into the anchor branch and produces a confident, meaningless class
|
|
131
|
+
residual. A layout guess that reads the numbers it is interpreting
|
|
132
|
+
always finds a self-consistent story.
|
|
133
|
+
"""
|
|
134
|
+
self.assertEqual(classify_layout((1, 300, 6), ["output0"]), END2END)
|
|
135
|
+
self.assertEqual(classify_layout((1, 14, 8400), ["output0"]), ANCHOR)
|
|
136
|
+
self.assertEqual(classify_layout((1, 6, 8400), ["output0"]), ANCHOR)
|
|
137
|
+
self.assertEqual(classify_layout((1, 300, 4), ["dets", "labels"]), DETS_LABELS)
|
|
138
|
+
with self.assertRaises(ParityError):
|
|
139
|
+
classify_layout((1, 8400), ["output0"])
|
|
140
|
+
|
|
141
|
+
def test_no_layout_gets_a_scale_by_default(self):
|
|
142
|
+
"""An unknown layout must refuse, never fall through to 1.0.
|
|
143
|
+
|
|
144
|
+
A silent wrong-layout decode is worse than a refusal: it returns boxes,
|
|
145
|
+
and boxes that are plausible and wrong are what this file is about.
|
|
146
|
+
"""
|
|
147
|
+
with self.assertRaises(ParityError):
|
|
148
|
+
box_scale_for("some_future_layout", 640)
|
|
149
|
+
|
|
150
|
+
def test_rfdetr_is_classified_but_not_advertised_as_comparable(self):
|
|
151
|
+
"""Naming a layout is not the same as being able to compare it.
|
|
152
|
+
|
|
153
|
+
`classify_layout` recognises RF-DETR's two-output form and
|
|
154
|
+
`box_scale_for` knows its scale is 1.0, but this bench cannot split a
|
|
155
|
+
two-output graph into box and class halves, and RF-DETR ships no LiteRT
|
|
156
|
+
twin. So it must be absent from the comparable set — otherwise the
|
|
157
|
+
verifier advertises a capability that fails at the point of use.
|
|
158
|
+
"""
|
|
159
|
+
self.assertNotIn(DETS_LABELS, COMPARABLE_LAYOUTS)
|
|
160
|
+
self.assertEqual(COMPARABLE_LAYOUTS, frozenset({ANCHOR, END2END}))
|
|
161
|
+
self.assertEqual(box_scale_for(DETS_LABELS, 640), 1.0)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
if __name__ == "__main__":
|
|
165
|
+
unittest.main()
|
package/dist/EdgeAgent.js
CHANGED
|
@@ -44,7 +44,7 @@ const edge_ads_1 = require("@trillboards/edge-ads");
|
|
|
44
44
|
const CommandRouter_1 = require("./CommandRouter");
|
|
45
45
|
const config_1 = require("./config");
|
|
46
46
|
const dataDir_1 = require("./dataDir");
|
|
47
|
-
const SDK_VERSION = '0.2.
|
|
47
|
+
const SDK_VERSION = '0.2.93';
|
|
48
48
|
class EdgeAgent extends events_1.EventEmitter {
|
|
49
49
|
constructor(config, deps) {
|
|
50
50
|
super();
|
package/dist/diagnose.d.ts
CHANGED
|
@@ -31,7 +31,7 @@ export interface DiagnosticReport {
|
|
|
31
31
|
skipped: number;
|
|
32
32
|
};
|
|
33
33
|
}
|
|
34
|
-
declare const SDK_VERSION = "0.2.
|
|
34
|
+
declare const SDK_VERSION = "0.2.93";
|
|
35
35
|
declare function checkNodeVersion(): CheckResult;
|
|
36
36
|
declare function checkPlatform(): CheckResult;
|
|
37
37
|
declare function checkMemory(): CheckResult;
|
package/dist/diagnose.js
CHANGED
|
@@ -63,7 +63,7 @@ const localNetworkAccess_1 = require("./localNetworkAccess");
|
|
|
63
63
|
// ---------------------------------------------------------------------------
|
|
64
64
|
// SDK version — mirrors package.json; kept in sync by CI
|
|
65
65
|
// ---------------------------------------------------------------------------
|
|
66
|
-
const SDK_VERSION = '0.2.
|
|
66
|
+
const SDK_VERSION = '0.2.93';
|
|
67
67
|
exports._SDK_VERSION = SDK_VERSION;
|
|
68
68
|
// ---------------------------------------------------------------------------
|
|
69
69
|
// ANSI helpers (no external deps)
|