@trillboards/edge-sdk 0.2.110 → 0.2.113
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/detector/bench/litert-twin-parity.json +25 -0
- package/deploy/detector/bench/regrade_control_selection.py +291 -28
- package/deploy/detector/bench/twin_parity_contract.py +82 -13
- package/deploy/detector/bench/venue_adjudication.py +101 -8
- package/deploy/detector/bench/verify_litert_twin_parity.py +22 -18
- package/deploy/detector/tests/test_box_units_are_measured.py +137 -0
- package/deploy/detector/tests/test_regrade_control_selection.py +108 -0
- package/deploy/detector/trillboards_detector/labels.py +24 -1
- package/deploy/macos/TrillboardsSetupApp.m +135 -0
- package/deploy/windows/trillboards-setup.ps1 +166 -0
- package/dist/AudienceAnalyzeClient.d.ts +11 -0
- package/dist/AudienceAnalyzeClient.d.ts.map +1 -1
- package/dist/AudienceAnalyzeClient.js +7 -0
- package/dist/AudienceAnalyzeClient.js.map +1 -1
- package/dist/ChannelProber.d.ts.map +1 -1
- package/dist/ChannelProber.js +15 -1
- package/dist/ChannelProber.js.map +1 -1
- package/dist/CloudSensingManager.d.ts +20 -0
- package/dist/CloudSensingManager.d.ts.map +1 -1
- package/dist/CloudSensingManager.js +30 -0
- package/dist/CloudSensingManager.js.map +1 -1
- package/dist/EdgeAgent.js +1 -1
- package/dist/SensingProfileClient.d.ts +10 -0
- package/dist/SensingProfileClient.d.ts.map +1 -1
- package/dist/SensingProfileClient.js +16 -0
- package/dist/SensingProfileClient.js.map +1 -1
- package/dist/diagnose.d.ts +1 -1
- package/dist/diagnose.js +1 -1
- package/dist/discovery.d.ts +1 -1
- package/dist/discovery.d.ts.map +1 -1
- package/dist/discovery.js +15 -4
- package/dist/discovery.js.map +1 -1
- package/dist/localSensing.d.ts +70 -0
- package/dist/localSensing.d.ts.map +1 -1
- package/dist/localSensing.js +182 -1
- package/dist/localSensing.js.map +1 -1
- package/package.json +1 -1
|
@@ -334,6 +334,31 @@
|
|
|
334
334
|
"litert_input_layout": "nhwc",
|
|
335
335
|
"measured_at": "2026-08-26",
|
|
336
336
|
"note": "The 22nd twin, added 2026-08-26, and the FIRST whose LiteRT input is NHWC. gh_fall_tonlong had no parity measurement at all until now: upstream ships `Violence` at slot 0 and `violence` at slot 6, normalize_class_name folds them to one name and ModelVocabulary refuses at construction, so neither format could be probed. Both were re-exported from ok.pt (sha256 1e12d89198e28d7c650ba86886c511fecd9f70b9d7f8bf985e76704690e58dd9) with the slots disambiguated by INDEX -- violence_slot0 / violence_slot6 -- which preserves upstream truth and asserts no semantics we have not measured."
|
|
337
|
+
},
|
|
338
|
+
{
|
|
339
|
+
"layout": "anchor_4_plus_nc",
|
|
340
|
+
"imgsz": 640,
|
|
341
|
+
"box_abs_max": 1.052759051322937,
|
|
342
|
+
"units": "normalized",
|
|
343
|
+
"box_scale_applied": 640.0,
|
|
344
|
+
"box_residual_px": 0.001220703125,
|
|
345
|
+
"box_residual_unscaled": 672.7130490541458,
|
|
346
|
+
"cls_residual": 1.3779020591186963e-07,
|
|
347
|
+
"pass": true,
|
|
348
|
+
"reason": "ok",
|
|
349
|
+
"onnx": "fire_smoke_weslleyskah_11n.onnx",
|
|
350
|
+
"litert": "fire_smoke_weslleyskah_11n.tflite",
|
|
351
|
+
"model_id": "fire_smoke_weslleyskah_11n",
|
|
352
|
+
"box_units": "normalized",
|
|
353
|
+
"box_scale_to_pixels": 640.0,
|
|
354
|
+
"litert_input_layout": "nhwc",
|
|
355
|
+
"litert_sha256": "1511903fece4e163348105cfddb89c657ccc43f1482cde7b090e3f1f7b003c84",
|
|
356
|
+
"litert_bytes": 9809490,
|
|
357
|
+
"onnx_sha256": "af8b648caf175edb928eeb3dc6ef73dffc55feaca4097f8133809d8d0ac16419",
|
|
358
|
+
"onnx_bytes": 10216160,
|
|
359
|
+
"measured_at": "2026-08-27",
|
|
360
|
+
"evidence": "twin parity: box residual 0.001221 px after x640, UNSCALED 672.7 px \u2014 the scale is load-bearing, not cosmetic. Class residual 1.38e-07 with no scaling at all. The ONNX half emits PIXELS (max|box| 945.30 over 1,670 real frames, ratio 1.4770) and the LiteRT half NORMALIZED (box_abs_max 1.0528), which is the pair this contract exists for.",
|
|
361
|
+
"note": "Exported from weslleyskah/fire_smoke_detection_box weights/best.pt (sha256 d2b0be21...) with ultralytics 8.3.201, imgsz 640, opset 19, nms=False. `units` and `box_abs_max` are DERIVED from the LiteRT artifact's own forward pass, not assumed from the layout. Verified twice \u2014 once under the contract at production HEAD and once under PR #13224's derived-units contract \u2014 byte-identical numbers and `pass` under both, because this twin IS normalised, the case the hardcoded x640 already handled. The row does not depend on that PR."
|
|
337
362
|
}
|
|
338
363
|
]
|
|
339
364
|
}
|
|
@@ -37,6 +37,7 @@ from __future__ import annotations
|
|
|
37
37
|
import argparse
|
|
38
38
|
import json
|
|
39
39
|
import os
|
|
40
|
+
import time
|
|
40
41
|
|
|
41
42
|
#: Grades whose own note says the set cannot rate a class. Kept in step with
|
|
42
43
|
#: `venue_adjudication.py::LIVENESS_ONLY_GRADES`; the test pins them equal.
|
|
@@ -67,7 +68,124 @@ def rateable_controls(per_control: dict, ctlq: dict) -> dict:
|
|
|
67
68
|
return out
|
|
68
69
|
|
|
69
70
|
|
|
70
|
-
|
|
71
|
+
#: Schemas this tool will read a NEW control rate out of. A rescore artifact is
|
|
72
|
+
#: the ONLY way a control the original screen never scored can reach a verdict
|
|
73
|
+
#: here, and it is admissible for exactly one reason: it was produced by
|
|
74
|
+
#: `venue_adjudication.py`'s OWN rule -- same GRID, same QUIET_MAX_FIRE, same
|
|
75
|
+
#: worst-valid-bucket quiet_t, same bar -- against a control that carries a
|
|
76
|
+
#: contact-sheet grade. Anything else would be an assertion wearing a filename.
|
|
77
|
+
RESCORE_SCHEMAS = ("trillboards.fire-smoke-rescore/1",)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def load_rescores(paths) -> dict:
|
|
81
|
+
"""Committed rescore artifacts, keyed by (model, class).
|
|
82
|
+
|
|
83
|
+
Returns {(model, class): row}. A file whose schema is not in
|
|
84
|
+
`RESCORE_SCHEMAS` is REFUSED rather than skipped: a rescore this tool cannot
|
|
85
|
+
read is a number somebody expected to count, and silently ignoring it is how
|
|
86
|
+
a verdict stays stale while looking re-derived.
|
|
87
|
+
"""
|
|
88
|
+
out = {}
|
|
89
|
+
for path in paths or []:
|
|
90
|
+
doc = json.load(open(path))
|
|
91
|
+
if doc.get("schema") not in RESCORE_SCHEMAS:
|
|
92
|
+
raise SystemExit(
|
|
93
|
+
"%s has schema %r, which this tool cannot read (known: %s). Refusing rather "
|
|
94
|
+
"than skipping -- a rescore that is silently ignored leaves the verdict stale "
|
|
95
|
+
"while the run reports success."
|
|
96
|
+
% (path, doc.get("schema"), ", ".join(RESCORE_SCHEMAS)))
|
|
97
|
+
model = (doc.get("artifact") or {}).get("model_id")
|
|
98
|
+
if not model:
|
|
99
|
+
raise SystemExit("%s names no artifact.model_id, so its rates belong to nobody" % path)
|
|
100
|
+
# A rescore may carry caveats that survive its own verdict -- "this clears
|
|
101
|
+
# the bar AND no live route reaches it" is one fact, not two, and the
|
|
102
|
+
# record of record is where a reader decides whether to SELL. Carried
|
|
103
|
+
# rather than restated so there is one home for the measurement.
|
|
104
|
+
caveats = doc.get("sellability_caveats") or []
|
|
105
|
+
for cls, e in (doc.get("per_class") or {}).items():
|
|
106
|
+
out[(model, cls)] = {
|
|
107
|
+
"control": e["control_set"],
|
|
108
|
+
"n": e["control_frames"],
|
|
109
|
+
"rate_025": e["control_rate_025"],
|
|
110
|
+
"rate_at_quiet_t": e["control_rate_at_quiet_t"],
|
|
111
|
+
"quiet_t": e["quiet_t"],
|
|
112
|
+
"source_file": os.path.basename(path),
|
|
113
|
+
"sellability_caveats": caveats,
|
|
114
|
+
}
|
|
115
|
+
return out
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def apply_rescore(cand: dict, rescores: dict, ctlq: dict):
|
|
119
|
+
"""Merge a rescored control into one candidate's `per_control`.
|
|
120
|
+
|
|
121
|
+
Returns (candidate, note). The candidate is a COPY when anything changed.
|
|
122
|
+
|
|
123
|
+
THE OFF-TARGET BOUND IS THE WHOLE CARE HERE. `decide()`'s third gate is
|
|
124
|
+
`off_target_mean <= bar`, and a rescore does not recompute it -- the
|
|
125
|
+
off-target control sets the original screen fired against are gone. What it
|
|
126
|
+
can do is INHERIT the committed value, and only in the direction where that
|
|
127
|
+
is sound: `off_target` is `(scores >= quiet_t).mean()`, which is monotone
|
|
128
|
+
NON-INCREASING in the threshold, so a value measured at a LOWER quiet_t is a
|
|
129
|
+
valid UPPER BOUND at a higher one. If the rescore's threshold is BELOW the
|
|
130
|
+
committed one the bound runs the wrong way, and the row is refused rather
|
|
131
|
+
than passed on a number nobody measured at that operating point.
|
|
132
|
+
"""
|
|
133
|
+
key = (cand.get("model"), cand.get("class"))
|
|
134
|
+
r = (rescores or {}).get(key)
|
|
135
|
+
if not r:
|
|
136
|
+
return cand, None
|
|
137
|
+
grade = (ctlq.get(r["control"]) or {}).get("grade")
|
|
138
|
+
if grade is None:
|
|
139
|
+
raise SystemExit(
|
|
140
|
+
"rescore %s supplies control %r for %s/%s, which has no contact-sheet grade. "
|
|
141
|
+
"An ungraded set is neither `unusable` nor `liveness_only`, so it would be "
|
|
142
|
+
"eligible to carry a verdict on nobody's inspection."
|
|
143
|
+
% (r["source_file"], r["control"], key[0], key[1]))
|
|
144
|
+
committed_t = cand.get("quiet_t")
|
|
145
|
+
if committed_t is not None and r["quiet_t"] < committed_t:
|
|
146
|
+
return cand, {
|
|
147
|
+
"model": key[0], "class": key[1], "control": r["control"],
|
|
148
|
+
"refused": ("rescore quiet_t %s is BELOW the committed %s, so the committed "
|
|
149
|
+
"off_target_mean is not an upper bound at the rescored operating "
|
|
150
|
+
"point and no off-target number exists for it"
|
|
151
|
+
% (r["quiet_t"], committed_t)),
|
|
152
|
+
"source_file": r["source_file"],
|
|
153
|
+
}
|
|
154
|
+
merged = dict(cand)
|
|
155
|
+
# A RATE BELONGS TO THE THRESHOLD IT WAS MEASURED AT. Raising the candidate's
|
|
156
|
+
# `quiet_t` does not re-measure the controls already in `per_control` -- their
|
|
157
|
+
# `rate_at_quiet_t` is still the value at the COMMITTED threshold. Because
|
|
158
|
+
# `(scores >= t).mean()` is monotone non-increasing in t, that number is an
|
|
159
|
+
# UPPER BOUND at the higher threshold, so reading it as the recall at the new
|
|
160
|
+
# operating point OVERSTATES recall and can manufacture a PUBLISH. Every entry
|
|
161
|
+
# is therefore stamped with the threshold that produced it, and selection
|
|
162
|
+
# admits only the ones measured at the threshold actually applied.
|
|
163
|
+
merged["per_control"] = {
|
|
164
|
+
name: dict(row, measured_at_t=row.get("measured_at_t", committed_t))
|
|
165
|
+
for name, row in (cand.get("per_control") or {}).items()
|
|
166
|
+
}
|
|
167
|
+
merged["per_control"][r["control"]] = {
|
|
168
|
+
"n": r["n"], "rate_025": r["rate_025"], "rate_at_quiet_t": r["rate_at_quiet_t"],
|
|
169
|
+
"measured_at_t": r["quiet_t"],
|
|
170
|
+
}
|
|
171
|
+
merged["quiet_t"] = r["quiet_t"]
|
|
172
|
+
note = {
|
|
173
|
+
"model": key[0], "class": key[1], "control": r["control"], "grade": grade,
|
|
174
|
+
"n": r["n"], "rate_at_quiet_t": r["rate_at_quiet_t"],
|
|
175
|
+
"quiet_t_committed": committed_t, "quiet_t_rescored": r["quiet_t"],
|
|
176
|
+
"off_target_mean_inherited": cand.get("off_target_mean"),
|
|
177
|
+
"off_target_bound_holds_because": (
|
|
178
|
+
"off_target is (scores >= t).mean(), monotone non-increasing in t, and the "
|
|
179
|
+
"rescored t (%s) is >= the committed t (%s), so the committed value is an "
|
|
180
|
+
"upper bound at the rescored operating point"
|
|
181
|
+
% (r["quiet_t"], committed_t)),
|
|
182
|
+
"source_file": r["source_file"],
|
|
183
|
+
"sellability_caveats": r.get("sellability_caveats") or [],
|
|
184
|
+
}
|
|
185
|
+
return merged, note
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def regrade_capability(cap: dict, ctlq: dict, bar: dict, rescores: dict = None) -> dict:
|
|
71
189
|
"""Re-apply the bar to one capability using only rateable controls."""
|
|
72
190
|
recall_bar = bar["recall_retention"]
|
|
73
191
|
off_bar = bar["off_target_mean"]
|
|
@@ -88,8 +206,11 @@ def regrade_capability(cap: dict, ctlq: dict, bar: dict) -> dict:
|
|
|
88
206
|
"why": "resolved before any control is read: %s" % (prior,),
|
|
89
207
|
}
|
|
90
208
|
|
|
91
|
-
rows, unknown = [], []
|
|
209
|
+
rows, unknown, rescored, stale_t = [], [], [], []
|
|
92
210
|
for cand in cap["candidates"]:
|
|
211
|
+
cand, note = apply_rescore(cand, rescores, ctlq)
|
|
212
|
+
if note:
|
|
213
|
+
rescored.append(note)
|
|
93
214
|
elided = "_elided" in (cand.get("by_bucket") or {})
|
|
94
215
|
rate = rateable_controls(cand.get("per_control"), ctlq)
|
|
95
216
|
if not rate:
|
|
@@ -120,9 +241,39 @@ def regrade_capability(cap: dict, ctlq: dict, bar: dict) -> dict:
|
|
|
120
241
|
"elided": elided,
|
|
121
242
|
})
|
|
122
243
|
continue
|
|
244
|
+
# Only a control measured at the threshold this candidate is DECIDED at may
|
|
245
|
+
# carry the verdict. Absent a rescore nothing is stamped and every control
|
|
246
|
+
# is at the committed threshold, so this is a no-op on the untouched path.
|
|
247
|
+
applied_t = cand.get("quiet_t")
|
|
248
|
+
fresh = {k: v for k, v in rate.items()
|
|
249
|
+
if v.get("measured_at_t") is None or v["measured_at_t"] == applied_t}
|
|
250
|
+
stale = {k: v for k, v in rate.items() if k not in fresh}
|
|
251
|
+
for k, v in sorted(stale.items()):
|
|
252
|
+
stale_t.append({
|
|
253
|
+
"model": cand["model"], "class": cand["class"], "control": k,
|
|
254
|
+
"measured_at_t": v.get("measured_at_t"), "applied_t": applied_t,
|
|
255
|
+
"rate_at_measured_t": v.get("rate_at_quiet_t"),
|
|
256
|
+
"why_excluded": ("its rate is an upper bound at the applied threshold, "
|
|
257
|
+
"not a measurement of it, so it may not carry a verdict"),
|
|
258
|
+
})
|
|
259
|
+
if not fresh:
|
|
260
|
+
unknown.append({
|
|
261
|
+
"model": cand["model"], "class": cand["class"],
|
|
262
|
+
"decided_on": cand.get("best_control"),
|
|
263
|
+
"grade": (ctlq.get(cand.get("best_control")) or {}).get("grade"),
|
|
264
|
+
"why_unknown": ("a rescore moved quiet_t to %s and every rateable control "
|
|
265
|
+
"here was measured at a lower threshold (%s), where its rate "
|
|
266
|
+
"is only an upper bound -- no control has a rate AT the "
|
|
267
|
+
"applied operating point"
|
|
268
|
+
% (applied_t,
|
|
269
|
+
", ".join(sorted(str(v.get("measured_at_t"))
|
|
270
|
+
for v in stale.values())))),
|
|
271
|
+
"elided": elided,
|
|
272
|
+
})
|
|
273
|
+
continue
|
|
123
274
|
# argmax(rate_025) among the rateable, exactly as the fixed tool does.
|
|
124
|
-
name = max(
|
|
125
|
-
row =
|
|
275
|
+
name = max(fresh, key=lambda k: (fresh[k]["rate_025"] or 0))
|
|
276
|
+
row = fresh[name]
|
|
126
277
|
rows.append({
|
|
127
278
|
"model": cand["model"], "class": cand["class"],
|
|
128
279
|
"control": name, "control_n": row["n"], "grade": row["grade"],
|
|
@@ -130,8 +281,13 @@ def regrade_capability(cap: dict, ctlq: dict, bar: dict) -> dict:
|
|
|
130
281
|
"off_target_mean": cand.get("off_target_mean"),
|
|
131
282
|
"quiet_t": cand.get("quiet_t"),
|
|
132
283
|
"n_valid_negatives": cand.get("n_valid_negatives"),
|
|
133
|
-
"also_scored": {k:
|
|
134
|
-
"
|
|
284
|
+
"also_scored": {k: dict(
|
|
285
|
+
{"n": v["n"], "grade": (ctlq.get(k) or {}).get("grade"),
|
|
286
|
+
"rate_at_quiet_t": v["rate_at_quiet_t"]},
|
|
287
|
+
**({} if v.get("measured_at_t") in (None, applied_t) else {
|
|
288
|
+
"measured_at_t": v["measured_at_t"],
|
|
289
|
+
"upper_bound_only": ("measured at %s, not the applied %s"
|
|
290
|
+
% (v["measured_at_t"], applied_t))}))
|
|
135
291
|
for k, v in (cand.get("per_control") or {}).items() if k != name},
|
|
136
292
|
"source": "recomputed from committed per_control",
|
|
137
293
|
})
|
|
@@ -208,6 +364,10 @@ def regrade_capability(cap: dict, ctlq: dict, bar: dict) -> dict:
|
|
|
208
364
|
"winner": win,
|
|
209
365
|
"candidates_regraded": len(rows),
|
|
210
366
|
"candidates_unknown": blocking,
|
|
367
|
+
"rescored_controls": rescored,
|
|
368
|
+
# Never silent: a control dropped for being measured at another
|
|
369
|
+
# threshold is a stated exclusion, exactly like a withheld model.
|
|
370
|
+
"stale_threshold_controls": stale_t,
|
|
211
371
|
}
|
|
212
372
|
|
|
213
373
|
|
|
@@ -260,7 +420,35 @@ def _selftest() -> None:
|
|
|
260
420
|
measured = regrade_capability(cap("NO MODEL CLEARS BAR"), ctlq, bar)
|
|
261
421
|
assert measured["regraded_decision"] == "NO MODEL CLEARS BAR", measured["regraded_decision"]
|
|
262
422
|
|
|
263
|
-
|
|
423
|
+
# 3. A RESCORE SETTLES AN UNRESOLVED. This is the whole point of --rescore: a
|
|
424
|
+
# control added after the screen ran reaches the ladder, and a candidate
|
|
425
|
+
# that clears the bar on it turns "not known" into PUBLISH. The elided row
|
|
426
|
+
# is still unknown and still reported -- it just cannot block any more,
|
|
427
|
+
# which is this tool's own stated monotonicity ("an unknown can only ever
|
|
428
|
+
# ADD a passing candidate").
|
|
429
|
+
ctlq3 = dict(ctlq, fresh={"grade": "good"})
|
|
430
|
+
rescore = {("m1", "fire"): {"control": "fresh", "n": 72, "rate_025": 0.82,
|
|
431
|
+
"rate_at_quiet_t": 0.86, "quiet_t": 0.30,
|
|
432
|
+
"source_file": "rescore.json"}}
|
|
433
|
+
settled = regrade_capability(cap(UNRESOLVED), ctlq3, bar, rescore)
|
|
434
|
+
assert settled["regraded_decision"] == "PUBLISH", settled["regraded_decision"]
|
|
435
|
+
assert settled["winner"]["control"] == "fresh", settled["winner"]
|
|
436
|
+
assert settled["rescored_controls"], "the rescore must be recorded, not silently used"
|
|
437
|
+
assert settled["changed"] is True
|
|
438
|
+
|
|
439
|
+
# 4. A RESCORE AT A LOWER THRESHOLD IS REFUSED. `off_target_mean` is inherited
|
|
440
|
+
# from the committed row and is only an UPPER BOUND when the rescored
|
|
441
|
+
# threshold is >= the committed one. Below it, no off-target number exists
|
|
442
|
+
# at that operating point and passing the row would assert one.
|
|
443
|
+
low = {("m1", "fire"): {"control": "fresh", "n": 72, "rate_025": 0.82,
|
|
444
|
+
"rate_at_quiet_t": 0.86, "quiet_t": 0.10,
|
|
445
|
+
"source_file": "rescore.json"}}
|
|
446
|
+
refused = regrade_capability(cap(UNRESOLVED), ctlq3, bar, low)
|
|
447
|
+
assert refused["regraded_decision"] == UNRESOLVED, refused["regraded_decision"]
|
|
448
|
+
assert any("refused" in r for r in refused["rescored_controls"]), refused["rescored_controls"]
|
|
449
|
+
|
|
450
|
+
print("selftest OK: idempotent on UNRESOLVED; a measured failure is left alone; "
|
|
451
|
+
"a rescore settles an unknown; a rescore below the committed threshold is refused")
|
|
264
452
|
|
|
265
453
|
|
|
266
454
|
def main() -> None:
|
|
@@ -269,7 +457,24 @@ def main() -> None:
|
|
|
269
457
|
_selftest()
|
|
270
458
|
return
|
|
271
459
|
ap.add_argument("--adjudication", required=True)
|
|
272
|
-
ap.add_argument("--control-quality", required=True
|
|
460
|
+
ap.add_argument("--control-quality", required=True, nargs="+",
|
|
461
|
+
help="positive-control-quality-*.json. TAKES A LIST and merges them, later "
|
|
462
|
+
"files winning, exactly as venue_adjudication.py does. It read ONE "
|
|
463
|
+
"path until 2026-08-27, which is unsafe for the reason that file "
|
|
464
|
+
"states about itself: `liveness_only` and `unusable` are both built "
|
|
465
|
+
"by MEMBERSHIP, so a set whose grade lives in a file this run did not "
|
|
466
|
+
"read falls through both and is eligible to carry a verdict on "
|
|
467
|
+
"nobody's inspection. The grades live in DATED files because the "
|
|
468
|
+
"2026-08-25 record is pinned by the score cards that cite it, so a "
|
|
469
|
+
"set graded later lands in a new file rather than editing that one -- "
|
|
470
|
+
"which means a single-path reader was guaranteed to go stale.")
|
|
471
|
+
ap.add_argument("--rescore", nargs="+", default=None,
|
|
472
|
+
help="committed rescore artifact(s) supplying a control the ORIGINAL "
|
|
473
|
+
"screen never scored. Admissible only because they are produced by "
|
|
474
|
+
"venue_adjudication.py's own rule -- same GRID, same QUIET_MAX_FIRE, "
|
|
475
|
+
"same worst-valid-bucket quiet_t, same bar. Without this there is no "
|
|
476
|
+
"path for a control added after the screen ran, and a capability "
|
|
477
|
+
"whose verdict a later measurement settles stays UNRESOLVED for ever.")
|
|
273
478
|
ap.add_argument("--out", required=True)
|
|
274
479
|
ap.add_argument("--apply", action="store_true",
|
|
275
480
|
help="also write the corrected verdict back into --adjudication. The "
|
|
@@ -278,10 +483,15 @@ def main() -> None:
|
|
|
278
483
|
args = ap.parse_args()
|
|
279
484
|
|
|
280
485
|
adj = json.load(open(args.adjudication))
|
|
281
|
-
ctlq =
|
|
486
|
+
ctlq = {}
|
|
487
|
+
for _q in args.control_quality:
|
|
488
|
+
for _k, _v in json.load(open(_q)).items():
|
|
489
|
+
if isinstance(_v, dict) and isinstance(_v.get("grade"), str):
|
|
490
|
+
ctlq[_k] = _v
|
|
282
491
|
bar = adj["bar"]
|
|
283
492
|
|
|
284
|
-
|
|
493
|
+
rescores = load_rescores(args.rescore)
|
|
494
|
+
rows = [regrade_capability(c, ctlq, bar, rescores) for c in adj["capabilities"]]
|
|
285
495
|
changed = [r for r in rows if r["changed"]]
|
|
286
496
|
|
|
287
497
|
# The regrade artifact is a record OF A TRANSITION. Once `--apply` has run, the
|
|
@@ -313,6 +523,8 @@ def main() -> None:
|
|
|
313
523
|
"This re-reads the numbers the adjudication already committed. A "
|
|
314
524
|
"capability marked `needs_remeasure` has a candidate whose "
|
|
315
525
|
"rateable-control rate was elided and must come from a re-run.",
|
|
526
|
+
"control_quality_inputs": sorted(os.path.basename(x) for x in args.control_quality),
|
|
527
|
+
"rescore_inputs": sorted({os.path.basename(x) for x in (args.rescore or [])}),
|
|
316
528
|
"liveness_only_grades": list(LIVENESS_ONLY_GRADES),
|
|
317
529
|
"unusable_grades": list(UNUSABLE_GRADES),
|
|
318
530
|
"bar": bar,
|
|
@@ -327,34 +539,85 @@ def main() -> None:
|
|
|
327
539
|
}, open(args.out, "w"), indent=1)
|
|
328
540
|
|
|
329
541
|
if args.apply and changed:
|
|
542
|
+
# THE WRITER MUST HANDLE BOTH DIRECTIONS. It was built for the WITHDRAWAL
|
|
543
|
+
# only -- it hardcoded the date `2026-08-26`, always wrote `ships: False`,
|
|
544
|
+
# and never recorded a winner, because a withdrawal has none. All three
|
|
545
|
+
# are correct for taking a PUBLISH away and wrong for giving one back,
|
|
546
|
+
# and a restoration written through the withdrawal's fields would land a
|
|
547
|
+
# PUBLISH in the record with no model, no control and `ships: False`
|
|
548
|
+
# beside it -- a verdict a reader cannot act on and a sales claim nobody
|
|
549
|
+
# can trace to an artifact.
|
|
550
|
+
today = time.strftime("%Y-%m-%d", time.gmtime())
|
|
330
551
|
by_cap = {r["capability"]: r for r in changed}
|
|
331
552
|
for cap in adj["capabilities"]:
|
|
332
553
|
r = by_cap.get(cap["capability"])
|
|
333
554
|
if not r:
|
|
334
555
|
continue
|
|
335
556
|
was = dict(cap["decision"])
|
|
336
|
-
|
|
557
|
+
win = r.get("winner") or {}
|
|
558
|
+
publishes = r["regraded_decision"] == "PUBLISH"
|
|
559
|
+
if publishes:
|
|
560
|
+
why_corrected = (
|
|
561
|
+
"a control that CAN rate this class was added after the screen ran "
|
|
562
|
+
"(`%s`, n=%s, grade %s) and is committed in %s, which computed its rate "
|
|
563
|
+
"with venue_adjudication.py's OWN rule -- same GRID, same QUIET_MAX_FIRE, "
|
|
564
|
+
"same worst-valid-bucket quiet_t, same bar. Re-applying that bar with the "
|
|
565
|
+
"new control in scope clears it. The elided rows stay unknown and cannot "
|
|
566
|
+
"block: an unknown can only ever ADD a passing candidate."
|
|
567
|
+
% (win.get("control"), win.get("control_n"), win.get("grade"),
|
|
568
|
+
", ".join(sorted({os.path.basename(x) for x in (args.rescore or [])}))
|
|
569
|
+
or "a rescore artifact"))
|
|
570
|
+
else:
|
|
571
|
+
why_corrected = (
|
|
572
|
+
"the verdict rested on `%s` (n=%s, grade %s), a set whose own "
|
|
573
|
+
"contact-sheet note says it can only show liveness. Re-applying "
|
|
574
|
+
"this run's OWN bar to the controls that can rate a class "
|
|
575
|
+
"overturns it." % (was.get("control"), was.get("control_n"),
|
|
576
|
+
r.get("prior_control_grade")))
|
|
577
|
+
decision = {
|
|
337
578
|
"decision": r["regraded_decision"],
|
|
338
579
|
"why": r["why"],
|
|
339
|
-
#
|
|
340
|
-
#
|
|
341
|
-
# the
|
|
580
|
+
# NOT re-derived here in either direction, and for the same
|
|
581
|
+
# reason: `events_lost` / `vlm_calls_saved` are computed by
|
|
582
|
+
# venue_adjudication.py from the FULL per-frame score matrices
|
|
583
|
+
# against the whole negative set, which this tool does not hold.
|
|
584
|
+
# A withdrawal must not keep a promise computed from a recall it
|
|
585
|
+
# no longer accepts; a restoration must not invent one.
|
|
342
586
|
"gate": None,
|
|
343
|
-
"
|
|
344
|
-
"
|
|
345
|
-
"
|
|
346
|
-
"
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
587
|
+
"gate_is_null_because": (
|
|
588
|
+
"the cascade-gate arithmetic is a function of the full score matrices, "
|
|
589
|
+
"which this tool does not read. A verdict is derivable from committed "
|
|
590
|
+
"rates; a savings promise is not."),
|
|
591
|
+
}
|
|
592
|
+
caveats = [c for n in (r.get("rescored_controls") or [])
|
|
593
|
+
for c in (n.get("sellability_caveats") or [])]
|
|
594
|
+
if caveats:
|
|
595
|
+
# TOP LEVEL, beside the verdict, not buried in `corrected`. This
|
|
596
|
+
# file is quoted in the sales claim sheet, and a PUBLISH whose
|
|
597
|
+
# caveat is one level down is a PUBLISH that gets quoted alone.
|
|
598
|
+
decision["sellability_caveats"] = caveats
|
|
599
|
+
if publishes:
|
|
600
|
+
decision.update({
|
|
601
|
+
"model": win.get("model"),
|
|
602
|
+
"class": win.get("class"),
|
|
603
|
+
"control": win.get("control"),
|
|
604
|
+
"control_n": win.get("control_n"),
|
|
605
|
+
"control_grade": win.get("grade"),
|
|
606
|
+
"operating_conf": win.get("quiet_t"),
|
|
607
|
+
"n_valid_negatives": win.get("n_valid_negatives"),
|
|
608
|
+
"also_scored": win.get("also_scored"),
|
|
609
|
+
})
|
|
610
|
+
decision["corrected"] = {
|
|
611
|
+
"on": today,
|
|
612
|
+
"by": "deploy/detector/bench/regrade_control_selection.py --apply",
|
|
613
|
+
"was": was,
|
|
614
|
+
"ships": publishes,
|
|
615
|
+
"why_corrected": why_corrected,
|
|
616
|
+
"needs_remeasure": r["candidates_unknown"],
|
|
617
|
+
"rescored_controls": r.get("rescored_controls") or [],
|
|
618
|
+
"regrade_artifact": args.out,
|
|
357
619
|
}
|
|
620
|
+
cap["decision"] = decision
|
|
358
621
|
cap["changed"] = bool(cap.get("prior_decision_taxi_negatives")
|
|
359
622
|
and cap["prior_decision_taxi_negatives"]
|
|
360
623
|
!= r["regraded_decision"])
|
|
@@ -74,17 +74,51 @@ def classify_layout(shape, output_names) -> str:
|
|
|
74
74
|
raise ParityError(f"shape {shape} matches no known detector layout")
|
|
75
75
|
|
|
76
76
|
|
|
77
|
-
def box_scale_for(layout: str, imgsz: int) -> float:
|
|
77
|
+
def box_scale_for(layout: str, imgsz: int, box_abs_max: "float | None" = None) -> float:
|
|
78
78
|
"""The multiplier that puts LiteRT boxes into the ONNX pixel frame.
|
|
79
79
|
|
|
80
|
-
|
|
81
|
-
model in `_NormalizeCoords`, which divides the four BOX channels by (W, H)
|
|
82
|
-
and leaves the class channels alone. An end2end head is `end2end=True`, so
|
|
83
|
-
the wrapper never runs and both runtimes emit pixels.
|
|
80
|
+
UNITS ARE A PROPERTY OF THE CHECKPOINT, NOT OF THE LAYOUT.
|
|
84
81
|
|
|
85
|
-
|
|
86
|
-
|
|
82
|
+
This returned `imgsz` for every ANCHOR-layout twin, on the reasoning that
|
|
83
|
+
ultralytics wraps a `task=detect` / `end2end=False` LiteRT export in
|
|
84
|
+
`_NormalizeCoords`. That reasoning is disproved by a control: two models
|
|
85
|
+
exported MINUTES APART in one venv, same tool (8.3.201), same flags
|
|
86
|
+
(opset 19, imgsz 640, `nms=False`), both classified ANCHOR, measured
|
|
87
|
+
|
|
88
|
+
0.0024 (normalized) and 2.8526 (pixels)
|
|
89
|
+
|
|
90
|
+
Same exporter, same settings, same run, opposite units. So no function of
|
|
91
|
+
(layout, imgsz) can be right in general, and the previous signature could
|
|
92
|
+
not express the truth even when it knew it.
|
|
93
|
+
|
|
94
|
+
`box_abs_max` — the largest absolute box value the LiteRT artifact actually
|
|
95
|
+
produced on a forward pass — is what settles it. Pass it and the units are
|
|
96
|
+
DERIVED via `classify_box_units`; the layout is then only a sanity check.
|
|
97
|
+
|
|
98
|
+
Omitting it is still allowed for the historical rows in
|
|
99
|
+
`litert-twin-parity.json`, which were recorded before anything measured
|
|
100
|
+
box_abs_max — those keep the old layout-derived answer, and the ANCHOR case
|
|
101
|
+
is exactly where that answer may be wrong. It is deliberately NOT the path
|
|
102
|
+
a new measurement takes: `verify_litert_twin_parity.py` always passes it,
|
|
103
|
+
and `verdict()` refuses a new ANCHOR row that arrives without one.
|
|
104
|
+
|
|
105
|
+
No layout and no measurement gets a scale by default. A fall-through to 1.0
|
|
106
|
+
would be a guess about an artifact nobody measured, and the wrong guess is
|
|
107
|
+
silent.
|
|
87
108
|
"""
|
|
109
|
+
if box_abs_max is not None:
|
|
110
|
+
# Deferred import: the contract module is stdlib-only so the bare
|
|
111
|
+
# `detector-service` CI lane can import it, and box_units is too.
|
|
112
|
+
from box_units import classify_box_units # noqa: PLC0415
|
|
113
|
+
|
|
114
|
+
measured = classify_box_units(box_abs_max, imgsz)
|
|
115
|
+
if measured["scale_to_pixels"] is None:
|
|
116
|
+
raise ParityError(
|
|
117
|
+
f"box units are unmeasured for this artifact: {measured['refusal']}. "
|
|
118
|
+
"Refusing to assume a scale from the layout — that assumption is "
|
|
119
|
+
"what the 0.0024-vs-2.8526 control disproved."
|
|
120
|
+
)
|
|
121
|
+
return float(measured["scale_to_pixels"])
|
|
88
122
|
if layout == ANCHOR:
|
|
89
123
|
return float(imgsz)
|
|
90
124
|
if layout in (END2END, DETS_LABELS):
|
|
@@ -95,14 +129,49 @@ def box_scale_for(layout: str, imgsz: int) -> float:
|
|
|
95
129
|
def verdict(result) -> "tuple[bool, str]":
|
|
96
130
|
"""Does this recorded measurement pass, and if not, which assertion failed."""
|
|
97
131
|
layout = result["layout"]
|
|
98
|
-
|
|
99
|
-
#
|
|
100
|
-
#
|
|
101
|
-
|
|
132
|
+
scale = result.get("box_scale_applied")
|
|
133
|
+
# THE LOAD-BEARING CHECK APPLIES TO A NORMALISED ARTIFACT, NOT TO A LAYOUT.
|
|
134
|
+
#
|
|
135
|
+
# This keyed on `layout == ANCHOR` and read an unscaled match as "the layout
|
|
136
|
+
# is misread". The control disproved the premise: an ANCHOR twin can emit
|
|
137
|
+
# PIXELS, and for that artifact an unscaled match is the CORRECT result, not
|
|
138
|
+
# a misread. Keying it on the layout refused a good artifact and is why the
|
|
139
|
+
# two models published 2026-08-27 shipped ONNX-only.
|
|
140
|
+
#
|
|
141
|
+
# Keyed on the applied scale, the assertion says what it always meant: if we
|
|
142
|
+
# scaled, the scaling has to matter. A scale of 1.0 is the measured answer
|
|
143
|
+
# for a pixel artifact and asserts nothing about normalisation.
|
|
144
|
+
if scale is not None and scale > 1.0 and result["box_residual_unscaled"] <= BOX_TOL_PX:
|
|
102
145
|
return False, (
|
|
103
|
-
"the
|
|
104
|
-
"
|
|
146
|
+
f"the x{scale:g} scale changed nothing — a twin whose RAW boxes already match "
|
|
147
|
+
"its ONNX source is not normalised, so the measured units are wrong"
|
|
148
|
+
)
|
|
149
|
+
# THE ROW MUST ESTABLISH ITS UNITS, BY EITHER ROUTE.
|
|
150
|
+
#
|
|
151
|
+
# `box_abs_max` settles it directly. But a row recorded before anything
|
|
152
|
+
# measured that field can still be decisive: if the scaled residual passes
|
|
153
|
+
# and the UNSCALED one misses by orders of magnitude, the artifact is
|
|
154
|
+
# demonstrably normalised — the 22 committed rows carry exactly that shape
|
|
155
|
+
# (e.g. 0.00087 px scaled against 641.03 unscaled). Demanding the field when
|
|
156
|
+
# the evidence is already present in another form would refuse good rows to
|
|
157
|
+
# satisfy a schema, so the check asks whether the units are ESTABLISHED, not
|
|
158
|
+
# whether a particular key exists.
|
|
159
|
+
#
|
|
160
|
+
# What it still refuses is the ambiguous row: no measurement, and two
|
|
161
|
+
# residuals that do not discriminate. That is a twin whose units nobody knows.
|
|
162
|
+
if result.get("box_abs_max") is None and result.get("units") is None:
|
|
163
|
+
scaled = result.get("box_residual_px")
|
|
164
|
+
unscaled = result.get("box_residual_unscaled")
|
|
165
|
+
decisive = (
|
|
166
|
+
scaled is not None and unscaled is not None
|
|
167
|
+
and scaled <= BOX_TOL_PX and unscaled > BOX_TOL_PX * 100
|
|
105
168
|
)
|
|
169
|
+
if not decisive:
|
|
170
|
+
return False, (
|
|
171
|
+
"twin recorded without box_abs_max, and its scaled/unscaled residuals do "
|
|
172
|
+
"not discriminate — units were assumed from the layout, and layout does "
|
|
173
|
+
"not determine units (0.0024 vs 2.8526 control, one venv, one exporter)"
|
|
174
|
+
)
|
|
106
175
|
if result["box_residual_px"] > BOX_TOL_PX:
|
|
107
176
|
return False, (
|
|
108
177
|
f"box residual {result['box_residual_px']:.4g} px exceeds {BOX_TOL_PX} "
|