@trillboards/edge-sdk 0.2.110 → 0.2.113
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/detector/bench/litert-twin-parity.json +25 -0
- package/deploy/detector/bench/regrade_control_selection.py +291 -28
- package/deploy/detector/bench/twin_parity_contract.py +82 -13
- package/deploy/detector/bench/venue_adjudication.py +101 -8
- package/deploy/detector/bench/verify_litert_twin_parity.py +22 -18
- package/deploy/detector/tests/test_box_units_are_measured.py +137 -0
- package/deploy/detector/tests/test_regrade_control_selection.py +108 -0
- package/deploy/detector/trillboards_detector/labels.py +24 -1
- package/deploy/macos/TrillboardsSetupApp.m +135 -0
- package/deploy/windows/trillboards-setup.ps1 +166 -0
- package/dist/AudienceAnalyzeClient.d.ts +11 -0
- package/dist/AudienceAnalyzeClient.d.ts.map +1 -1
- package/dist/AudienceAnalyzeClient.js +7 -0
- package/dist/AudienceAnalyzeClient.js.map +1 -1
- package/dist/ChannelProber.d.ts.map +1 -1
- package/dist/ChannelProber.js +15 -1
- package/dist/ChannelProber.js.map +1 -1
- package/dist/CloudSensingManager.d.ts +20 -0
- package/dist/CloudSensingManager.d.ts.map +1 -1
- package/dist/CloudSensingManager.js +30 -0
- package/dist/CloudSensingManager.js.map +1 -1
- package/dist/EdgeAgent.js +1 -1
- package/dist/SensingProfileClient.d.ts +10 -0
- package/dist/SensingProfileClient.d.ts.map +1 -1
- package/dist/SensingProfileClient.js +16 -0
- package/dist/SensingProfileClient.js.map +1 -1
- package/dist/diagnose.d.ts +1 -1
- package/dist/diagnose.js +1 -1
- package/dist/discovery.d.ts +1 -1
- package/dist/discovery.d.ts.map +1 -1
- package/dist/discovery.js +15 -4
- package/dist/discovery.js.map +1 -1
- package/dist/localSensing.d.ts +70 -0
- package/dist/localSensing.d.ts.map +1 -1
- package/dist/localSensing.js +182 -1
- package/dist/localSensing.js.map +1 -1
- package/package.json +1 -1
|
@@ -130,7 +130,13 @@ CLASS_CONTROL = {
|
|
|
130
130
|
"gloves": ["ppe_full", "wc_glove"], "glove": ["ppe_full", "wc_glove"], "glove_hand": ["wc_glove", "ppe_full"],
|
|
131
131
|
"apron": ["wc_apron"], "Apron": ["wc_apron"], "hairnet": ["n_hairnet"], "Hairnet": ["n_hairnet"],
|
|
132
132
|
"safety cone": ["n_cone"], "safety_cone": ["n_cone"],
|
|
133
|
-
"fire_extinguisher": ["n_extinguisher"],
|
|
133
|
+
"fire_extinguisher": ["n_extinguisher"],
|
|
134
|
+
# `ladder_openimages` (22 frames / 22 scenes, `good`) leads because it is the set that
|
|
135
|
+
# can rate the class: human-verified Open Images boxes, hand-graded on the pixels.
|
|
136
|
+
# `n_ladder` is KEPT and stays second so its POOR number remains visible in
|
|
137
|
+
# `per_control` -- the same reason `fire_generic` was kept after it was demoted. A
|
|
138
|
+
# control that was refused is evidence about the search, and deleting it hides that.
|
|
139
|
+
"ladder": ["ladder_openimages", "n_ladder"],
|
|
134
140
|
"person": ["helmet", "ppe_full", "n_crowd"], "worker": ["ppe_full", "helmet", "n_crowd"],
|
|
135
141
|
"human": ["helmet", "ppe_full", "n_crowd"], "head": ["helmet", "n_crowd"],
|
|
136
142
|
"forklift": ["forklift"], "forklift-partial": ["forklift"], "Forklift-Partial": ["forklift"], "yoto": ["forklift"],
|
|
@@ -145,8 +151,19 @@ CLASS_CONTROL = {
|
|
|
145
151
|
"firearm": ["gun_clean", "weapon_cctv"], "knife": ["knife_cctv", "weapon_cctv", "n_knife"],
|
|
146
152
|
"fall": ["fall_cctv"], "fallen": ["fall_cctv"], "fall-detected": ["fall_cctv"],
|
|
147
153
|
"Fall-Detected": ["fall_cctv"], "down": ["fall_cctv"],
|
|
148
|
-
"door": ["n_door"], "window": ["n_window"],
|
|
149
|
-
|
|
154
|
+
"door": ["n_door"], "window": ["n_window"],
|
|
155
|
+
# `phone_use_meva` (22 frames / 17 fixed cameras, `good`) leads because it is the only
|
|
156
|
+
# set that can rate this class at all TODAY: `n_phone` is graded `good` but its FRAMES
|
|
157
|
+
# ARE GONE -- s3://trillboards-media-prod/DEV/regression-corpus/ holds no `n_phone`
|
|
158
|
+
# prefix, measured 2026-08-27, the same as fire_cctv/fire_generic/wc_spill. `n_phone` is
|
|
159
|
+
# KEPT and stays second for the same reason `fire_generic` was kept after it was
|
|
160
|
+
# demoted: its number is in the committed adjudication and deleting the mapping would
|
|
161
|
+
# hide what the earlier screen scored. The two also answer different questions --
|
|
162
|
+
# `n_phone` is web stills of people holding smartphones, `phone_use_meva` is fixed CCTV
|
|
163
|
+
# at 79-690 px of person height, which is the deployment.
|
|
164
|
+
"phone": ["phone_use_meva", "n_phone"], "cell phone": ["phone_use_meva", "n_phone"],
|
|
165
|
+
"phone_call": ["phone_use_meva", "n_phone"],
|
|
166
|
+
"mouse": ["n_rodent"], "void": ["n_shelfgap"],
|
|
150
167
|
"cart": ["wc_cart", "n_trolley"], "trolley": ["wc_cart", "n_trolley"], "shopping_cart": ["wc_cart", "n_trolley"],
|
|
151
168
|
"spill": ["wc_spill", "n_spill"], "Spill": ["wc_spill", "n_spill"],
|
|
152
169
|
"cigarette": ["n_smoking"], "Cigarette": ["n_smoking"], "smoking": ["n_smoking"],
|
|
@@ -209,6 +226,33 @@ def assert_every_control_is_graded(pos_sets, ctlq) -> None:
|
|
|
209
226
|
"would be treated as able to rate a class." % ", ".join(missing))
|
|
210
227
|
|
|
211
228
|
|
|
229
|
+
#: The sellable-capability SHAPE contract, read by this tool and by
|
|
230
|
+
#: `.github/scripts/check-sensing-capability-shape.js`. ONE definition, two consumers --
|
|
231
|
+
#: the same posture as `LIVENESS_ONLY_GRADES` above and for the same reason: a tool and a
|
|
232
|
+
#: guard that each carried their own idea of which capabilities a frame can answer would
|
|
233
|
+
#: let a file through that the other would never emit.
|
|
234
|
+
CAPABILITY_SHAPE_PATH = os.path.join(
|
|
235
|
+
os.path.dirname(os.path.abspath(__file__)),
|
|
236
|
+
*([os.pardir] * 6), ".github", "contracts", "sensing-capability-shape.json")
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def load_capability_shape(path: str = CAPABILITY_SHAPE_PATH) -> dict:
|
|
240
|
+
"""`capability -> row` from the shape contract.
|
|
241
|
+
|
|
242
|
+
REFUSES rather than degrading when the contract is missing. An adjudication that could
|
|
243
|
+
not read it would silently fall back to grading every capability as if a frame answered
|
|
244
|
+
it, which is exactly the state this file is being changed to end: on 2026-08-26 five
|
|
245
|
+
capabilities came back UNINFORMATIVE under one sentence -- "no usable positive control
|
|
246
|
+
exists for this class" -- and for four of them that was a statement about a set somebody
|
|
247
|
+
sourced and refused, while for `shoplifting / theft` it was a statement about a dict that
|
|
248
|
+
never had an entry. The reader could not tell, and `cctv-detector-tier.md` s10.7 then
|
|
249
|
+
told them all five needed "an afternoon of labelling".
|
|
250
|
+
"""
|
|
251
|
+
with open(path) as fh:
|
|
252
|
+
doc = json.load(fh)
|
|
253
|
+
return doc.get("capabilities") or {}
|
|
254
|
+
|
|
255
|
+
|
|
212
256
|
def normalise_class(name: str) -> str:
|
|
213
257
|
"""`labels.py::normalize_class_name`: lower, and `-`/`_` folded to space.
|
|
214
258
|
|
|
@@ -403,6 +447,10 @@ def main() -> None:
|
|
|
403
447
|
"`assert_every_control_is_graded` refuses the run rather than "
|
|
404
448
|
"adjudicating on a set nobody looked at."
|
|
405
449
|
)
|
|
450
|
+
ap.add_argument("--capability-shape", default=CAPABILITY_SHAPE_PATH,
|
|
451
|
+
help="the sellable-capability SHAPE contract. Defaults to "
|
|
452
|
+
".github/contracts/sensing-capability-shape.json; an override "
|
|
453
|
+
"exists for replaying an old run, not for the normal path.")
|
|
406
454
|
ap.add_argument("--out", required=True)
|
|
407
455
|
ap.add_argument("--markdown", help="also write the runbook table here")
|
|
408
456
|
args = ap.parse_args()
|
|
@@ -420,6 +468,15 @@ def main() -> None:
|
|
|
420
468
|
for _k, _v in json.load(open(_q)).items():
|
|
421
469
|
if isinstance(_v, dict) and isinstance(_v.get("grade"), str):
|
|
422
470
|
ctlq[_k] = _v
|
|
471
|
+
shape_rows = load_capability_shape(args.capability_shape)
|
|
472
|
+
missing_shape = sorted(set(CAPS) - set(shape_rows))
|
|
473
|
+
if missing_shape:
|
|
474
|
+
raise SystemExit(
|
|
475
|
+
"capability shape undeclared for: %s. Every sellable capability must say "
|
|
476
|
+
"whether a FRAME can answer it before a firing rate on it means anything; "
|
|
477
|
+
"add a row to .github/contracts/sensing-capability-shape.json. The "
|
|
478
|
+
"`sensing-capability-shape` guard refuses the same state statically."
|
|
479
|
+
% ", ".join(missing_shape))
|
|
423
480
|
index = json.load(open(os.path.join(args.out_dir, "_index.json")))
|
|
424
481
|
|
|
425
482
|
meta = combo["meta"]
|
|
@@ -779,7 +836,7 @@ def main() -> None:
|
|
|
779
836
|
c["per_control"] = {}
|
|
780
837
|
c["gate"] = None
|
|
781
838
|
|
|
782
|
-
now = decide(cap, cands)
|
|
839
|
+
now = decide(cap, cands, shape_rows)
|
|
783
840
|
was, was_model = PRIOR_VERDICT.get(cap, (None, None))
|
|
784
841
|
rows.append({"capability": cap, "n_candidates": len(cands),
|
|
785
842
|
"prior_decision_taxi_negatives": was,
|
|
@@ -790,6 +847,10 @@ def main() -> None:
|
|
|
790
847
|
json.dump({
|
|
791
848
|
"schema": "trillboards.venue-adjudication/1",
|
|
792
849
|
"negative_set": "venue-phase3-source-2026-08-26",
|
|
850
|
+
"capability_shape_contract": os.path.relpath(args.capability_shape, os.path.join(
|
|
851
|
+
os.path.dirname(os.path.abspath(__file__)), *([os.pardir] * 6))),
|
|
852
|
+
"inexpressible_capabilities": sorted(
|
|
853
|
+
k for k, v in shape_rows.items() if v.get("expressible_today") is False),
|
|
793
854
|
"supersedes": "the 1,500-frame `production` negative set of 2026-08-25, "
|
|
794
855
|
"1,494 of which were taxi_rideshare_tv cabin interiors",
|
|
795
856
|
"bar": {
|
|
@@ -892,11 +953,32 @@ def _other_controls_clause(win: dict) -> str:
|
|
|
892
953
|
return "; the same artifact at that threshold keeps " + ", ".join(parts)
|
|
893
954
|
|
|
894
955
|
|
|
895
|
-
def decide(cap: str, cands: list) -> dict:
|
|
896
|
-
"""PUBLISH / NO MODEL CLEARS BAR / NOT SCREENABLE, with the reason.
|
|
956
|
+
def decide(cap: str, cands: list, shape_rows: dict = None) -> dict:
|
|
957
|
+
"""NOT A FRAME QUESTION / PUBLISH / NO MODEL CLEARS BAR / NOT SCREENABLE, with the reason.
|
|
958
|
+
|
|
959
|
+
Same ladder as the 2026-08-25 screen below the first rung. Only the negative set differs.
|
|
897
960
|
|
|
898
|
-
|
|
961
|
+
THE FIRST RUNG IS NEW, and it runs before candidates are considered at all, because
|
|
962
|
+
whether a frame can answer the question is not a fact about which artifacts were sourced.
|
|
963
|
+
A capability the shape contract declares inexpressible gets `NOT A FRAME QUESTION` and
|
|
964
|
+
names the substrate it is blocked on -- never `UNINFORMATIVE`, which in this file means
|
|
965
|
+
"a control was sourced for this class and its contact-sheet grade refuses it" and is a
|
|
966
|
+
corpus problem an afternoon of labelling fixes.
|
|
899
967
|
"""
|
|
968
|
+
row = (shape_rows or {}).get(cap) or {}
|
|
969
|
+
if row.get("expressible_today") is False:
|
|
970
|
+
return {"decision": "NOT A FRAME QUESTION",
|
|
971
|
+
"shape": row.get("shape"),
|
|
972
|
+
"frame_observable": row.get("frame_observable"),
|
|
973
|
+
"substrate_required": row.get("substrate_required"),
|
|
974
|
+
"n_candidates_screened": len(cands),
|
|
975
|
+
"why": "this product cannot express the question, so no positive control "
|
|
976
|
+
"would make a firing rate here mean anything. Shape is `%s`; the "
|
|
977
|
+
"per-frame observable is %s; blocked on `%s`. See "
|
|
978
|
+
".github/contracts/sensing-capability-shape.json for the evidence and "
|
|
979
|
+
"docs/runbooks/sensing-model-library.md s1a for the working."
|
|
980
|
+
% (row.get("shape"), row.get("frame_observable"),
|
|
981
|
+
row.get("substrate_required"))}
|
|
900
982
|
if not cands:
|
|
901
983
|
return {"decision": "NO CANDIDATE",
|
|
902
984
|
"why": "no sourced artifact declares a class for this capability"}
|
|
@@ -909,9 +991,20 @@ def decide(cap: str, cands: list) -> dict:
|
|
|
909
991
|
live = [c for c in cands if (c["control_rate_025"] or 0) > 0]
|
|
910
992
|
if not live:
|
|
911
993
|
if all(c["best_control"] is None for c in cands):
|
|
994
|
+
# NAME the sets, because "no usable positive control exists" reads as a claim
|
|
995
|
+
# about the world and is a claim about two dicts. A class with a control that was
|
|
996
|
+
# sourced, rendered on a contact sheet and REFUSED is one afternoon of labelling
|
|
997
|
+
# from a verdict; a class that was never mapped may be a class no frame answers,
|
|
998
|
+
# and `NOT A FRAME QUESTION` above is where that ends up. Both used to print here.
|
|
999
|
+
mapped = sorted({n for c in cands
|
|
1000
|
+
for n in CLASS_CONTROL_N.get(normalise_class(c["class"]), [])})
|
|
1001
|
+
detail = ("the mapped control set(s) %s were all refused by their contact-sheet "
|
|
1002
|
+
"grade" % ", ".join("`%s`" % m for m in mapped)) if mapped else (
|
|
1003
|
+
"no control set is mapped to this class in CLASS_CONTROL at all")
|
|
912
1004
|
return {"decision": "UNINFORMATIVE",
|
|
1005
|
+
"controls_mapped": mapped,
|
|
913
1006
|
"why": "no usable positive control exists for this class, so no firing "
|
|
914
|
-
"rate here is a finding"}
|
|
1007
|
+
"rate here is a finding -- %s" % detail}
|
|
915
1008
|
return {"decision": "NO MODEL CLEARS BAR",
|
|
916
1009
|
"why": "no candidate fires on its positive control"}
|
|
917
1010
|
retained = [c for c in live if (c["control_rate_at_quiet_t"] or 0) >= RECALL_RETENTION_BAR]
|
|
@@ -53,6 +53,7 @@ import numpy as np
|
|
|
53
53
|
|
|
54
54
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
55
55
|
|
|
56
|
+
from box_units import classify_box_units
|
|
56
57
|
from twin_parity_contract import ( # noqa: E402 (same directory)
|
|
57
58
|
ANCHOR,
|
|
58
59
|
BOX_TOL_PX,
|
|
@@ -93,12 +94,19 @@ def compare(onnx_out: np.ndarray, litert_out: np.ndarray, layout: str, imgsz: in
|
|
|
93
94
|
raise ParityError(f"shape mismatch onnx={onnx_out.shape} litert={litert_out.shape}")
|
|
94
95
|
a = litert_out.astype(np.float64)
|
|
95
96
|
b = onnx_out.astype(np.float64)
|
|
96
|
-
scale = box_scale_for(layout, imgsz)
|
|
97
97
|
box_l, cls_l = split_channels(a, layout)
|
|
98
98
|
box_o, cls_o = split_channels(b, layout)
|
|
99
|
+
# MEASURE the units off this artifact's own boxes rather than inferring them
|
|
100
|
+
# from the layout. Two anchor-layout twins exported minutes apart in one venv
|
|
101
|
+
# measured 0.0024 and 2.8526 — same tool, same flags, opposite units.
|
|
102
|
+
box_abs_max = float(np.abs(box_l).max()) if box_l.size else 0.0
|
|
103
|
+
units = classify_box_units(box_abs_max, imgsz)
|
|
104
|
+
scale = box_scale_for(layout, imgsz, box_abs_max=box_abs_max)
|
|
99
105
|
return {
|
|
100
106
|
"layout": layout,
|
|
101
107
|
"imgsz": imgsz,
|
|
108
|
+
"box_abs_max": box_abs_max,
|
|
109
|
+
"units": units["units"],
|
|
102
110
|
"box_scale_applied": scale,
|
|
103
111
|
"box_residual_px": float(np.abs(box_l * scale - box_o).max()),
|
|
104
112
|
"box_residual_unscaled": float(np.abs(box_l - box_o).max()),
|
|
@@ -106,23 +114,19 @@ def compare(onnx_out: np.ndarray, litert_out: np.ndarray, layout: str, imgsz: in
|
|
|
106
114
|
}
|
|
107
115
|
|
|
108
116
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
)
|
|
123
|
-
if result["cls_residual"] > CLS_TOL:
|
|
124
|
-
return False, f"class residual {result['cls_residual']:.4g} exceeds {CLS_TOL}"
|
|
125
|
-
return True, "ok"
|
|
117
|
+
# `verdict` is NOT redefined here. It lives in `twin_parity_contract` and is
|
|
118
|
+
# imported above.
|
|
119
|
+
#
|
|
120
|
+
# A second copy stood at this exact spot and SHADOWED the import. It was a
|
|
121
|
+
# faithful duplicate when written, so nothing looked wrong — and then the
|
|
122
|
+
# contract's copy was fixed to derive box units from the artifact while this one
|
|
123
|
+
# kept keying on the layout. The verifier went on refusing the pixel-space
|
|
124
|
+
# anchor twin the fix was written to admit, and the contract's own tests passed
|
|
125
|
+
# the whole time, because they exercise the module that is not the one running.
|
|
126
|
+
#
|
|
127
|
+
# Caught by codex on the very PR that introduced the divergence. One rule, one
|
|
128
|
+
# home: everything that decides pass/fail for a twin is in the contract module,
|
|
129
|
+
# which is also the module the stdlib-only CI lane can import without numpy.
|
|
126
130
|
|
|
127
131
|
|
|
128
132
|
def main() -> int:
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Box units are a property of the CHECKPOINT, not of the layout.
|
|
2
|
+
|
|
3
|
+
`box_scale_for` returned `imgsz` for every anchor-layout twin, reasoning that
|
|
4
|
+
ultralytics wraps a `task=detect` / `end2end=False` LiteRT export in
|
|
5
|
+
`_NormalizeCoords`. A control disproves the premise: two models exported MINUTES
|
|
6
|
+
APART in one venv — same tool 8.3.201, same flags (opset 19, imgsz 640,
|
|
7
|
+
nms=False), both classified anchor — measured
|
|
8
|
+
|
|
9
|
+
box residual 0.0024 under the normalized reading, and 2.8526 under the
|
|
10
|
+
pixel reading — one artifact matches its ONNX twin only when scaled, the
|
|
11
|
+
other only when not
|
|
12
|
+
|
|
13
|
+
so no function of (layout, imgsz) can be right in general. (Those two figures are
|
|
14
|
+
RESIDUALS, not box maxima; the maxima below are what `classify_box_units` grades,
|
|
15
|
+
and a normalized artifact's boxes top out near 1.0 while a pixel artifact's
|
|
16
|
+
approach imgsz.) Keying the verdict on
|
|
17
|
+
the layout refused a good artifact, which is why two models published
|
|
18
|
+
2026-08-27 shipped ONNX-only.
|
|
19
|
+
|
|
20
|
+
Stdlib only, like the rest of this suite, so the bare `detector-service` lane can
|
|
21
|
+
run it with no numpy and no artifact.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
import os
|
|
25
|
+
import sys
|
|
26
|
+
import unittest
|
|
27
|
+
|
|
28
|
+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "bench"))
|
|
29
|
+
|
|
30
|
+
from twin_parity_contract import ( # noqa: E402
|
|
31
|
+
ANCHOR,
|
|
32
|
+
BOX_TOL_PX,
|
|
33
|
+
ParityError,
|
|
34
|
+
box_scale_for,
|
|
35
|
+
verdict,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def row(**kw):
|
|
40
|
+
base = {
|
|
41
|
+
"layout": ANCHOR, "imgsz": 640, "cls_residual": 1e-7,
|
|
42
|
+
"box_scale_applied": 640.0, "box_residual_px": 0.0008,
|
|
43
|
+
"box_residual_unscaled": 641.0,
|
|
44
|
+
}
|
|
45
|
+
base.update(kw)
|
|
46
|
+
return base
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class BoxUnitsAreMeasured(unittest.TestCase):
|
|
50
|
+
def test_pixel_anchor_twin_passes(self):
|
|
51
|
+
"""The artifact the old rule refused. Anchor layout, PIXEL units."""
|
|
52
|
+
ok, why = verdict(row(box_abs_max=512.0, units="pixels",
|
|
53
|
+
box_scale_applied=1.0, box_residual_unscaled=0.0009,
|
|
54
|
+
box_residual_px=0.0009))
|
|
55
|
+
self.assertTrue(ok, why)
|
|
56
|
+
|
|
57
|
+
def test_normalised_anchor_twin_passes(self):
|
|
58
|
+
ok, why = verdict(row(box_abs_max=0.87, units="normalized"))
|
|
59
|
+
self.assertTrue(ok, why)
|
|
60
|
+
|
|
61
|
+
def test_the_640x_bug_is_still_caught(self):
|
|
62
|
+
"""Scaled by 640 and the scale changed nothing — the original defect."""
|
|
63
|
+
ok, why = verdict(row(box_abs_max=0.87, units="normalized",
|
|
64
|
+
box_residual_unscaled=0.0008))
|
|
65
|
+
self.assertFalse(ok)
|
|
66
|
+
self.assertIn("changed nothing", why)
|
|
67
|
+
|
|
68
|
+
def test_unmeasured_and_undiscriminating_is_refused(self):
|
|
69
|
+
"""No measurement AND residuals that do not separate = units unknown."""
|
|
70
|
+
ok, _ = verdict({"layout": ANCHOR, "imgsz": 640, "box_scale_applied": 640.0,
|
|
71
|
+
"box_residual_px": 0.0009,
|
|
72
|
+
"box_residual_unscaled": BOX_TOL_PX * 2, "cls_residual": 1e-7})
|
|
73
|
+
self.assertFalse(ok)
|
|
74
|
+
|
|
75
|
+
def test_historical_rows_establish_units_without_the_field(self):
|
|
76
|
+
"""The 22 committed rows predate box_abs_max and are still decisive.
|
|
77
|
+
|
|
78
|
+
Demanding the field when the evidence is present in another form would
|
|
79
|
+
refuse good rows to satisfy a schema.
|
|
80
|
+
"""
|
|
81
|
+
ok, why = verdict({"layout": ANCHOR, "imgsz": 640, "box_scale_applied": 640.0,
|
|
82
|
+
"box_residual_px": 0.00087, "box_residual_unscaled": 641.03,
|
|
83
|
+
"cls_residual": 2e-6})
|
|
84
|
+
self.assertTrue(ok, why)
|
|
85
|
+
|
|
86
|
+
def test_measurement_overrides_the_layout(self):
|
|
87
|
+
"""Same layout, same imgsz, opposite scales — decided by the artifact."""
|
|
88
|
+
self.assertEqual(box_scale_for(ANCHOR, 640, box_abs_max=0.87), 640.0)
|
|
89
|
+
self.assertEqual(box_scale_for(ANCHOR, 640, box_abs_max=512.0), 1.0)
|
|
90
|
+
|
|
91
|
+
def test_a_useless_measurement_refuses_rather_than_guessing(self):
|
|
92
|
+
with self.assertRaises(ParityError):
|
|
93
|
+
box_scale_for(ANCHOR, 640, box_abs_max=0.0)
|
|
94
|
+
|
|
95
|
+
def test_layout_route_survives_for_callers_with_no_measurement(self):
|
|
96
|
+
self.assertEqual(box_scale_for(ANCHOR, 640), 640.0)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class TheVerifierUsesTheContractsVerdict(unittest.TestCase):
|
|
100
|
+
"""One rule, one home — the verifier must not carry its own copy.
|
|
101
|
+
|
|
102
|
+
A faithful duplicate of `verdict` sat in verify_litert_twin_parity.py and
|
|
103
|
+
SHADOWED the import. It was identical when written, so nothing looked wrong;
|
|
104
|
+
then the contract's copy learned to derive box units from the artifact and
|
|
105
|
+
this one kept keying on the layout. The verifier went on refusing the
|
|
106
|
+
pixel-space anchor twin the fix existed to admit — and every contract test
|
|
107
|
+
passed throughout, because they exercise the module that is not the one
|
|
108
|
+
running. Caught by review on the PR that introduced the divergence.
|
|
109
|
+
|
|
110
|
+
Source-level, deliberately: importing the verifier needs numpy, and this
|
|
111
|
+
suite runs on the bare `detector-service` lane where numpy is absent.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
def test_verifier_defines_no_verdict_of_its_own(self):
|
|
115
|
+
path = os.path.join(
|
|
116
|
+
os.path.dirname(__file__), "..", "bench", "verify_litert_twin_parity.py"
|
|
117
|
+
)
|
|
118
|
+
with open(path, encoding="utf-8") as fh:
|
|
119
|
+
src = fh.read()
|
|
120
|
+
self.assertNotIn(
|
|
121
|
+
"\ndef verdict(", src,
|
|
122
|
+
"verify_litert_twin_parity.py defines its own verdict(), which shadows the "
|
|
123
|
+
"imported contract copy — the two then drift silently and the tests exercise "
|
|
124
|
+
"the wrong one",
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
def test_verifier_imports_the_contract_verdict(self):
|
|
128
|
+
path = os.path.join(
|
|
129
|
+
os.path.dirname(__file__), "..", "bench", "verify_litert_twin_parity.py"
|
|
130
|
+
)
|
|
131
|
+
with open(path, encoding="utf-8") as fh:
|
|
132
|
+
src = fh.read()
|
|
133
|
+
self.assertIn("verdict", src.split("from twin_parity_contract import (")[1][:400])
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
if __name__ == "__main__":
|
|
137
|
+
unittest.main()
|
|
@@ -29,7 +29,10 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "bench")
|
|
|
29
29
|
|
|
30
30
|
from regrade_control_selection import ( # noqa: E402
|
|
31
31
|
PRESERVE_ON_BLOCKING,
|
|
32
|
+
RESCORE_SCHEMAS,
|
|
32
33
|
UNRESOLVED,
|
|
34
|
+
apply_rescore,
|
|
35
|
+
load_rescores,
|
|
33
36
|
rateable_controls,
|
|
34
37
|
regrade_capability,
|
|
35
38
|
)
|
|
@@ -116,5 +119,110 @@ class TestRegradeControlSelection(unittest.TestCase):
|
|
|
116
119
|
self.assertEqual(out["status"], "not_affected")
|
|
117
120
|
|
|
118
121
|
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class RescoreReachesTheLadder(unittest.TestCase):
|
|
125
|
+
"""A control added AFTER the screen ran must be able to settle a verdict.
|
|
126
|
+
|
|
127
|
+
Without this the tool has no input for a set the original run never scored,
|
|
128
|
+
so a capability whose verdict a later measurement settles stays `UNRESOLVED`
|
|
129
|
+
for ever while the registry publishes a model — two stores disagreeing about
|
|
130
|
+
whether a capability is sellable, which is the state this input exists to
|
|
131
|
+
end. Proved empirically first: with `fire_firesense` graded `good` and
|
|
132
|
+
visible to it, the tool still returned UNRESOLVED, because the new control
|
|
133
|
+
appears in no candidate's `per_control`.
|
|
134
|
+
"""
|
|
135
|
+
|
|
136
|
+
RESCORE = {("m1", "fire"): {"control": "fresh", "n": 72, "rate_025": 0.82,
|
|
137
|
+
"rate_at_quiet_t": 0.86, "quiet_t": 0.30,
|
|
138
|
+
"source_file": "rescore.json"}}
|
|
139
|
+
CTLQ3 = dict(CTLQ, fresh={"grade": "good"})
|
|
140
|
+
|
|
141
|
+
def test_a_rescore_settles_an_unresolved(self):
|
|
142
|
+
r = regrade_capability(_cap(UNRESOLVED), self.CTLQ3, BAR, self.RESCORE)
|
|
143
|
+
self.assertEqual(r["regraded_decision"], "PUBLISH")
|
|
144
|
+
self.assertEqual(r["winner"]["control"], "fresh")
|
|
145
|
+
self.assertTrue(r["rescored_controls"], "the rescore must be recorded, not silently used")
|
|
146
|
+
|
|
147
|
+
def test_a_rescore_below_the_committed_threshold_is_REFUSED(self):
|
|
148
|
+
"""`off_target_mean` is INHERITED, and only bounds upward.
|
|
149
|
+
|
|
150
|
+
It is `(scores >= t).mean()`, monotone non-increasing in t, so a value
|
|
151
|
+
measured at a LOWER threshold is an upper bound at a higher one. Below
|
|
152
|
+
the committed threshold the bound runs the wrong way and there is no
|
|
153
|
+
off-target number at that operating point — passing the row would assert
|
|
154
|
+
one nobody measured.
|
|
155
|
+
"""
|
|
156
|
+
low = {("m1", "fire"): dict(self.RESCORE[("m1", "fire")], quiet_t=0.10)}
|
|
157
|
+
r = regrade_capability(_cap(UNRESOLVED), self.CTLQ3, BAR, low)
|
|
158
|
+
self.assertEqual(r["regraded_decision"], UNRESOLVED)
|
|
159
|
+
self.assertTrue(any("refused" in n for n in r["rescored_controls"]))
|
|
160
|
+
|
|
161
|
+
def test_an_ungraded_rescored_control_is_refused_not_trusted(self):
|
|
162
|
+
"""An ungraded set falls through both `liveness_only` and `unusable`."""
|
|
163
|
+
with self.assertRaises(SystemExit):
|
|
164
|
+
apply_rescore(_cap(UNRESOLVED)["candidates"][0], self.RESCORE, CTLQ)
|
|
165
|
+
|
|
166
|
+
def test_a_rescore_never_overturns_a_measured_failure_into_silence(self):
|
|
167
|
+
"""A passing rescore may raise a failure to PUBLISH — that is the point —
|
|
168
|
+
but it must do so through the ladder, with a winner and a recorded source."""
|
|
169
|
+
r = regrade_capability(_cap("NO MODEL CLEARS BAR"), self.CTLQ3, BAR, self.RESCORE)
|
|
170
|
+
self.assertEqual(r["regraded_decision"], "PUBLISH")
|
|
171
|
+
self.assertEqual(r["rescored_controls"][0]["source_file"], "rescore.json")
|
|
172
|
+
|
|
173
|
+
def test_an_unreadable_rescore_schema_is_refused_not_skipped(self):
|
|
174
|
+
import json as _json
|
|
175
|
+
import tempfile
|
|
176
|
+
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fh:
|
|
177
|
+
_json.dump({"schema": "something.else/1"}, fh)
|
|
178
|
+
bad = fh.name
|
|
179
|
+
with self.assertRaises(SystemExit):
|
|
180
|
+
load_rescores([bad])
|
|
181
|
+
|
|
182
|
+
def test_a_stale_threshold_control_may_not_carry_the_verdict(self):
|
|
183
|
+
"""THE RATE BELONGS TO THE THRESHOLD IT WAS MEASURED AT.
|
|
184
|
+
|
|
185
|
+
A rescore raises the candidate's `quiet_t` but re-measures only its OWN
|
|
186
|
+
control. Every other rate in `per_control` is still the value at the
|
|
187
|
+
committed, LOWER threshold, where `(scores >= t).mean()` is an UPPER
|
|
188
|
+
BOUND — so reading it as the recall at the new operating point overstates
|
|
189
|
+
recall. Here `big` is stale at 0.90 and the rescored control is a genuine
|
|
190
|
+
0.30 failure: without per-control threshold tracking `argmax(rate_025)`
|
|
191
|
+
picks `big` and manufactures a PUBLISH out of a number measured somewhere
|
|
192
|
+
else.
|
|
193
|
+
"""
|
|
194
|
+
cap = _cap(UNRESOLVED, with_elided=False)
|
|
195
|
+
cap["candidates"][0]["per_control"]["big"] = {
|
|
196
|
+
"n": 9, "rate_025": 0.90, "rate_at_quiet_t": 0.90}
|
|
197
|
+
weak = {("m1", "fire"): dict(self.RESCORE[("m1", "fire")],
|
|
198
|
+
rate_025=0.30, rate_at_quiet_t=0.30)}
|
|
199
|
+
r = regrade_capability(cap, self.CTLQ3, BAR, weak)
|
|
200
|
+
self.assertNotEqual(
|
|
201
|
+
r["regraded_decision"], "PUBLISH",
|
|
202
|
+
"a rate measured at the committed threshold was used to clear the bar "
|
|
203
|
+
"at the higher rescored threshold")
|
|
204
|
+
stale = r["stale_threshold_controls"]
|
|
205
|
+
self.assertTrue(stale, "the stale-threshold refusal must be stated, not silent")
|
|
206
|
+
self.assertEqual(stale[0]["control"], "big")
|
|
207
|
+
self.assertEqual((stale[0]["measured_at_t"], stale[0]["applied_t"]), (0.25, 0.30))
|
|
208
|
+
|
|
209
|
+
def test_a_control_at_the_applied_threshold_still_carries_the_verdict(self):
|
|
210
|
+
"""The positive half: the rescored control IS at the applied threshold,
|
|
211
|
+
so it decides — this is the path fire and smoke actually take."""
|
|
212
|
+
r = regrade_capability(_cap(UNRESOLVED, with_elided=False),
|
|
213
|
+
self.CTLQ3, BAR, self.RESCORE)
|
|
214
|
+
self.assertEqual(r["regraded_decision"], "PUBLISH")
|
|
215
|
+
self.assertEqual(r["winner"]["control"], "fresh")
|
|
216
|
+
|
|
217
|
+
def test_without_a_rescore_no_control_is_stamped_stale(self):
|
|
218
|
+
"""The untouched path must be byte-identical — nothing is stamped, so
|
|
219
|
+
every control remains admissible."""
|
|
220
|
+
a = regrade_capability(_cap("NO MODEL CLEARS BAR", rateable_rate=0.8), CTLQ, BAR)
|
|
221
|
+
self.assertEqual(a["regraded_decision"], "PUBLISH")
|
|
222
|
+
|
|
223
|
+
def test_the_known_schema_list_is_not_empty(self):
|
|
224
|
+
self.assertTrue(RESCORE_SCHEMAS)
|
|
225
|
+
|
|
226
|
+
|
|
119
227
|
if __name__ == "__main__":
|
|
120
228
|
unittest.main()
|
|
@@ -574,7 +574,29 @@ GH_PPE_VINAYAK_L = ModelVocabulary(
|
|
|
574
574
|
),
|
|
575
575
|
)
|
|
576
576
|
|
|
577
|
-
|
|
577
|
+
#: Fire and Smoke Detector — published 2026-08-27 by
|
|
578
|
+
#: scripts/sensing/publish-sensing-model.js. The class list is the artifact's own
|
|
579
|
+
#: (graph metadata key 'names'); the footprint is MEASURED, 123.7031 MiB after a
|
|
580
|
+
#: probe inference via rusage-maxrss, not an estimate from the file size.
|
|
581
|
+
#: Coordinate space MEASURED, not assumed: max|box channel| = 945.3029 over 1670 real frames; / imgsz 640 = 1.4770. pixels at >= 0.2, normalized at <= 0.02, refused in between.
|
|
582
|
+
FIRE_SMOKE_WESLLEYSKAH_11N_CLASS_NAMES: "MappingProxyType[int, str]" = MappingProxyType({
|
|
583
|
+
0: "fire",
|
|
584
|
+
1: "smoke",
|
|
585
|
+
})
|
|
586
|
+
|
|
587
|
+
FIRE_SMOKE_WESLLEYSKAH_11N = ModelVocabulary(
|
|
588
|
+
model_name="fire_smoke_weslleyskah_11n",
|
|
589
|
+
slot_names=FIRE_SMOKE_WESLLEYSKAH_11N_CLASS_NAMES,
|
|
590
|
+
logit_slots=2,
|
|
591
|
+
approx_ram_bytes=129712102,
|
|
592
|
+
onnx_variant=VariantContract(
|
|
593
|
+
output_layout="anchor_4_plus_nc",
|
|
594
|
+
box_units="pixels",
|
|
595
|
+
box_scale_to_pixels=1.0,
|
|
596
|
+
),
|
|
597
|
+
)
|
|
598
|
+
|
|
599
|
+
BUILTIN_VOCABULARIES: "tuple[ModelVocabulary, ...]" = (RFDETR_MEDIUM, GH_FORKLIFT_COMBINE, PPE_VYRA_14CLS_V8M, PPE_CONSTRUCTION_17CLS_V8N, PPE_KILLUMINATI_19CLS_V8S, FALL_1CLS_11X, GH_PPE_VINAYAK_L, FIRE_SMOKE_WESLLEYSKAH_11N,)
|
|
578
600
|
|
|
579
601
|
|
|
580
602
|
class ClassIndex:
|
|
@@ -668,6 +690,7 @@ class ClassIndex:
|
|
|
668
690
|
|
|
669
691
|
|
|
670
692
|
__all__ = [
|
|
693
|
+
"FIRE_SMOKE_WESLLEYSKAH_11N",
|
|
671
694
|
"GH_PPE_VINAYAK_L",
|
|
672
695
|
"FALL_1CLS_11X",
|
|
673
696
|
"PPE_KILLUMINATI_19CLS_V8S",
|