@trillboards/edge-sdk 0.2.106 → 0.2.109

package/deploy/detector/bench/litert-twin-parity.json

@@ -20,7 +20,18 @@
     "its scores agree to 7.2e-08, but row-wise the two disagree and lexsorting by box does",
     "not collapse it. The model returns ZERO detections above 0.25 on 40 real production",
     "frames, so there is no above-threshold sample to compare and row-level parity is",
-    "UNVERIFIED. It is not published. Settle it on a corpus the model actually fires on."
+    "UNVERIFIED. It is not published. Settle it on a corpus the model actually fires on.",
+    "",
+    "2026-08-26: the 22nd twin, gh_fall_tonlong_v2, is the first whose LiteRT input is",
+    "NHWC [1,640,640,3]. The line above saying an ultralytics detect export is NOT NHWC",
+    "described the TOOLCHAIN the first 21 were exported with, not LiteRT: onnx2tf",
+    "transposes to channels-last by design and ultralytics 8.3.145 leaves it that way.",
+    "verify_litert_twin_parity.py used to ASSERT NCHW and refused a correct twin; it now",
+    "reads the input layout off the graph -- the same rule it already applied to outputs",
+    "-- and feeds each runtime the SAME IMAGE in the layout that runtime declares. The",
+    "warning that motivated the assertion still stands: transposing because 'LiteRT is",
+    "channels-last' feeds a rotated image to a model that wanted NCHW, and every residual",
+    "then reads as export noise."
   ],
   "measured_at": "2026-08-25",
   "tolerances": {
@@ -308,6 +319,21 @@
       "reason": "ok",
       "onnx": "weapon_gunknife_11n.onnx",
       "litert": "weapon_gunknife_11n.tflite"
+    },
+    {
+      "layout": "anchor_4_plus_nc",
+      "imgsz": 640,
+      "box_scale_applied": 640.0,
+      "box_residual_px": 0.0007171630859375,
+      "box_residual_unscaled": 646.9068156480789,
+      "cls_residual": 1.3665612286217765e-07,
+      "pass": true,
+      "reason": "ok",
+      "onnx": "gh_fall_tonlong_v2.onnx",
+      "litert": "gh_fall_tonlong_v2.tflite",
+      "litert_input_layout": "nhwc",
+      "measured_at": "2026-08-26",
+      "note": "The 22nd twin, added 2026-08-26, and the FIRST whose LiteRT input is NHWC. gh_fall_tonlong had no parity measurement at all until now: upstream ships `Violence` at slot 0 and `violence` at slot 6, normalize_class_name folds them to one name and ModelVocabulary refuses at construction, so neither format could be probed. Both were re-exported from ok.pt (sha256 1e12d89198e28d7c650ba86886c511fecd9f70b9d7f8bf985e76704690e58dd9) with the slots disambiguated by INDEX -- violence_slot0 / violence_slot6 -- which preserves upstream truth and asserts no semantics we have not measured."
     }
   ]
 }

package/deploy/detector/bench/regrade_control_selection.py

@@ -0,0 +1,371 @@
+#!/usr/bin/env python3
+"""Re-apply the venue adjudication's own bar with the control-selection fix.
+
+WHY THIS EXISTS
+---------------
+`venue_adjudication.py` chose the control a verdict rests on with
+`argmax(rate_025)` over every set listed for the class. That is a selector with no
+access to the contact-sheet grade, so it reliably picks the set that flatters the
+model. On 2026-08-26 it picked `fire_generic` -- n=5, graded `good_small_n`, whose
+own note reads "n=5 so only a coarse liveness signal" -- over `fire_cctv`, n=39,
+graded `good`, "realistic scale for a venue camera", scored in the same run against
+the same artifact at the same threshold. `fire` and `smoke` were awarded PUBLISH on
+the 5-frame set.
+
+The tool is fixed. Re-running it end to end needs the per-frame score matrices
+(`<alias>.npz` + `combo.json`) from the eval box, which are not committed. This
+re-applies the SAME bar to the numbers the artifact already carries, so the
+corrected verdict is derivable from what is in the repo -- and, crucially, marks
+each capability `proven` or `needs_remeasure` rather than pretending the elided
+rows are known.
+
+WHAT IT CANNOT DO, AND SAYS SO
+------------------------------
+`venue_adjudication.py` elides `per_control` for candidates ranked 5+ that carry no
+shipped bytes. For those rows only the WINNING set's rate survives. Where the
+winning set was liveness-only, that candidate's rateable-control rate is unknown
+and the capability is reported `needs_remeasure` with the candidate named. An
+unknown is not a pass and it is not a fail; it is the reason to re-run.
+
+    python3 regrade_control_selection.py \
+        --adjudication trillboard-api/scripts/ai/eval-baselines/venue-adjudication-2026-08-26.json \
+        --control-quality trillboard-api/scripts/ai/eval-baselines/positive-control-quality-2026-08-25.json \
+        --out trillboard-api/scripts/ai/eval-baselines/venue-adjudication-control-regrade-2026-08-26.json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+
+#: Grades whose own note says the set cannot rate a class. Kept in step with
+#: `venue_adjudication.py::LIVENESS_ONLY_GRADES`; the test pins them equal.
+LIVENESS_ONLY_GRADES = ("good_small_n", "weak")
+UNUSABLE_GRADES = ("DISCARDED", "POOR", "CONTAMINATED")
+
+#: What a capability becomes when the PUBLISH is provably withdrawn but what
+#: replaces it is not. NOT a verdict a model may ship on, and NOT a failure: the
+#: elided rows could still clear the bar, and writing "NO MODEL CLEARS BAR" over
+#: them would be the same mistake in the other direction -- a definitive claim
+#: standing on evidence that was never committed.
+UNRESOLVED = "UNRESOLVED PENDING REMEASURE"
+
+#: Priors that an unknown may hold open. A PUBLISH because it is exactly what this
+#: tool exists to withdraw; `UNRESOLVED` because this tool writes it, and a run
+#: over its own output must not harden "not known" into "known bad".
+PRESERVE_ON_BLOCKING = ("PUBLISH", UNRESOLVED)
+
+
+def rateable_controls(per_control: dict, ctlq: dict) -> dict:
+    """The controls that may carry a verdict: graded, usable, not liveness-only."""
+    out = {}
+    for name, row in (per_control or {}).items():
+        grade = (ctlq.get(name) or {}).get("grade")
+        if grade in LIVENESS_ONLY_GRADES or grade in UNUSABLE_GRADES:
+            continue
+        out[name] = dict(row, grade=grade)
+    return out
+
+
+def regrade_capability(cap: dict, ctlq: dict, bar: dict) -> dict:
+    """Re-apply the bar to one capability using only rateable controls."""
+    recall_bar = bar["recall_retention"]
+    off_bar = bar["off_target_mean"]
+    prior = cap["decision"].get("decision")
+
+    # `decide()` resolves two branches BEFORE it ever looks at a control, and both
+    # are properties of the negative set. Re-deriving them here would only
+    # reproduce them from less information, and getting them subtly wrong is how a
+    # regrade starts reporting movement it did not cause -- the first run of this
+    # tool "moved" eight capabilities that have no control involved at all.
+    if not cap["candidates"] or prior in ("NO CANDIDATE", "NOT SCREENABLE HERE"):
+        return {
+            "capability": cap["capability"],
+            "prior_decision": prior,
+            "regraded_decision": prior,
+            "changed": False,
+            "status": "not_affected",
+            "why": "resolved before any control is read: %s" % (prior,),
+        }
+
+    rows, unknown = [], []
+    for cand in cap["candidates"]:
+        elided = "_elided" in (cand.get("by_bucket") or {})
+        rate = rateable_controls(cand.get("per_control"), ctlq)
+        if not rate:
+            # No per-control detail survived the elision. If the set this row was
+            # decided on is itself rateable, the surviving number IS the rateable
+            # number and the row is knowable; otherwise it is not.
+            best = cand.get("best_control")
+            bg = (ctlq.get(best) or {}).get("grade") if best else None
+            if best and bg not in LIVENESS_ONLY_GRADES and bg not in UNUSABLE_GRADES:
+                rows.append({
+                    "model": cand["model"], "class": cand["class"],
+                    "control": best, "control_n": cand.get("control_n"),
+                    "grade": bg,
+                    "rate_at_quiet_t": cand.get("control_rate_at_quiet_t"),
+                    "rate_025": cand.get("control_rate_025"),
+                    "off_target_mean": cand.get("off_target_mean"),
+                    "quiet_t": cand.get("quiet_t"),
+                    "n_valid_negatives": cand.get("n_valid_negatives"),
+                    "source": "winning set was already rateable",
+                })
+            else:
+                unknown.append({
+                    "model": cand["model"], "class": cand["class"],
+                    "decided_on": best, "grade": bg,
+                    "why_unknown": ("per_control elided for this rank and the set it was "
+                                    "decided on cannot rate a class, so its rateable-control "
+                                    "rate was never committed"),
+                    "elided": elided,
+                })
+            continue
+        # argmax(rate_025) among the rateable, exactly as the fixed tool does.
+        name = max(rate, key=lambda k: (rate[k]["rate_025"] or 0))
+        row = rate[name]
+        rows.append({
+            "model": cand["model"], "class": cand["class"],
+            "control": name, "control_n": row["n"], "grade": row["grade"],
+            "rate_at_quiet_t": row["rate_at_quiet_t"], "rate_025": row["rate_025"],
+            "off_target_mean": cand.get("off_target_mean"),
+            "quiet_t": cand.get("quiet_t"),
+            "n_valid_negatives": cand.get("n_valid_negatives"),
+            "also_scored": {k: {"n": v["n"], "grade": (ctlq.get(k) or {}).get("grade"),
+                                "rate_at_quiet_t": v["rate_at_quiet_t"]}
+                            for k, v in (cand.get("per_control") or {}).items() if k != name},
+            "source": "recomputed from committed per_control",
+        })
+
+    live = [r for r in rows if (r["rate_025"] or 0) > 0]
+    retained = [r for r in live if (r["rate_at_quiet_t"] or 0) >= recall_bar]
+    ok = [r for r in retained
+          if r["off_target_mean"] is not None and r["off_target_mean"] <= off_bar]
+
+    if ok:
+        # The artifact's own ordering is preserved: candidates arrive sorted.
+        win = ok[0]
+        verdict = "PUBLISH"
+        why = ("clears the bar on %s (n=%d, grade %s): keeps %.1f%% at conf %s"
+               % (win["control"], win["control_n"], win["grade"],
+                  100 * (win["rate_at_quiet_t"] or 0), win["quiet_t"]))
+    elif live:
+        best = max(live, key=lambda r: (r["rate_at_quiet_t"] or 0))
+        verdict = "NO MODEL CLEARS BAR"
+        why = ("no candidate keeps >=%.0f%% of a control that can rate a class. Best is "
+               "%s / '%s' at %.1f%% of %s (n=%d, grade %s)."
+               % (100 * recall_bar, best["model"], best["class"],
+                  100 * (best["rate_at_quiet_t"] or 0), best["control"],
+                  best["control_n"], best["grade"]))
+        win = None
+    else:
+        verdict, why, win = "UNINFORMATIVE", "no candidate fires on a rateable control", None
+
+    # An unknown row can only ever ADD a passing candidate, so it can overturn a
+    # failing verdict and can never overturn a PUBLISH.
+    blocking = unknown if verdict != "PUBLISH" else []
+
+    # ...and a verdict an unknown could overturn must not be stated as if it could
+    # not. Withdrawing the PUBLISH is CERTAIN -- it rested on a set that cannot
+    # rate a class, and no re-measurement changes that. Replacing it with a
+    # definitive failure is NOT certain, because an elided candidate could still
+    # clear the bar. Two different claims; only the first is proved here, and
+    # asserting the second would be the same error in the other direction.
+    #
+    # Gated on the PRIOR. Where the prior was already a MEASURED failure, the
+    # original run reached it with MORE controls and better data than this tool
+    # has for the elided rows, so overwriting it with `UNRESOLVED` would be this
+    # tool asserting its own blindness over a measurement. Those rows still carry
+    # `status: needs_remeasure`; their verdict is left alone.
+    #
+    # `UNRESOLVED` is in the set because THIS TOOL WRITES IT. Without it, running
+    # `--apply` twice would see `prior == UNRESOLVED`, skip this branch, and
+    # overwrite its own careful "not known" with a definitive `NO MODEL CLEARS
+    # BAR` -- the exact claim the first run refused to make, on the same
+    # evidence. The guard would not catch it either: it audits PUBLISH rows only.
+    # `--selftest` pins the idempotence.
+    if blocking and verdict == "NO MODEL CLEARS BAR" and prior in PRESERVE_ON_BLOCKING:
+        why = ("the prior verdict is WITHDRAWN -- it rested on a control that cannot rate a "
+               "class, which no re-measurement changes. What REPLACES it is not yet known: "
+               + why
+               + (" %d candidate(s) had their rateable-control rate elided and could still "
+                  "clear the bar (%s). Re-run venue_adjudication.py against the score "
+                  "matrices to settle it."
+                  % (len(blocking), ", ".join("%s/%s" % (b["model"], b["class"])
+                                              for b in blocking))))
+        verdict = UNRESOLVED
+
+    return {
+        "capability": cap["capability"],
+        "prior_decision": prior,
+        "prior_control": cap["decision"].get("control"),
+        "prior_control_n": cap["decision"].get("control_n"),
+        "prior_control_grade": (ctlq.get(cap["decision"].get("control")) or {}).get("grade"),
+        "prior_events_lost": (cap["decision"].get("gate") or {}).get("events_lost"),
+        "regraded_decision": verdict,
+        "changed": bool(prior and prior != verdict),
+        "status": "needs_remeasure" if blocking else "proven",
+        "why": why,
+        "winner": win,
+        "candidates_regraded": len(rows),
+        "candidates_unknown": blocking,
+    }
+
+
+def _selftest() -> None:
+    """Pin the two things a reader cannot see by reading the ladder.
+
+    1. IDEMPOTENCE. `--apply` writes `UNRESOLVED` into the artifact it reads. A
+       second run must reproduce it, not harden it into a failure.
+    2. A MEASURED failure is left alone. A prior that was already
+       `NO MODEL CLEARS BAR` came from a run with more controls than this tool
+       holds for an elided row, and must not be softened to `UNRESOLVED` either.
+    """
+    ctlq = {"big": {"grade": "good"}, "tiny": {"grade": "good_small_n"}}
+    bar = {"recall_retention": 0.5, "off_target_mean": 0.1}
+
+    def cap(prior):
+        return {
+            "capability": "fire",
+            "decision": {"decision": prior, "control": "tiny", "control_n": 5},
+            "candidates": [
+                {   # a candidate that fails on the rateable set
+                    "model": "m1", "class": "fire", "quiet_t": 0.25,
+                    "off_target_mean": 0.0, "n_valid_negatives": 100,
+                    "by_bucket": {}, "best_control": "big", "control_n": 9,
+                    "control_rate_025": 0.4, "control_rate_at_quiet_t": 0.4,
+                    "per_control": {"big": {"n": 9, "rate_025": 0.4, "rate_at_quiet_t": 0.4},
+                                    "tiny": {"n": 5, "rate_025": 0.9, "rate_at_quiet_t": 0.9}},
+                },
+                {   # ...and one whose rateable rate was elided
+                    "model": "m2", "class": "fire", "quiet_t": 0.25,
+                    "off_target_mean": 0.0, "n_valid_negatives": 100,
+                    "by_bucket": {"_elided": "summary row"},
+                    "best_control": "tiny", "control_n": 5,
+                    "control_rate_025": 0.9, "control_rate_at_quiet_t": 0.9,
+                    "per_control": {},
+                },
+            ],
+        }
+
+    first = regrade_capability(cap("PUBLISH"), ctlq, bar)
+    assert first["regraded_decision"] == UNRESOLVED, first["regraded_decision"]
+    assert first["candidates_unknown"], "the elided row must be reported"
+
+    second = regrade_capability(cap(first["regraded_decision"]), ctlq, bar)
+    assert second["regraded_decision"] == UNRESOLVED, (
+        "re-running over this tool's own output hardened an unknown into %r"
+        % second["regraded_decision"])
+    assert second["changed"] is False, "a settled UNRESOLVED must not read as changed"
+
+    measured = regrade_capability(cap("NO MODEL CLEARS BAR"), ctlq, bar)
+    assert measured["regraded_decision"] == "NO MODEL CLEARS BAR", measured["regraded_decision"]
+
+    print("selftest OK: idempotent on UNRESOLVED; a measured failure is left alone")
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    if "--selftest" in __import__("sys").argv[1:]:
+        _selftest()
+        return
+    ap.add_argument("--adjudication", required=True)
+    ap.add_argument("--control-quality", required=True)
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--apply", action="store_true",
+                    help="also write the corrected verdict back into --adjudication. The "
+                         "adjudication is the record; leaving a verdict in it that its own "
+                         "numbers refute is a second source of truth, not a safer one.")
+    args = ap.parse_args()
+
+    adj = json.load(open(args.adjudication))
+    ctlq = json.load(open(args.control_quality))
+    bar = adj["bar"]
+
+    rows = [regrade_capability(c, ctlq, bar) for c in adj["capabilities"]]
+    changed = [r for r in rows if r["changed"]]
+
+    # The regrade artifact is a record OF A TRANSITION. Once `--apply` has run, the
+    # adjudication no longer holds the verdict that moved, so a second run reads
+    # nothing to change and would write an empty `changed_list` over the record of
+    # what did. The adjudication itself is idempotent (proved by `--selftest` and
+    # by re-running `--apply`, which leaves it byte-identical); this file is not,
+    # and erasing it silently would leave the corrected artifact with nothing
+    # beside it saying what it was corrected FROM.
+    if not changed and os.path.exists(args.out):
+        try:
+            prev = json.load(open(args.out))
+        except (OSError, ValueError):
+            prev = {}
+        if (prev.get("summary") or {}).get("changed"):
+            raise SystemExit(
+                "refusing to overwrite %s: it records %d capability transition(s) and this "
+                "run found none, because the adjudication it read has ALREADY been corrected. "
+                "That is the expected result of re-running, not a reason to erase the record. "
+                "Point --out somewhere else, or re-run against an uncorrected adjudication."
+                % (args.out, (prev["summary"] or {})["changed"]))
+
+    json.dump({
+        "schema": "trillboards.venue-adjudication-control-regrade/1",
+        "regrades": adj["schema"] + " @ " + adj["negative_set"],
+        "_what": "the SAME bar, re-applied with the control-selection fix: a verdict may "
+                 "only rest on a control whose contact-sheet grade can rate a class.",
+        "_not_a_re_run": "the per-frame scores are unchanged and are NOT re-derived here. "
+                         "This re-reads the numbers the adjudication already committed. A "
+                         "capability marked `needs_remeasure` has a candidate whose "
+                         "rateable-control rate was elided and must come from a re-run.",
+        "liveness_only_grades": list(LIVENESS_ONLY_GRADES),
+        "unusable_grades": list(UNUSABLE_GRADES),
+        "bar": bar,
+        "summary": {
+            "capabilities": len(rows),
+            "changed": len(changed),
+            "changed_list": [{"capability": r["capability"],
+                              "was": r["prior_decision"], "now": r["regraded_decision"],
+                              "status": r["status"]} for r in changed],
+        },
+        "capabilities": rows,
+    }, open(args.out, "w"), indent=1)
+
+    if args.apply and changed:
+        by_cap = {r["capability"]: r for r in changed}
+        for cap in adj["capabilities"]:
+            r = by_cap.get(cap["capability"])
+            if not r:
+                continue
+            was = dict(cap["decision"])
+            cap["decision"] = {
+                "decision": r["regraded_decision"],
+                "why": r["why"],
+                # The gate is a promise computed from a recall this verdict no
+                # longer accepts. Keeping it would leave an events-lost figure in
+                # the file with nothing behind it.
+                "gate": None,
+                "corrected": {
+                    "on": "2026-08-26",
+                    "by": "deploy/detector/bench/regrade_control_selection.py --apply",
+                    "was": was,
+                    "ships": False,
+                    "why_corrected": (
+                        "the verdict rested on `%s` (n=%s, grade %s), a set whose own "
+                        "contact-sheet note says it can only show liveness. Re-applying "
+                        "this run's OWN bar to the controls that can rate a class "
+                        "overturns it." % (was.get("control"), was.get("control_n"),
+                                           r.get("prior_control_grade"))),
+                    "needs_remeasure": r["candidates_unknown"],
+                    "regrade_artifact": args.out,
+                },
+            }
+            cap["changed"] = bool(cap.get("prior_decision_taxi_negatives")
+                                  and cap["prior_decision_taxi_negatives"]
+                                  != r["regraded_decision"])
+        json.dump(adj, open(args.adjudication, "w"), indent=1)
+
+    print(json.dumps({"capabilities": len(rows), "changed": len(changed),
+                      "applied": bool(args.apply and changed),
+                      "changed_list": [(r["capability"], r["prior_decision"],
+                                        r["regraded_decision"], r["status"])
+                                       for r in changed]}, indent=1))
+
+
+if __name__ == "__main__":
+    main()