@trillboards/edge-sdk 0.2.73 → 0.2.76

package/deploy/detector/trillboards_detector/loader.py

@@ -0,0 +1,257 @@
+"""Model lifecycle: load it, prove it, watch for the OTA, and say which.
+
+WHY LOADING IS A BACKGROUND STATE MACHINE AND NOT A STARTUP STEP
+---------------------------------------------------------------
+Three constraints, none optional:
+
+1. THE ARTIFACT MAY NOT BE THERE YET. It is OTA'd, so on a fresh install the
+   supervisor starts this service before edge-core's `ModelManager` has fetched
+   anything. A service that exits when the file is missing gets restarted
+   forever by launchd/systemd/Task Scheduler — the crash-loop each of those
+   definitions has a ThrottleInterval for. It must START and report UNHEALTHY.
+
+2. THE ARTIFACT ARRIVES LATER, WITHOUT ANYONE RESTARTING US. `ModelManager`
+   writes `.tmp` then renames. Nothing tells this process. So the file is
+   polled, and a size/mtime change reloads the session — the same problem
+   `gemmaModelOta.ts` solves by restarting `litert-lm serve` on `modelUpdated`,
+   solved here without needing the connector to reach in.
+
+3. HEALTH MUST REFLECT A LOAD THAT ACTUALLY HAPPENED. This codebase has been
+   burned twice by the opposite: `/v1/models` answers 200 without ever building
+   an engine, so a UTF-8 BOM in litert-lm's config.json 500'd every frame on
+   every Windows connector while the installer's health check AND its
+   CPU-fallback re-check both passed. `docs/runbooks/cctv-connector-runtime-edge-cases.md`
+   is a catalogue of the same shape. So the load runs a REAL forward pass on a
+   synthetic frame before declaring ready, and `/health` reports that outcome
+   rather than re-deriving anything — which is also how `/health` stays cheap
+   and inference-free.
+"""
+
+from __future__ import annotations
+
+import io
+import threading
+import time
+from pathlib import Path
+
+from .errors import DetectorNotReady
+from .session import DetectionBatch, ModelContractError, OnnxDetector
+
+STATE_LOADING = "loading"
+STATE_READY = "ready"
+STATE_UNAVAILABLE = "unavailable"
+
+DEFAULT_POLL_INTERVAL_S = 30.0
+
+
+def _warmup_jpeg() -> bytes:
+    """A tiny real JPEG. Small enough to be free, real enough to decode.
+
+    Mirrors `setup-litertlm.sh`'s inline probe image, for the same reason it
+    exists there: a capability is established by ATTEMPTING THE OPERATION, and
+    a file that merely exists proves nothing.
+    """
+    from PIL import Image
+
+    buf = io.BytesIO()
+    Image.new("RGB", (64, 48), (128, 128, 128)).save(buf, format="JPEG", quality=70)
+    return buf.getvalue()
+
+
+class DetectorLoader:
+    """Owns the session and the one word that describes it."""
+
+    def __init__(
+        self,
+        model_path: Path,
+        model_name: str,
+        intra_op_threads: "int | None" = None,
+        poll_interval_s: float = DEFAULT_POLL_INTERVAL_S,
+        log=print,
+    ):
+        self.model_path = model_path
+        self.model_name = model_name
+        self._intra_op_threads = intra_op_threads
+        self._poll_interval_s = poll_interval_s
+        self._log = log
+
+        self._lock = threading.Lock()
+        self._detector: "OnnxDetector | None" = None
+        self._state = STATE_LOADING
+        self._reason: "str | None" = None
+        self._stamp: "tuple[int, int] | None" = None
+        self._warmup_ms: "float | None" = None
+        self._loaded_at: "float | None" = None
+
+        self._degenerate_dropped = 0
+        self._unnamed_class_dropped = 0
+        self._detect_count = 0
+        self._detect_failures = 0
+
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._run, name="detector-loader", daemon=True)
+
+    def start(self) -> None:
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._stop.set()
+
+    def join(self, timeout: "float | None" = None) -> None:
+        self._thread.join(timeout)
+
+    # ── what the server reads ────────────────────────────────────────────────
+
+    @property
+    def state(self) -> str:
+        with self._lock:
+            return self._state
+
+    @property
+    def reason(self) -> "str | None":
+        with self._lock:
+            return self._reason
+
+    def info(self) -> dict:
+        with self._lock:
+            d = self._detector
+            payload = {
+                "state": self._state,
+                "model": self.model_name,
+                "model_path": str(self.model_path),
+                "detect_count": self._detect_count,
+                "detect_failures": self._detect_failures,
+                # Published because a drop nobody can see is the failure this
+                # whole stack keeps repeating. Both are normally 0.
+                "degenerate_boxes_dropped": self._degenerate_dropped,
+                "unnamed_class_dropped": self._unnamed_class_dropped,
+            }
+            if self._reason:
+                payload["reason"] = self._reason
+            if self._warmup_ms is not None:
+                payload["warmup_inference_ms"] = round(self._warmup_ms, 1)
+            if self._loaded_at is not None:
+                payload["loaded_at"] = self._loaded_at
+            if d is not None:
+                payload["input"] = f"{d.spec.width}x{d.spec.height}"
+                payload["classes"] = d.spec.num_classes
+                payload["providers"] = d.spec.providers
+            return payload
+
+    def detect(self, jpeg: bytes, min_confidence: float, wanted: "set[str] | None") -> DetectionBatch:
+        with self._lock:
+            detector, reason = self._detector, self._reason
+        if detector is None:
+            raise DetectorNotReady(reason or "model is not loaded")
+        try:
+            batch = detector.detect(jpeg, min_confidence, wanted)
+        except Exception:
+            with self._lock:
+                self._detect_failures += 1
+            raise
+        with self._lock:
+            self._detect_count += 1
+            self._degenerate_dropped += batch.degenerate_dropped
+            self._unnamed_class_dropped += batch.unnamed_class_dropped
+        if batch.degenerate_dropped or batch.unnamed_class_dropped:
+            self._log(
+                f"[detector] dropped {batch.degenerate_dropped} zero-extent box(es) and "
+                f"{batch.unnamed_class_dropped} unnamed-class detection(s) from one frame"
+            )
+        return batch
+
+    # ── the loop ─────────────────────────────────────────────────────────────
+
+    def _run(self) -> None:
+        while not self._stop.is_set():
+            try:
+                self._tick()
+            except Exception as err:  # never let the watcher die
+                self._set(STATE_UNAVAILABLE, f"loader crashed: {err}")
+            self._stop.wait(self._poll_interval_s)
+
+    def _tick(self) -> None:
+        stamp = self._file_stamp()
+        if stamp is None:
+            if self._state != STATE_UNAVAILABLE or self._detector is not None:
+                self._log(f"[detector] model artifact not present at {self.model_path}")
+            self._drop(
+                f"model artifact not found at {self.model_path}; it arrives over the air "
+                "and this service will pick it up within "
+                f"{int(self._poll_interval_s)}s of it landing"
+            )
+            return
+
+        with self._lock:
+            unchanged = self._detector is not None and self._stamp == stamp
+        if unchanged:
+            return
+
+        if self._detector is not None:
+            self._log(f"[detector] artifact changed on disk — reloading {self.model_path}")
+        self._load(stamp)
+
+    def _load(self, stamp: "tuple[int, int]") -> None:
+        self._set(STATE_LOADING, None)
+        try:
+            detector = OnnxDetector(self.model_path, self.model_name, self._intra_op_threads)
+        except ModelContractError as err:
+            self._drop(f"artifact loaded but does not match the detector contract: {err}")
+            return
+        except Exception as err:
+            self._drop(f"could not load {self.model_path}: {err}")
+            return
+
+        # PROVE IT. A session that constructs is not a session that infers.
+        try:
+            started = time.perf_counter()
+            detector.detect(_warmup_jpeg(), 0.99, None)
+            warmup_ms = (time.perf_counter() - started) * 1000.0
+        except Exception as err:
+            self._drop(f"artifact loaded but a probe inference failed: {err}")
+            return
+
+        with self._lock:
+            self._detector = detector
+            self._stamp = stamp
+            self._state = STATE_READY
+            self._reason = None
+            self._warmup_ms = warmup_ms
+            self._loaded_at = time.time()
+        self._log(
+            f"[detector] ready — {self.model_name} at {self.model_path} "
+            f"({detector.spec.width}x{detector.spec.height}, {detector.spec.num_classes} classes, "
+            f"probe {warmup_ms:.0f} ms, providers {detector.spec.providers})"
+        )
+
+    def _file_stamp(self) -> "tuple[int, int] | None":
+        try:
+            st = self.model_path.stat()
+        except OSError:
+            return None
+        if st.st_size == 0:
+            return None
+        return (st.st_size, st.st_mtime_ns)
+
+    def _drop(self, reason: str) -> None:
+        with self._lock:
+            self._detector = None
+            self._stamp = None
+            self._warmup_ms = None
+            self._loaded_at = None
+            self._state = STATE_UNAVAILABLE
+            self._reason = reason
+
+    def _set(self, state: str, reason: "str | None") -> None:
+        with self._lock:
+            self._state = state
+            self._reason = reason
+
+
+__all__ = [
+    "DetectorLoader",
+    "DetectorNotReady",
+    "STATE_LOADING",
+    "STATE_READY",
+    "STATE_UNAVAILABLE",
+]

package/deploy/detector/trillboards_detector/modelpath.py

@@ -0,0 +1,74 @@
+"""Where the detector artifact lives — ONE canonical home, on every platform.
+
+THE ARTIFACT IS OTA'd, NEVER BUNDLED. `LocalDetectorClient.ts` records why: the
+macOS code-only build is hard-capped at `MAX_SLIM_PKG_BYTES` = 250 MiB
+(`build-macos-pkg.sh`) and the code slice is already ~207 MB, so staging a
+~128 MiB ONNX into the payload makes the build `die`. It arrives the way Gemma's
+weights do — through edge-core's `ModelManager`, atomic `.tmp` + rename.
+
+So this module has exactly one job: name the path `ModelManager` writes to, in a
+form both the Python service and the Node connector derive rather than repeat.
+It deliberately mirrors `gemmaModelOta.ts`'s
+`resolveLiteRtLmStoreModelPath` — same shape, one directory per model name, a
+fixed file name inside it:
+
+    ${TRILLBOARDS_DATA_DIR:-~/.trillboards}/models/<model-name>/model.onnx
+
+ONE DIRECTORY PER MODEL NAME, NOT PER VERSION — deliberately. The model OTA
+already orphans 752 MiB per update because the cleaner's `includes('xnnpack')`
+misses the `mldrift_*_cache.bin` family
+(`docs/runbooks/cctv-connector-runtime-edge-cases.md`), and there is no GC for
+abandoned model dirs anywhere on this path. A version-named directory would add
+a second, larger leak — 128 MiB per release on a box that has no disk-space
+preflight either. `ModelManager` rewrites this one file in place, so an update
+reclaims the old bytes by definition.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+#: The artifact this service is built around. `docs/runbooks/cctv-detector-tier.md`
+#: §6: RF-DETR Medium, fp32 ONNX, NO quantisation — INT8 loses 31.7 points of
+#: recall silently AND is slower, and `.export(quantization='int8')` is a silent
+#: no-op that writes an all-float file anyway.
+DEFAULT_MODEL_NAME = "rfdetr_medium"
+
+#: Fixed file name inside the per-model directory. Matches litert-lm's
+#: `models/<id>/model.litertlm` convention so an operator reading one store can
+#: read the other.
+MODEL_FILE_NAME = "model.onnx"
+
+
+def resolve_data_dir(env: "os._Environ[str] | dict[str, str] | None" = None) -> Path:
+    """The connector's data dir, matching `config.ts`'s `DEFAULTS.dataDir`.
+
+    `path.join(process.env.HOME || process.env.USERPROFILE || '/tmp',
+    '.trillboards')` — the same three-step fallback, in the same order, because
+    a service that resolves a DIFFERENT directory from the process that
+    downloads the model is a detector that is permanently unhealthy on a machine
+    whose artifact is present.
+    """
+    e = os.environ if env is None else env
+    override = (e.get("TRILLBOARDS_DATA_DIR") or "").strip()
+    if override:
+        return Path(override)
+    home = (e.get("HOME") or "").strip() or (e.get("USERPROFILE") or "").strip() or "/tmp"
+    return Path(home) / ".trillboards"
+
+
+def resolve_model_path(
+    model_name: str = DEFAULT_MODEL_NAME,
+    env: "os._Environ[str] | dict[str, str] | None" = None,
+) -> Path:
+    """Absolute path to the ONNX artifact.
+
+    `TRILLBOARDS_DETECTOR_MODEL` overrides it outright — that is the escape
+    hatch for a bench run, and the one the integration test uses.
+    """
+    e = os.environ if env is None else env
+    override = (e.get("TRILLBOARDS_DETECTOR_MODEL") or "").strip()
+    if override:
+        return Path(override)
+    return resolve_data_dir(e) / "models" / model_name / MODEL_FILE_NAME