@trillboards/edge-sdk 0.2.81 → 0.2.82

package/deploy/detector/trillboards_detector/registry.py

@@ -0,0 +1,836 @@
+"""Many models on one box: who is resident, who gets evicted, and who answers.
+
+WHY A REGISTRY AND NOT A SECOND `DetectorLoader`
+------------------------------------------------
+`docs/runbooks/sensing-model-library.md` §3 decides the shape and this module
+implements it. Three of its sentences are the whole specification:
+
+  * "The loader holds a name -> session map with a class index built at load."
+  * "Models load LAZILY on first request for their classes, with LRU eviction
+    under a stated RAM budget."
+  * "A request spanning two models runs both and merges."
+
+§2 is why it cannot be eager. The binding field box is a Ryzen 5 7640HS with a
+measured 9.8 GB, carrying 6 of the 9 spec-reporting screens; Gemma's working set
+alone is 3.31 GB and RF-DETR Medium peaks at 632 MB. "The sum has to fit, not
+each member" — and a selector that approves each model independently approves a
+set that OOM-kills the box, where the failure then looks like a camera problem.
+
+STDLIB ONLY, LIKE `server.py`, AND FOR THE SAME REASON. The ONNX session arrives
+through an injected factory, so every rule below — routing, admission,
+eviction, the refusals — is exercisable on a bare interpreter with no
+onnxruntime, no numpy, no Pillow and no 128 MiB artifact. That is every CI
+runner. A residency policy that could only be tested on a box with the model
+present is a residency policy nothing tests, which is the same argument
+`test_server.py` already makes for the wire.
+
+WHAT IS PINNED AND WHY IT IS NOT "EAGER LOADING BY ANOTHER NAME"
+---------------------------------------------------------------
+The DEFAULT model — the general cascade gate — is loaded by the watcher and is
+never evicted. Specialists are lazy. That is not a softening of §3; it is what
+makes §3 coexist with the health contract that shipped first:
+
+  * `/health` promises 200 only when "a load completed AND a probe inference
+    succeeded". Under pure lazy loading nothing is resident at startup, so
+    `/health` would answer 503 until the first request — and `LocalDetectorClient
+    .isHealthy()` returning false is precisely how the connector decides the
+    detector is down and sends every frame to the VLM. The cascade would then
+    never make its first call, and the 84.7%-of-calls saving would silently
+    never happen on any box.
+  * §3's arithmetic is about the SET: "loading all of them eagerly does not fit
+    the 9.8 GB box". One 632 MB model does, and no budget that can hold anything
+    at all cannot hold it.
+  * Evicting the general detector to make room for a specialist and reloading it
+    on the next frame is pure thrash — it is on the hot path for essentially
+    every profile.
+
+CONCURRENCY. One coarse admission lock covers plan -> evict -> load, so two
+requests can never each admit a model and together exceed the budget. Inference
+itself runs OUTSIDE it, holding only strong references to the sessions, and a
+model with a request in flight is not evictable — otherwise the accounting would
+be describing a set that is not the resident set. `/health` takes a different,
+short-lived lock and therefore never queues behind a load, which is the property
+that keeps it cheap.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import threading
+import time
+from pathlib import Path
+
+from .detections import DetectionBatch, merge_batches
+from .errors import DetectorNotReady, ModelContractError, UnknownClasses
+from .labels import ClassIndex, ModelVocabulary, normalize_class_name
+
+#: Per-model residency states. `absent` and `idle` are new and they are
+#: different facts: `absent` means the artifact is not on disk (it is OTA'd, so
+#: on a fresh install that is the normal starting state), `idle` means the file
+#: is there and no session is built — never loaded, or evicted to make room.
+#: Collapsing them would make "the OTA has not landed" indistinguishable from
+#: "the LRU reclaimed it", which are opposite problems.
+STATE_ABSENT = "absent"
+STATE_IDLE = "idle"
+STATE_LOADING = "loading"
+STATE_READY = "ready"
+STATE_UNAVAILABLE = "unavailable"
+
+#: THIS SERVICE OWNS NO RAM POLICY, AND MUST NOT.
+#:
+#: There is already an allocation authority and it is server-side:
+#: `sensingProfileService.selectModelForDeviceTier` picks a model from
+#: `deviceRamMb` + `deviceTier`, and `registerInternalControlSurface` gates each
+#: model on `ramMb >= model.min_ram_mb`. A ceiling chosen HERE would be a second
+#: authority that cannot see the first — the server deciding what a device gets
+#: while the service independently decides what it keeps. That is the
+#: `yolov8_nano` shape exactly: advertised by one component, unloadable by
+#: another, for six months, with every surface green.
+#:
+#: So the ceiling is INJECTED and has NO DEFAULT. `None` does not mean
+#: "unlimited" and does not mean a number this module picked; it means NOBODY
+#: HAS TOLD US ONE, and the registry refuses to make any portfolio decision in
+#: that state (see `_admit`).
+#:
+#: WHY THE SERVER CANNOT SIMPLY SUM A COLUMN IT ALREADY HAS: `min_ram_mb` is a
+#: GATE, not a footprint. `gemma_4_e2b` carries `min_ram_mb` 8000,
+#: `model_size_mb` 2580, and a measured working set of 3.31 GB — three different
+#: quantities, and the one portfolio admission needs (resident footprint) is in
+#: none of them. Summing `min_ram_mb` over-rejects; summing `model_size_mb`
+#: under-rejects. `ModelVocabulary.approx_ram_bytes` is that missing quantity as
+#: a COMMITTED figure, and `rss_delta_bytes` on /health is this service
+#: MEASURING it per load so the authority above can eventually stop guessing.
+
+
+class ModelEntry:
+    """One catalogue row: a committed vocabulary bound to a path on this box."""
+
+    __slots__ = ("vocabulary", "path", "pinned")
+
+    def __init__(self, vocabulary: ModelVocabulary, path: Path, pinned: bool = False):
+        self.vocabulary = vocabulary
+        self.path = Path(path)
+        self.pinned = pinned
+
+    @property
+    def name(self) -> str:
+        return self.vocabulary.model_name
+
+    def __repr__(self) -> str:  # pragma: no cover - debugging aid
+        return f"ModelEntry({self.name!r}, {str(self.path)!r}, pinned={self.pinned})"
+
+
+class _ModelState:
+    """Everything mutable about one catalogue row. Guarded by the registry lock."""
+
+    def __init__(self, entry: ModelEntry):
+        self.entry = entry
+        # LOADING, NOT ABSENT, BEFORE THE FIRST `stat`.
+        #
+        # `absent` is a claim — "the artifact is not on this box" — and at
+        # construction nobody has looked. The service starts before the watcher
+        # thread does, so a `/health` probe can arrive in that window, and the
+        # single-model loader it replaced correctly said `loading` there. Saying
+        # `absent` would report a fact we had not established, and the three
+        # installers' post-install probes read exactly this word to decide
+        # whether to wait or to tell the operator the OTA has not landed.
+        self.state = STATE_LOADING
+        self.reason: "str | None" = None
+        self.session = None
+        self.stamp: "tuple[int, int] | None" = None
+        self.present = False
+        self.warmup_ms: "float | None" = None
+        self.loaded_at: "float | None" = None
+        self.last_used_at: "float | None" = None
+        self.detect_count = 0
+        self.detect_failures = 0
+        self.degenerate_dropped = 0
+        self.unnamed_class_dropped = 0
+        self.load_failures = 0
+        self.evictions = 0
+        self.in_flight = 0
+        #: MEASURED resident cost of this load, and the instrument that read it.
+        #: REPORTED, NEVER USED FOR ADMISSION — a service that admitted on its
+        #: own measurements would be back to being a second allocation
+        #: authority. This is the footprint column the server does not have
+        #: (`min_ram_mb` is a gate, `model_size_mb` is a file size), produced so
+        #: the authority above can eventually stop estimating.
+        self.rss_delta_bytes: "int | None" = None
+        self.rss_source: "str | None" = None
+
+    @property
+    def resident(self) -> bool:
+        return self.session is not None
+
+
+class ModelRegistry:
+    """The name -> session map, its class index, and the residency policy."""
+
+    def __init__(
+        self,
+        entries: "list[ModelEntry]",
+        session_factory,
+        probe,
+        ram_budget_bytes: "int | None",
+        log=print,
+        clock=time.monotonic,
+    ):
+        if not entries:
+            raise ValueError("a registry with no models can answer nothing")
+        names = [e.name for e in entries]
+        if len(set(names)) != len(names):
+            raise ValueError(f"duplicate model names in the catalogue: {names}")
+        pinned = [e for e in entries if e.pinned]
+        if len(pinned) != 1:
+            raise ValueError(
+                "exactly one catalogue entry must be pinned — it is what `/health` reports and "
+                f"what the cascade gates on; got {[e.name for e in pinned]}"
+            )
+        if ram_budget_bytes is not None:
+            if ram_budget_bytes <= 0:
+                raise ValueError(f"an injected RAM ceiling must be positive, got {ram_budget_bytes}")
+            smallest_possible = pinned[0].vocabulary.approx_ram_bytes
+            if ram_budget_bytes < smallest_possible:
+                raise ValueError(
+                    f"the injected RAM ceiling {ram_budget_bytes} B cannot hold the pinned model "
+                    f"{pinned[0].name} ({smallest_possible} B); the service would be "
+                    "permanently unavailable rather than merely constrained"
+                )
+
+        self._entries = list(entries)
+        self._session_factory = session_factory
+        self._probe = probe
+        #: `None` means NO CEILING WAS INJECTED — not "unlimited".
+        self._budget = None if ram_budget_bytes is None else int(ram_budget_bytes)
+        self._log = log
+        self._clock = clock
+
+        #: SHORT-LIVED. Guards the state dict and the counters, and nothing
+        #: slow ever happens under it — that is what keeps `/health` cheap.
+        self._lock = threading.Lock()
+        #: COARSE. Held across plan -> evict -> load so two concurrent requests
+        #: cannot both admit and together exceed the budget. Never held during
+        #: inference.
+        self._admission = threading.RLock()
+
+        self._evictions = 0
+        self._states = {e.name: _ModelState(e) for e in entries}
+        self._pinned_name = pinned[0].name
+        self._index: "ClassIndex | None" = None
+
+    # ── the catalogue's public face ──────────────────────────────────────────
+
+    @property
+    def pinned_name(self) -> str:
+        return self._pinned_name
+
+    @property
+    def ram_budget_bytes(self) -> "int | None":
+        return self._budget
+
+    def class_index(self) -> ClassIndex:
+        """The union vocabulary of the models whose ARTIFACT IS ON THIS BOX.
+
+        Presence, not residency. A specialist that has been evicted is still
+        answerable — the next request for its classes reloads it — so gating the
+        offer on residency would make the requestable vocabulary depend on cache
+        state, and a caller would get a 400 or a 200 for the same profile
+        depending on what ran a minute ago. Presence is the honest line: the
+        artifact is here, so the question can be answered.
+
+        Gating on presence rather than on the committed list is also what keeps
+        §4 rule 5 true by construction — a vocabulary added to
+        `BUILTIN_VOCABULARIES` before its ONNX ships advertises nothing at all,
+        which is the `yolov8_nano` failure made unreachable rather than merely
+        forbidden.
+
+        Cached, and rebuilt only when presence changes: it is read on every
+        request.
+        """
+        with self._lock:
+            if self._index is None:
+                self._index = ClassIndex(
+                    [s.entry.vocabulary for s in self._states.values() if s.present]
+                )
+            return self._index
+
+    def resolve_classes(self, requested: "list[str]") -> "set[str]":
+        """Canonicalise a caller's class list, or raise `UnknownClasses`.
+
+        ANY unknown is a refusal, not just all of them — the partial case is the
+        dangerous one and it becomes the COMMON one the moment specialists
+        exist. A PPE profile asks for `glove` and `hairnet` alongside `person`,
+        and on a box holding only the general detector the operator would
+        otherwise get person counts and silence about gloves.
+
+        The message distinguishes the two reasons a class can be unanswerable,
+        because they are opposite problems with opposite fixes: a class no build
+        has ever heard of is a profile bug, while a class a committed vocabulary
+        names but whose artifact has not landed is an OTA that has not arrived
+        yet and will fix itself.
+        """
+        index = self.class_index()
+        canonical, unknown = index.resolve(requested)
+        if unknown:
+            raise UnknownClasses(unknown, index.class_names, self._unknown_detail(unknown, index))
+        if not canonical:
+            raise UnknownClasses([], index.class_names, "`classes` resolved to nothing requestable")
+        return canonical
+
+    def _unknown_detail(self, unknown: "list[str]", index: ClassIndex) -> str:
+        pending: "dict[str, str]" = {}
+        with self._lock:
+            for state in self._states.values():
+                if state.present:
+                    continue
+                for raw in unknown:
+                    if normalize_class_name(raw) in state.entry.vocabulary.class_names:
+                        pending[raw] = state.entry.name
+        detail = f"requested classes not in this box's vocabulary: {unknown}. "
+        if pending:
+            detail += (
+                "Some are named by a model whose artifact has not arrived yet "
+                f"({sorted(set(pending.items()))}); it is OTA'd and this service picks it up "
+                "within one poll interval of it landing. "
+            )
+        detail += f"Known classes: {sorted(index.class_names)}"
+        return detail
+
+    # ── the watcher's entry point ────────────────────────────────────────────
+
+    def refresh(self) -> None:
+        """Stat every artifact; load the pinned model; react to an OTA.
+
+        NO INFERENCE HAPPENS HERE EXCEPT THE PROBE, and no specialist is loaded:
+        a `stat` per catalogue row is the whole cost on the common path.
+        """
+        for entry in self._entries:
+            self._refresh_one(entry)
+
+    def _refresh_one(self, entry: ModelEntry) -> None:
+        state = self._states[entry.name]
+        stamp = _file_stamp(entry.path)
+
+        if stamp is None:
+            with self._lock:
+                was_present, was_resident = state.present, state.resident
+                state.present = False
+                state.session = None
+                state.stamp = None
+                state.warmup_ms = None
+                state.loaded_at = None
+                state.state = STATE_ABSENT
+                state.reason = (
+                    f"model artifact not found at {entry.path}; it arrives over the air and this "
+                    "service will pick it up within one poll interval of it landing"
+                )
+                if was_present or was_resident:
+                    self._index = None
+            if was_present or was_resident:
+                self._log(f"[detector] {entry.name}: artifact went away at {entry.path}")
+            return
+
+        with self._lock:
+            first_sighting = not state.present
+            state.present = True
+            if first_sighting:
+                self._index = None
+            unchanged = state.resident and state.stamp == stamp
+            failed_on_this_stamp = state.state == STATE_UNAVAILABLE and state.stamp == stamp
+            if not state.resident and state.state in (STATE_ABSENT, STATE_LOADING):
+                state.state = STATE_IDLE
+                state.reason = None
+
+        if unchanged:
+            return
+
+        if failed_on_this_stamp:
+            # The exact bytes that failed to load are still on disk. Retrying
+            # every poll would rebuild a session that cannot work, per poll,
+            # forever. The stamp moves when `ModelManager` rewrites the file,
+            # and that is the event worth reacting to.
+            return
+
+        with self._admission:
+            with self._lock:
+                was_resident = state.resident
+                if was_resident:
+                    state.session = None
+                    state.state = STATE_IDLE
+                    state.warmup_ms = None
+                    state.loaded_at = None
+            if was_resident:
+                self._log(f"[detector] {entry.name}: artifact changed on disk — dropping the session")
+
+            if entry.pinned:
+                # ONLY THE PINNED MODEL LOADS HERE, and the constructor
+                # guarantees it fits on its own. Rebuilding a specialist from
+                # the watcher would be an allocation nobody asked for, made
+                # outside a request that could have been refused for not
+                # fitting — the "approves each model independently" mistake
+                # §2 names. A dropped specialist is `idle`: present,
+                # answerable, one lazy load away on the next request for its
+                # classes.
+                self._ensure_loaded(state, stamp)
+            else:
+                with self._lock:
+                    state.stamp = stamp
+
+    # ── the request path ─────────────────────────────────────────────────────
+
+    def detect(self, jpeg: bytes, min_confidence: float, wanted: "set[str] | None") -> DetectionBatch:
+        """Run every model this request needs, in catalogue order, and merge.
+
+        ALL OR NOTHING. If any model in the plan cannot be made resident — the
+        artifact vanished, the load failed, the probe failed, or the set does not
+        fit the budget — the whole request is a `DetectorNotReady`, which the
+        server turns into a 503 and `LocalDetectorClient` into a
+        `DetectorUnavailableError`. Answering with the models that DID load
+        would be a thinner frame that nothing downstream can distinguish from a
+        quieter venue, which is the single failure this component exists to
+        prevent.
+        """
+        plan = self.class_index().assign(wanted)
+        if not plan:
+            raise DetectorNotReady(
+                "no model on this box can answer that request; "
+                + self._availability_summary()
+            )
+
+        acquired = self._acquire([vocabulary.model_name for vocabulary, _ in plan])
+        try:
+            batches = []
+            for (vocabulary, classes), session in zip(plan, acquired):
+                state = self._states[vocabulary.model_name]
+                try:
+                    batch = session.detect(jpeg, min_confidence, classes)
+                except Exception:
+                    with self._lock:
+                        state.detect_failures += 1
+                    raise
+                with self._lock:
+                    state.detect_count += 1
+                    # PER MODEL, not per service. A whole-service drop counter
+                    # is the aggregation mistake `telemetry-producer-substrate.md`
+                    # records: `vision_call_log.latency_ms` was NULL on 100% of
+                    # local-vlm rows while the cloud model was 100% populated,
+                    # and a whole-table fill check called that a healthy 92%.
+                    # One specialist dropping every detection it makes must not
+                    # be averaged into silence by the general detector's zero.
+                    state.degenerate_dropped += batch.degenerate_dropped
+                    state.unnamed_class_dropped += batch.unnamed_class_dropped
+                batches.append(batch)
+        finally:
+            self._release([vocabulary.model_name for vocabulary, _ in plan])
+
+        merged = merge_batches(batches)
+        if merged.degenerate_dropped or merged.unnamed_class_dropped:
+            self._log(
+                f"[detector] dropped {merged.degenerate_dropped} zero-extent box(es) and "
+                f"{merged.unnamed_class_dropped} unnamed-class detection(s) from one frame"
+            )
+        return merged
+
+    def _acquire(self, needed: "list[str]") -> list:
+        """Make every named model resident under the budget, then pin it in flight.
+
+        The admission decision, the eviction and the loads all happen under one
+        lock. Splitting them is how two requests each pass a fit check and then
+        both allocate.
+        """
+        with self._admission:
+            self._admit(needed)
+            sessions = []
+            taken: "list[str]" = []
+            try:
+                for name in needed:
+                    state = self._states[name]
+                    stamp = _file_stamp(state.entry.path)
+                    if stamp is None:
+                        raise DetectorNotReady(
+                            f"{name}: model artifact not found at {state.entry.path}"
+                        )
+                    if not state.resident or state.stamp != stamp:
+                        self._ensure_loaded(state, stamp)
+                    with self._lock:
+                        if state.session is None:
+                            raise DetectorNotReady(f"{name}: {state.reason or 'model is not loaded'}")
+                        state.in_flight += 1
+                        state.last_used_at = self._clock()
+                        taken.append(name)
+                        sessions.append(state.session)
+            except BaseException:
+                # RELEASE WHAT WE ALREADY TOOK. A two-model plan whose second
+                # model fails to load would otherwise leave the first pinned
+                # in-flight for the life of the process — permanently
+                # unevictable, so the budget silently shrinks by one model every
+                # time a specialist's artifact is broken. A leak that only
+                # appears on the failure path is the kind nothing notices.
+                self._release(taken)
+                raise
+            return sessions
+
+    def _release(self, names: "list[str]") -> None:
+        with self._lock:
+            for name in names:
+                state = self._states[name]
+                if state.in_flight > 0:
+                    state.in_flight -= 1
+                state.last_used_at = self._clock()
+
+    def _admit(self, needed: "list[str]") -> None:
+        """Evict LRU until `needed` fits alongside what must stay, or refuse.
+
+        "Must stay" is the pinned model plus anything with a request in flight.
+        Evicting either would make the accounting describe a set that is not the
+        resident set — and in the in-flight case would not even free the memory,
+        since the running request still holds a reference.
+        """
+        needed_set = set(needed)
+        with self._lock:
+            resident = {n for n, s in self._states.items() if s.resident}
+            in_flight = {n for n, s in self._states.items() if s.in_flight > 0}
+            cost = {n: s.entry.vocabulary.approx_ram_bytes for n, s in self._states.items()}
+
+        if self._budget is None:
+            # NO CEILING WAS INJECTED, so there is no portfolio decision this
+            # service is entitled to make. The pinned model is exempt because it
+            # is not a portfolio: the server ALREADY chose it — that is what
+            # `selectModelForDeviceTier` does — and holding what we were told to
+            # hold is the mechanism doing its job. Making a SECOND model resident
+            # is an allocation, and an allocation needs an authority.
+            #
+            # Refusing here rather than guessing is the whole correction: a
+            # number picked in this file would be a second authority that cannot
+            # see `min_ram_mb`, and the two would disagree silently.
+            unentitled = sorted(needed_set - {self._pinned_name})
+            if unentitled:
+                raise DetectorNotReady(
+                    f"cannot admit {unentitled} — no RAM ceiling was injected, so this service "
+                    "has no basis for a portfolio decision and will not invent one. The ceiling "
+                    "comes from the allocation authority that already sizes models for this box "
+                    "(sensingProfileService.selectModelForDeviceTier / the model-manifest "
+                    "min_ram_mb gate); pass it as --ram-budget-mb or "
+                    "TRILLBOARDS_DETECTOR_RAM_BUDGET_MB"
+                )
+            return
+
+        keep = needed_set | {self._pinned_name} | (resident & in_flight)
+        required = sum(cost[n] for n in keep)
+        if required > self._budget:
+            raise DetectorNotReady(
+                f"this request needs {sorted(needed_set)} resident at once, which with "
+                f"{sorted(keep - needed_set)} comes to {required} B against a "
+                f"{self._budget} B portfolio budget. Nothing was dropped to make it fit and "
+                "no partial answer was produced; raise --ram-budget-mb or narrow the profile"
+            )
+
+        evictable = sorted(
+            (n for n in resident if n not in keep),
+            key=lambda n: (self._states[n].last_used_at or 0.0),
+        )
+        projected = sum(cost[n] for n in resident | needed_set)
+        for name in evictable:
+            if projected <= self._budget:
+                break
+            self._evict(name)
+            projected -= cost[name]
+
+        if projected > self._budget:  # pragma: no cover - `keep` bounds this
+            raise DetectorNotReady(
+                f"could not fit {sorted(needed_set)} in {self._budget} B even after evicting "
+                f"{evictable}; this is an accounting bug, not a capacity one"
+            )
+
+    def _evict(self, name: str) -> None:
+        """Drop the session. Refcounting does the freeing.
+
+        Nothing else holds a reference: `_acquire` takes one only for the life
+        of a request and in-flight models are never chosen here, so the ORT
+        session is released the moment this assignment lands. The model stays
+        `idle` — present on disk, answerable, one load away.
+        """
+        with self._lock:
+            state = self._states[name]
+            if state.session is None:
+                return
+            state.session = None
+            state.state = STATE_IDLE
+            state.reason = None
+            state.warmup_ms = None
+            state.loaded_at = None
+            state.evictions += 1
+            self._evictions += 1
+        # LOUD, AND COUNTED WHERE `/health` PUBLISHES IT. Eviction is a
+        # last-resort safety valve, not the allocator: a box evicting at all is
+        # oversubscribed against the ceiling it was given, and a service quietly
+        # thrashing under one looks exactly like a service working fine. The
+        # authority above needs to see this to correct the ceiling, so it is a
+        # WARNING and a counter, never a debug line.
+        self._log(
+            f"[detector] WARNING evicted {name} to stay inside the injected {self._budget} B "
+            f"ceiling — this box is oversubscribed against what it was allocated "
+            f"(evictions={self._evictions})"
+        )
+
+    def _ensure_loaded(self, state: _ModelState, stamp: "tuple[int, int]") -> None:
+        """Build the session and PROVE IT before calling it ready.
+
+        A session that constructs is not a session that infers. `/v1/models`
+        answers 200 without ever building an engine, which is how a UTF-8 BOM in
+        litert-lm's config.json 500'd every frame on every Windows connector
+        while the installer's health check AND its CPU-fallback re-check both
+        passed. So every model — the pinned one on startup and every specialist
+        on its first request — runs a real forward pass on a synthetic frame
+        before it is allowed to answer anything.
+        """
+        entry = state.entry
+        with self._lock:
+            state.state = STATE_LOADING
+            state.reason = None
+
+        before = read_process_rss()
+        try:
+            session = self._session_factory(entry)
+        except ModelContractError as err:
+            self._fail(state, stamp, f"artifact loaded but does not match the detector contract: {err}")
+            return
+        except Exception as err:
+            self._fail(state, stamp, f"could not load {entry.path}: {err}")
+            return
+
+        try:
+            started = time.perf_counter()
+            self._probe(session)
+            warmup_ms = (time.perf_counter() - started) * 1000.0
+        except Exception as err:
+            self._fail(state, stamp, f"artifact loaded but a probe inference failed: {err}")
+            return
+
+        # MEASURED AFTER THE PROBE, because a session that has not run a forward
+        # pass has not allocated its working buffers — the number an admission
+        # decision would need is the one AFTER inference, not after construction.
+        after = read_process_rss()
+        rss_delta = None
+        rss_source = None
+        if before is not None and after is not None and before[1] == after[1]:
+            rss_delta = max(0, after[0] - before[0])
+            rss_source = after[1]
+
+        with self._lock:
+            state.session = session
+            state.stamp = stamp
+            state.state = STATE_READY
+            state.reason = None
+            state.warmup_ms = warmup_ms
+            state.loaded_at = time.time()
+            state.last_used_at = self._clock()
+            state.rss_delta_bytes = rss_delta
+            state.rss_source = rss_source
+            self._index = None
+        measured = f", rss delta {rss_delta} B via {rss_source}" if rss_delta is not None else ""
+        self._log(
+            f"[detector] ready — {entry.name} at {entry.path} "
+            f"(probe {warmup_ms:.0f} ms, {len(entry.vocabulary.slot_names)} named classes{measured})"
+        )
+
+    def _fail(self, state: _ModelState, stamp: "tuple[int, int]", reason: str) -> None:
+        with self._lock:
+            state.session = None
+            state.stamp = stamp
+            state.state = STATE_UNAVAILABLE
+            state.reason = reason
+            state.warmup_ms = None
+            state.loaded_at = None
+            state.rss_delta_bytes = None
+            state.rss_source = None
+            state.load_failures += 1
+            self._index = None
+        self._log(f"[detector] {state.entry.name}: {reason}")
+
+    # ── what `/health` reads ─────────────────────────────────────────────────
+
+    def _availability_summary(self) -> str:
+        with self._lock:
+            return ", ".join(
+                f"{n}={s.state}" for n, s in sorted(self._states.items())
+            )
+
+    def snapshot(self) -> dict:
+        """Everything `/health` needs, from state that already exists.
+
+        CANNOT LOAD A MODEL AND CANNOT INFER. It reads the outcome of loads that
+        already happened, under the short lock only, so a probe issued while a
+        specialist is loading answers immediately rather than queueing behind
+        it. Reporting a load that has not happened is the exact dishonesty
+        `GET /v1/models` shipped.
+        """
+        with self._lock:
+            pinned = self._states[self._pinned_name]
+            resident_bytes = sum(
+                s.entry.vocabulary.approx_ram_bytes for s in self._states.values() if s.resident
+            )
+            models = {}
+            for name, s in self._states.items():
+                row = {
+                    "state": s.state,
+                    "path": str(s.entry.path),
+                    "present": s.present,
+                    "resident": s.resident,
+                    "pinned": s.entry.pinned,
+                    "named_classes": len(s.entry.vocabulary.slot_names),
+                    "approx_ram_bytes": s.entry.vocabulary.approx_ram_bytes,
+                    "detect_count": s.detect_count,
+                    "detect_failures": s.detect_failures,
+                    "degenerate_boxes_dropped": s.degenerate_dropped,
+                    "unnamed_class_dropped": s.unnamed_class_dropped,
+                    "load_failures": s.load_failures,
+                    "evictions": s.evictions,
+                }
+                if s.reason:
+                    row["reason"] = s.reason
+                if s.warmup_ms is not None:
+                    row["warmup_inference_ms"] = round(s.warmup_ms, 1)
+                if s.loaded_at is not None:
+                    row["loaded_at"] = s.loaded_at
+                if s.rss_delta_bytes is not None:
+                    # NAMED WITH ITS INSTRUMENT. `rusage-maxrss` is a high-water
+                    # mark, so its delta is a LOWER BOUND and 0 is a legitimate
+                    # reading; `proc-statm` and `psapi-workingset` are current
+                    # RSS. A reader that cannot tell them apart will average the
+                    # two into a number that describes neither.
+                    row["rss_delta_bytes"] = s.rss_delta_bytes
+                    row["rss_source"] = s.rss_source
+                if s.session is not None and getattr(s.session, "spec", None) is not None:
+                    spec = s.session.spec
+                    row["input"] = f"{spec.width}x{spec.height}"
+                    row["classes"] = spec.num_classes
+                    row["providers"] = list(spec.providers)
+                models[name] = row
+
+            # Counted from `present` directly rather than through
+            # `class_index()`, which takes this same non-reentrant lock. It is a
+            # set union over at most a handful of committed frozensets — no
+            # session is touched and nothing is built that a request would then
+            # have to wait for.
+            known = len(
+                frozenset().union(
+                    *[s.entry.vocabulary.class_names for s in self._states.values() if s.present]
+                ) if any(s.present for s in self._states.values()) else frozenset()
+            )
+
+            return {
+                "pinned_model": self._pinned_name,
+                "pinned": models[self._pinned_name],
+                "pinned_state": pinned.state,
+                "models": models,
+                "resident_models": sorted(n for n, s in self._states.items() if s.resident),
+                # THE CEILING AS INJECTED — `null` when nobody told us one,
+                # which is a state a reader must be able to see rather than a
+                # number this service picked.
+                "ram_budget_bytes": self._budget,
+                "ram_ceiling_injected": self._budget is not None,
+                "ram_resident_bytes": resident_bytes,
+                #: Non-zero means this box is OVERSUBSCRIBED against the ceiling
+                #: it was given. Published because a service quietly thrashing
+                #: under one is indistinguishable from a service working fine.
+                "evictions": self._evictions,
+                "known_class_count": known,
+                "detect_count": sum(s.detect_count for s in self._states.values()),
+                "detect_failures": sum(s.detect_failures for s in self._states.values()),
+                "degenerate_boxes_dropped": sum(s.degenerate_dropped for s in self._states.values()),
+                "unnamed_class_dropped": sum(s.unnamed_class_dropped for s in self._states.values()),
+            }
+
+
+_PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
+
+
+def read_process_rss() -> "tuple[int, str] | None":
+    """This process's resident set, in bytes, and HOW IT WAS READ.
+
+    The source is returned with the number because the three platforms do not
+    answer the same question and a caller that cannot tell them apart will
+    average a high-water mark into a current reading. `telemetry-producer-substrate.md`
+    is a whole runbook about that: `vision_call_log.latency_ms` was NULL on 100%
+    of `local-vlm` rows while the cloud model was 100% populated, and a
+    whole-table fill check reported a healthy 92% — grade a column PER PRODUCER,
+    never in aggregate.
+
+      proc-statm        Linux. CURRENT RSS, exact, one small file read.
+      psapi-workingset  Windows. CURRENT working set, exact, one syscall.
+      rusage-maxrss     macOS and the fallback. A HIGH-WATER MARK for the whole
+                        process, so a delta across a load is a LOWER BOUND and
+                        legitimately reads 0 once the peak is behind us.
+
+    Stdlib only, and cheap enough to sit on a load path that already spends
+    hundreds of milliseconds building an ORT session.
+    """
+    try:
+        with open("/proc/self/statm", "rb") as handle:
+            fields = handle.read().split()
+        if len(fields) >= 2:
+            return int(fields[1]) * _PAGE_SIZE, "proc-statm"
+    except (OSError, ValueError):
+        pass
+
+    if sys.platform == "win32":  # pragma: no cover - exercised on Windows only
+        try:
+            import ctypes
+            from ctypes import wintypes
+
+            class _Counters(ctypes.Structure):
+                _fields_ = [
+                    ("cb", wintypes.DWORD),
+                    ("PageFaultCount", wintypes.DWORD),
+                    ("PeakWorkingSetSize", ctypes.c_size_t),
+                    ("WorkingSetSize", ctypes.c_size_t),
+                    ("QuotaPeakPagedPoolUsage", ctypes.c_size_t),
+                    ("QuotaPagedPoolUsage", ctypes.c_size_t),
+                    ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t),
+                    ("QuotaNonPagedPoolUsage", ctypes.c_size_t),
+                    ("PagefileUsage", ctypes.c_size_t),
+                    ("PeakPagefileUsage", ctypes.c_size_t),
+                ]
+
+            counters = _Counters()
+            counters.cb = ctypes.sizeof(_Counters)
+            handle = ctypes.windll.kernel32.GetCurrentProcess()
+            if ctypes.windll.psapi.GetProcessMemoryInfo(
+                handle, ctypes.byref(counters), counters.cb
+            ):
+                return int(counters.WorkingSetSize), "psapi-workingset"
+        except Exception:
+            pass
+
+    try:
+        import resource
+
+        raw = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+        # macOS reports bytes; Linux and most others report kilobytes.
+        scale = 1 if sys.platform == "darwin" else 1024
+        return int(raw) * scale, "rusage-maxrss"
+    except Exception:  # pragma: no cover - no stdlib path left
+        return None
+
+
+def _file_stamp(path: Path) -> "tuple[int, int] | None":
+    try:
+        st = path.stat()
+    except OSError:
+        return None
+    if st.st_size == 0:
+        return None
+    return (st.st_size, st.st_mtime_ns)
+
+
+__all__ = [
+    "ModelEntry",
+    "ModelRegistry",
+    "STATE_ABSENT",
+    "STATE_IDLE",
+    "STATE_LOADING",
+    "STATE_READY",
+    "STATE_UNAVAILABLE",
+]