@trillboards/edge-sdk 0.2.74 → 0.2.78

package/deploy/detector/trillboards_detector/server.py

@@ -0,0 +1,460 @@
+"""The wire. Two routes, and a hard rule about what may go out of them.
+
+THE CONTRACT IS FROZEN AND IT LIVES ON THE OTHER SIDE.
+`trillboards-edge-sdk/packages/edge-sdk/src/LocalDetectorClient.ts` is the only
+consumer, it shipped first, and its test file enumerates fifteen payloads it
+REFUSES. This module's job is to never produce one of them. So the last thing
+`/detect` does before answering 200 is re-check every detection against the
+client's own rules (`_assert_wire_safe`), and a failure there becomes a 500
+rather than a malformed 200.
+
+That inversion is the whole design. The client rejects a bad envelope instead of
+defaulting it to `[]` because a cascade reading "detector broken" as "venue
+quiet" suppresses every alert while every health surface stays green. The
+service side of that bargain is to refuse loudly rather than emit something
+plausible — the failure mode catalogued over and over in
+`docs/runbooks/cctv-connector-runtime-edge-cases.md`, where `recordAnalysis`
+never reads `response.ok` and a 401 counts as a successful upload.
+
+STDLIB ONLY, DELIBERATELY. No onnxruntime, no numpy, no Pillow is imported from
+here — the detector arrives as an injected object. That is what lets the wire
+contract be tested without the 128 MiB artifact or a 50 MB runtime, which is the
+half of this service that a regression would actually silence.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import json
+import math
+import sys as _sys
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+from .errors import DetectorNotReady
+from .labels import KNOWN_CLASS_NAMES, resolve_requested_classes
+
+#: Loopback only. litert-lm owns 9379; this is a SECOND service, not a route on
+#: that one — see `LocalDetectorClient.ts` for why (its own uv tool, its own
+#: venv, its own OS supervisor), and `DEFAULT_DETECTOR_PORT` there for the
+#: matching constant.
+DEFAULT_HOST = "127.0.0.1"
+DEFAULT_PORT = 9380
+
+#: A frame arrives base64'd, so ~1.34x its bytes. 32 MiB covers a 4K JPEG many
+#: times over and still bounds what a runaway caller can make this process
+#: allocate on a box whose whole RAM budget is accounted for in
+#: `docs/runbooks/cctv-local-inference-hardware-floor.md`.
+MAX_BODY_BYTES = 32 * 1024 * 1024
+
+#: A chunk-size line is a few hex digits plus optional extensions. Bounding it
+#: keeps a caller from making `readline` grow without limit on a body whose
+#: total length is never declared.
+MAX_CHUNK_HEADER_BYTES = 1024
+
+#: This contract sends no trailers at all, so any is already unexpected.
+MAX_TRAILER_LINES = 32
+
+DEFAULT_MIN_CONFIDENCE = 0.25
+
+
+class WireError(Exception):
+    """A caller mistake. Carries the status code it becomes."""
+
+    def __init__(self, status: int, message: str):
+        super().__init__(message)
+        self.status = status
+        self.message = message
+
+
+def parse_detect_request(raw: bytes) -> tuple[bytes, float, "set[str] | None"]:
+    """Body bytes -> (jpeg, min_confidence, wanted classes or None for all).
+
+    Every refusal here is a 400: the caller sent something this service cannot
+    act on, and saying so beats guessing. Guessing is how a profile naming a
+    class the model cannot emit returns an empty frame forever while looking
+    healthy — `yolov8_nano`, offered to ~97 of 114 profiles and loadable
+    nowhere, `vehicle_count` at ~0% fleet-wide for six months.
+    """
+    try:
+        payload = json.loads(raw.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as err:
+        raise WireError(400, f"body is not JSON: {err}") from None
+    if not isinstance(payload, dict):
+        raise WireError(400, "body must be a JSON object")
+
+    b64 = payload.get("image_b64")
+    if not isinstance(b64, str) or not b64:
+        raise WireError(400, "`image_b64` must be a non-empty base64 string")
+    try:
+        jpeg = base64.b64decode(b64, validate=True)
+    except (binascii.Error, ValueError) as err:
+        raise WireError(400, f"`image_b64` is not valid base64: {err}") from None
+    if not jpeg:
+        raise WireError(400, "`image_b64` decoded to zero bytes")
+
+    raw_conf = payload.get("min_confidence", DEFAULT_MIN_CONFIDENCE)
+    if isinstance(raw_conf, bool) or not isinstance(raw_conf, (int, float)):
+        raise WireError(400, "`min_confidence` must be a number in [0,1]")
+    min_confidence = float(raw_conf)
+    if not math.isfinite(min_confidence) or not 0.0 <= min_confidence <= 1.0:
+        raise WireError(400, f"`min_confidence` must be in [0,1], got {raw_conf!r}")
+
+    raw_classes = payload.get("classes")
+    if raw_classes is None:
+        # ABSENT MEANS EVERY CLASS — `LocalDetectorClient` omits the key
+        # entirely rather than sending `[]`, and its test pins that. An empty
+        # list is therefore never the caller's way of saying "all", and
+        # collapsing the two would make an accidentally-empty profile silently
+        # return everything.
+        return jpeg, min_confidence, None
+    if not isinstance(raw_classes, list) or not all(isinstance(c, str) for c in raw_classes):
+        raise WireError(400, "`classes` must be an array of strings")
+    if not raw_classes:
+        raise WireError(400, "`classes` was an empty array; omit the key to mean every class")
+
+    wanted, unknown = resolve_requested_classes(raw_classes)
+    if not wanted:
+        raise WireError(
+            400,
+            f"none of the requested classes exist in this model's vocabulary: {unknown}. "
+            f"Known classes: {sorted(KNOWN_CLASS_NAMES)}",
+        )
+    return jpeg, min_confidence, wanted
+
+
+def _assert_wire_safe(detections: list) -> None:
+    """The client's fifteen refusals, checked on THIS side before answering 200.
+
+    Reproduced from `LocalDetectorClient.detect` rather than trusted: the client
+    rejects the WHOLE payload on any one of these, so a single bad row costs the
+    frame. Better to find it here, where the reason is still in hand, and answer
+    500 — a status that cannot be mistaken for a quiet venue.
+    """
+    for i, d in enumerate(detections):
+        label = d.get("label")
+        if not isinstance(label, str) or not label:
+            raise ValueError(f"detection {i} has no label")
+        conf = d.get("confidence")
+        if not isinstance(conf, (int, float)) or isinstance(conf, bool) or not math.isfinite(conf):
+            raise ValueError(f"detection {i} confidence is not a finite number: {conf!r}")
+        if not 0.0 <= conf <= 1.0:
+            raise ValueError(f"detection {i} confidence {conf} is outside [0,1]")
+        box = d.get("box")
+        if not isinstance(box, dict):
+            raise ValueError(f"detection {i} has no box")
+        for key in ("x", "y", "width", "height"):
+            v = box.get(key)
+            if not isinstance(v, (int, float)) or isinstance(v, bool) or not math.isfinite(v):
+                raise ValueError(f"detection {i} box.{key} is not a finite number: {v!r}")
+        if box["width"] <= 0 or box["height"] <= 0:
+            raise ValueError(
+                f"detection {i} box has no extent ({box['width']}x{box['height']})"
+            )
+
+
+def build_detect_response(batch, model_name: str) -> dict:
+    body = {
+        "detections": [
+            {
+                "label": d.label,
+                "confidence": d.confidence,
+                "box": {"x": d.x, "y": d.y, "width": d.width, "height": d.height},
+            }
+            for d in batch.detections
+        ],
+        # The client keeps this only when it is a finite number > 0, else null.
+        # It is the ORT forward pass alone, so it stays comparable with the
+        # ~63 ms in `docs/runbooks/cctv-detector-tier.md` — a decode cost folded
+        # in here would quietly invalidate that comparison later.
+        "inference_ms": batch.inference_ms,
+        "model": model_name,
+    }
+    # A non-finite timing would serialise as a bare `NaN`, which is INVALID JSON
+    # (`json.dumps` emits it by default; `JSON.parse` refuses it). The client
+    # would then report "returned a body that is not JSON" — a transport-shaped
+    # message for a producer-side arithmetic fault, pointing at the wrong side
+    # of the wire. A perf_counter delta cannot be non-finite, which is exactly
+    # why this must raise rather than be tolerated if it ever is.
+    if not isinstance(batch.inference_ms, (int, float)) or not math.isfinite(batch.inference_ms):
+        raise ValueError(f"inference_ms is not a finite number: {batch.inference_ms!r}")
+    _assert_wire_safe(body["detections"])
+    return body
+
+
+class DetectorHandler(BaseHTTPRequestHandler):
+    """Routes. The server instance carries `loader` and `log_line`."""
+
+    protocol_version = "HTTP/1.1"
+    server_version = "trillboards-detector"
+    sys_version = ""
+
+    #: Bound on how long one request may take to ARRIVE.
+    #:
+    #: `handle_one_request` catches TimeoutError, closes the connection and
+    #: moves on, so this converts "a caller opened a socket, sent half a chunked
+    #: body and stopped" from a worker thread pinned forever into a closed
+    #: connection. `docs/runbooks/cctv-connector-runtime-edge-cases.md` records
+    #: the same shape on the other side of the wire — a response aborted after
+    #: its headers never settles and kills that feed until restart — and the
+    #: lesson is the same: an unbounded read is a leak with a long fuse.
+    #:
+    #: Safe to set only BECAUSE every response closes its connection (see
+    #: `_json`). On a keep-alive connection this same timeout would fire during
+    #: the idle gap between analyses — 7 to 300 s at the analyze cadence — and
+    #: hand the client a socket the server had already closed.
+    timeout = 30
+
+    # ── routing ──────────────────────────────────────────────────────────────
+
+    def do_GET(self) -> None:  # noqa: N802 (stdlib naming)
+        if self._path() == "/health":
+            self._health()
+        else:
+            self._json(404, {"error": "not found"})
+
+    def do_POST(self) -> None:  # noqa: N802
+        if self._path() == "/detect":
+            self._detect()
+        elif self._path() == "/health":
+            self._json(405, {"error": "/health is a GET"})
+        else:
+            self._json(404, {"error": "not found"})
+
+    def _path(self) -> str:
+        return self.path.split("?", 1)[0].rstrip("/") or "/"
+
+    # ── /health ──────────────────────────────────────────────────────────────
+
+    def _health(self) -> None:
+        """Cheap, and it NEVER runs inference.
+
+        It reports the outcome of the load that already happened, which is the
+        only honest answer available for free. The alternative — answering 200
+        because a socket is open — is exactly `GET /v1/models`, which passed a
+        Windows fleet that could not analyze a single frame because it never
+        builds an engine. `docs/runbooks/cctv-local-inference-hardware-floor.md`
+        has the incident.
+        """
+        loader = self.server.loader  # type: ignore[attr-defined]
+        info = loader.info()
+        ready = info.get("state") == "ready"
+        info["status"] = "ok" if ready else info.get("state", "unavailable")
+        self._json(200 if ready else 503, info)
+
+    # ── /detect ──────────────────────────────────────────────────────────────
+
+    def _detect(self) -> None:
+        loader = self.server.loader  # type: ignore[attr-defined]
+        try:
+            raw = self._read_body()
+            jpeg, min_confidence, wanted = parse_detect_request(raw)
+        except WireError as err:
+            self._json(err.status, {"error": err.message})
+            return
+
+        try:
+            batch = loader.detect(jpeg, min_confidence, wanted)
+        except DetectorNotReady as err:
+            # 503, never a 200 with an empty array. The client raises
+            # DetectorUnavailableError on this and falls through to the VLM.
+            self._json(503, {"error": str(err), "status": loader.state})
+            return
+        except Exception as err:  # inference itself failed
+            self._log(f"[detector] inference failed: {type(err).__name__}: {err}")
+            self._json(500, {"error": f"inference failed: {type(err).__name__}"})
+            return
+
+        try:
+            body = build_detect_response(batch, loader.model_name)
+        except ValueError as err:
+            self._log(f"[detector] REFUSING to emit a malformed payload: {err}")
+            self._json(500, {"error": f"refused to emit a malformed payload: {err}"})
+            return
+
+        self._json(200, body)
+
+    # ── plumbing ─────────────────────────────────────────────────────────────
+
+    def _read_body(self) -> bytes:
+        """Read the request body — CHUNKED OR CONTENT-LENGTH.
+
+        CHUNKED IS THE PRODUCTION PATH, NOT AN EDGE CASE, AND THIS COST A REAL
+        BUG. `LocalDetectorClient.defaultDetectorPoster` sets exactly two
+        headers, Content-Type and Accept, and then `req.write(body); req.end()`.
+        Node has no declared length at that point, so it frames the request with
+        `Transfer-Encoding: chunked` — every single call from the shipped
+        client. `http.server` does not decode chunked bodies at all: it exposes
+        the raw socket and a Content-Length of zero.
+
+        So the first version of this method answered **400 "empty request body"
+        to every real request**, and then — because the undrained chunk bytes
+        were still in the socket — the next keep-alive request on that
+        connection was parsed as garbage and Node reported
+        `Parse Error: Expected HTTP/`. A stubbed transport cannot reach any of
+        that; only running the real client against the real service can, which
+        is what `localDetectorService.integration.test.ts` is for.
+
+        The client is FROZEN and it is not wrong here — chunked is legal
+        HTTP/1.1 and it shipped first. The service accommodates it.
+
+        On any refusal the connection is closed rather than kept alive: bytes we
+        did not drain would be read as the NEXT request line, turning one
+        rejected call into a corrupted connection.
+        """
+        encoding = (self.headers.get("Transfer-Encoding") or "").lower()
+        try:
+            if "chunked" in encoding:
+                return self._read_chunked_body()
+            return self._read_sized_body()
+        except WireError:
+            self.close_connection = True
+            raise
+
+    def _read_sized_body(self) -> bytes:
+        try:
+            length = int(self.headers.get("Content-Length") or 0)
+        except ValueError:
+            raise WireError(400, "Content-Length is not an integer") from None
+        if length <= 0:
+            raise WireError(400, "empty request body")
+        if length > MAX_BODY_BYTES:
+            raise WireError(413, f"body of {length} bytes exceeds {MAX_BODY_BYTES}")
+        raw = self.rfile.read(length)
+        if len(raw) != length:
+            raise WireError(400, "request body was shorter than its Content-Length")
+        return raw
+
+    def _read_chunked_body(self) -> bytes:
+        """Decode `<hex-size>[;ext]\r\n<data>\r\n` … `0\r\n<trailers>\r\n`.
+
+        A chunked body declares no total length, so MAX_BODY_BYTES is the ONLY
+        bound on what a caller can make this process allocate — it is checked as
+        the chunks accumulate, not after.
+        """
+        parts: list[bytes] = []
+        total = 0
+        while True:
+            line = self.rfile.readline(MAX_CHUNK_HEADER_BYTES + 1)
+            if not line:
+                raise WireError(400, "chunked body ended before its terminating chunk")
+            if len(line) > MAX_CHUNK_HEADER_BYTES:
+                raise WireError(400, "chunk size line is implausibly long")
+            # A chunk extension (`;name=value`) is legal and carries nothing we use.
+            size_token = line.split(b";", 1)[0].strip()
+            try:
+                size = int(size_token, 16)
+            except ValueError:
+                raise WireError(400, f"chunk size {size_token!r} is not hexadecimal") from None
+            if size < 0:
+                raise WireError(400, "negative chunk size")
+            if size == 0:
+                break
+            total += size
+            if total > MAX_BODY_BYTES:
+                raise WireError(413, f"chunked body exceeds {MAX_BODY_BYTES} bytes")
+            chunk = self.rfile.read(size)
+            if len(chunk) != size:
+                raise WireError(400, "chunk was shorter than its declared size")
+            parts.append(chunk)
+            if self.rfile.read(2) != b"\r\n":
+                raise WireError(400, "chunk was not terminated by CRLF")
+
+        # Trailers, then the final blank line. Read them so the connection stays
+        # usable; nothing in this contract sends any — which is why the count is
+        # BOUNDED. An unbounded loop here is the one place a caller could keep a
+        # worker busy without ever exceeding MAX_BODY_BYTES, since trailer lines
+        # are not body bytes.
+        for _ in range(MAX_TRAILER_LINES):
+            line = self.rfile.readline(MAX_CHUNK_HEADER_BYTES + 1)
+            if not line or line in (b"\r\n", b"\n"):
+                break
+        else:
+            raise WireError(400, f"more than {MAX_TRAILER_LINES} trailer lines after the final chunk")
+
+        if total == 0:
+            raise WireError(400, "empty request body")
+        return b"".join(parts)
+
+    def _json(self, status: int, payload: dict) -> None:
+        """One response, one connection. NO KEEP-ALIVE, deliberately.
+
+        The frozen client posts through Node's global agent, which pools free
+        sockets and does NOT retry one the server closed underneath it — the
+        classic keep-alive race, and here it would surface as a spurious
+        `DetectorUnavailableError` that reads as "detector down" and pushes a
+        frame to the VLM for no reason. Closing every connection removes the
+        race outright, and it costs nothing at the cadence this service runs at:
+        the detector gates the ANALYZE, which is a few frames a minute box-wide,
+        not the 500 ms LOOK.
+
+        It is also what makes the read timeout above safe to have at all.
+        """
+        # `allow_nan=False` is the backstop, not the primary check. Python's
+        # default emits bare `NaN`/`Infinity`, which no JSON parser accepts, so
+        # a single non-finite float anywhere in a payload would reach the client
+        # as "not JSON" — a message that blames the transport for a value we
+        # produced. Everything reaching here is already checked; this makes an
+        # unchecked future field fail loudly on OUR side instead.
+        try:
+            blob = json.dumps(payload, allow_nan=False).encode("utf-8")
+        except ValueError as err:
+            self._log(f"[detector] REFUSING to emit unserialisable JSON: {err}")
+            status = 500
+            blob = json.dumps({"error": "response contained a value that is not valid JSON"}).encode("utf-8")
+        self.close_connection = True
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(blob)))
+        self.send_header("Connection", "close")
+        self.end_headers()
+        self.wfile.write(blob)
+        if status >= 400:
+            self._log(f"[detector] {self.command} {self.path} -> {status}")
+
+    def _log(self, message: str) -> None:
+        logger = getattr(self.server, "log_line", None)
+        if logger:
+            logger(message)
+
+    def log_message(self, fmt: str, *args) -> None:
+        """Silence the per-request access log.
+
+        Not cosmetic. macOS has NO log rotation for this connector at all
+        (`docs/runbooks/cctv-connector-runtime-edge-cases.md`), and a 32-camera
+        install analyses continuously — an access line per frame is an unbounded
+        file on a box nobody visits. Non-2xx still logs, through `_json`.
+        """
+        return
+
+
+class DetectorServer(ThreadingHTTPServer):
+    daemon_threads = True
+
+    # SO_REUSEADDR EVERYWHERE EXCEPT WINDOWS, and the exception is the point.
+    #
+    # On POSIX it only lets a restart reclaim a port still in TIME_WAIT, which a
+    # supervised always-on service needs — otherwise a 1s blip becomes a 30s
+    # outage against launchd's ThrottleInterval. On Windows the same flag lets a
+    # SECOND process bind a port the first is actively listening on, and both
+    # then accept. `http.server` turns it on unconditionally, so this turns it
+    # back off there.
+    #
+    # `register-litertlm-task.ps1` documents exactly what that costs: a serve
+    # left alive keeps ANSWERING, so an upgrade installs the new binary and
+    # leaves the OLD one serving while every health probe passes. A hijacked
+    # bind is that incident with no way to even notice.
+    allow_reuse_address = _sys.platform != "win32"
+
+    def __init__(self, address, loader, log_line=print):
+        super().__init__(address, DetectorHandler)
+        self.loader = loader
+        self.log_line = log_line
+
+
+def serve(loader, host: str = DEFAULT_HOST, port: int = DEFAULT_PORT, log_line=print) -> DetectorServer:
+    """Start the server on a background thread and return it."""
+    server = DetectorServer((host, port), loader, log_line)
+    threading.Thread(target=server.serve_forever, name="detector-http", daemon=True).start()
+    return server