@camera.ui/server 0.0.49 → 0.0.50-internal.0

package/dist/server/src/camera/python/iou.py

@@ -0,0 +1,158 @@
+from typing import cast
+
+from _camera_ui_tools.camera_ui_types import (
+    BoundingBox,
+    Detection,
+)
+
+
+class DetectionWithId(Detection):
+    id: str  # type: ignore
+
+
+# Helper class for spatial indexing
+class SpatialIndex:
+    def __init__(self, cell_size: float = 100):
+        self.cells: dict[str, list[DetectionWithId]] = {}
+        self.cell_size = cell_size
+
+    def _get_cell_key(self, x: float, y: float) -> str:
+        cell_x = int(x // self.cell_size)
+        cell_y = int(y // self.cell_size)
+        return f"{cell_x},{cell_y}"
+
+    def _get_cells_for_box(self, box: BoundingBox, margin: float = 0) -> list[str]:
+        min_cell_x = int((box[0] - margin) // self.cell_size)
+        min_cell_y = int((box[1] - margin) // self.cell_size)
+        max_cell_x = int((box[2] + margin) // self.cell_size)
+        max_cell_y = int((box[3] + margin) // self.cell_size)
+
+        return [
+            f"{x},{y}" for x in range(min_cell_x, max_cell_x + 1) for y in range(min_cell_y, max_cell_y + 1)
+        ]
+
+    def insert(self, detection: DetectionWithId) -> None:
+        box = detection["boundingBox"]
+        cells = self._get_cells_for_box(box)
+
+        for cell in cells:
+            if cell not in self.cells:
+                self.cells[cell] = []
+            self.cells[cell].append(detection)
+
+    def search(self, box: BoundingBox, margin: float = 0) -> list[DetectionWithId]:
+        cells = self._get_cells_for_box(box, margin)
+        seen_ids: set[str] = set()
+        result: list[DetectionWithId] = []
+
+        for cell in cells:
+            detections = self.cells.get(cell, [])
+            for detection in detections:
+                if detection["id"] not in seen_ids:
+                    seen_ids.add(detection["id"])
+                    result.append(detection)
+
+        return result
+
+
+def calculate_iou(box1: BoundingBox, box2: BoundingBox) -> float:
+    intersection_x1 = max(box1[0], box2[0])
+    intersection_y1 = max(box1[1], box2[1])
+    intersection_x2 = min(box1[2], box2[2])
+    intersection_y2 = min(box1[3], box2[3])
+
+    if intersection_x2 < intersection_x1 or intersection_y2 < intersection_y1:
+        return 0
+
+    intersection_area = (intersection_x2 - intersection_x1) * (intersection_y2 - intersection_y1)
+    box1_area = (box1[2] - box1[0]) * (box1[3] - box1[1])
+    box2_area = (box2[2] - box2[0]) * (box2[3] - box2[1])
+
+    return intersection_area / (box1_area + box2_area - intersection_area)
+
+
+def merge_bounding_boxes(boxes: list[BoundingBox]) -> BoundingBox:
+    min_x = max(0, round(min(box[0] for box in boxes)))
+    min_y = max(0, round(min(box[1] for box in boxes)))
+    max_x = min(1000, round(max(box[2] for box in boxes)))
+    max_y = min(1000, round(max(box[3] for box in boxes)))
+
+    return (min_x, min_y, max_x, max_y)
+
+
+def generate_id(length: int = 7) -> str:
+    import random
+    import string
+
+    return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))
+
+
+def is_valid_box(box: BoundingBox) -> bool:
+    return box[0] < box[2] and box[1] < box[3]
+
+
+def merge_detections(
+    detections: list[Detection], iou_threshold: float = 0.01, close_threshold: float = 100
+) -> list[Detection]:
+    if not detections:
+        return []
+
+    # Create spatial index with cell size based on close_threshold
+    spatial_index = SpatialIndex(close_threshold)
+    processed_ids: set[str] = set()
+    merged_detections: list[Detection] = []
+
+    # Prepare detections with IDs
+    prepared_detections: list[DetectionWithId] = []
+    for d in detections:
+        det_copy = d.copy()
+        if "id" not in det_copy:
+            det_copy["id"] = generate_id()
+        prepared_detections.append(cast(DetectionWithId, det_copy))
+
+    # Load spatial index
+    for detection in prepared_detections:
+        spatial_index.insert(detection)
+
+    for detection in prepared_detections:
+        if detection["id"] in processed_ids:
+            continue
+
+        current_detection = detection.copy()
+        max_confidence = current_detection["confidence"]
+        overlapping_boxes: list[BoundingBox] = [current_detection["boundingBox"]]
+
+        # Find potential overlaps using spatial index
+        search_margin = max(close_threshold, 10)
+        potential_overlaps = spatial_index.search(current_detection["boundingBox"], search_margin)
+
+        for overlap in potential_overlaps:
+            if (
+                overlap["id"] == current_detection["id"]
+                or overlap["id"] in processed_ids
+                or overlap["label"] != current_detection["label"]
+            ):
+                continue
+
+            iou = calculate_iou(current_detection["boundingBox"], overlap["boundingBox"])
+            is_close = (
+                abs(current_detection["boundingBox"][0] - overlap["boundingBox"][0]) <= close_threshold
+                and abs(current_detection["boundingBox"][1] - overlap["boundingBox"][1]) <= close_threshold
+            )
+
+            if iou > iou_threshold or is_close:
+                overlapping_boxes.append(overlap["boundingBox"])
+                processed_ids.add(overlap["id"])
+                max_confidence = max(max_confidence, overlap["confidence"])
+
+        if len(overlapping_boxes) > 1:
+            merged_box = merge_bounding_boxes(overlapping_boxes)
+            if is_valid_box(merged_box):
+                current_detection["boundingBox"] = merged_box
+                current_detection["confidence"] = max_confidence
+
+        if is_valid_box(current_detection["boundingBox"]):
+            merged_detections.append(cast(Detection, current_detection))
+        processed_ids.add(current_detection["id"])
+
+    return merged_detections

package/dist/server/src/camera/python/polygon.py

@@ -0,0 +1,230 @@
+from _camera_ui_tools.camera_ui_types import (
+    BoundingBox,
+    CameraDetectionSettings,
+    Detection,
+    DetectionZone,
+    ObjectClass,
+    Point,
+)
+
+
+def points_equal(p1: Point, p2: Point) -> bool:
+    return p1[0] == p2[0] and p1[1] == p2[1]
+
+
+def is_point_in_polygon(point: Point, polygon: list[Point]) -> bool:
+    x, y = point
+    inside = False
+
+    for i in range(len(polygon)):
+        j = (i + 1) % len(polygon)
+        xi, yi = polygon[i]
+        xj, yj = polygon[j]
+
+        # Check if the point is on the line
+        if (min(xi, xj) <= x <= max(xi, xj)) and (min(yi, yj) <= y <= max(yi, yj)):
+            # For vertical lines
+            if abs(xi - xj) < 1e-10 and abs(x - xi) < 1e-10:
+                return True
+            # For non-vertical lines
+            if abs(xi - xj) > 1e-10:
+                m = (yj - yi) / (xj - xi)
+                b = yi - m * xi
+                if abs(y - (m * x + b)) < 1e-10:
+                    return True
+
+        # Original Ray-Casting Algorithm
+        if ((yi > y) != (yj > y)) and (x < (xj - xi) * (y - yi) / (yj - yi) + xi):
+            inside = not inside
+
+    return inside
+
+
+def is_box_contained_in_polygon(box: BoundingBox, polygon: list[Point]) -> bool:
+    corners: list[Point] = [
+        (box[0], box[1]),  # Top-left
+        (box[2], box[1]),  # Top-right
+        (box[2], box[3]),  # Bottom-right
+        (box[0], box[3]),  # Bottom-left
+    ]
+
+    return all(is_point_in_polygon(point, polygon) for point in corners)
+
+
+def is_box_completely_outside_polygon(box: BoundingBox, polygon: list[Point]) -> bool:
+    corners: list[Point] = [
+        (box[0], box[1]),  # Top-left
+        (box[2], box[1]),  # Top-right
+        (box[2], box[3]),  # Bottom-right
+        (box[0], box[3]),  # Bottom-left
+    ]
+
+    return not do_polygons_intersect(box, polygon) and all(
+        not is_point_in_polygon(point, polygon) for point in corners
+    )
+
+
+def do_lines_intersect(a1: Point, a2: Point, b1: Point, b2: Point) -> bool:
+    denominator = (b2[1] - b1[1]) * (a2[0] - a1[0]) - (b2[0] - b1[0]) * (a2[1] - a1[1])
+    if denominator == 0:
+        return False
+
+    ua = ((b2[0] - b1[0]) * (a1[1] - b1[1]) - (b2[1] - b1[1]) * (a1[0] - b1[0])) / denominator
+    ub = ((a2[0] - a1[0]) * (a1[1] - b1[1]) - (a2[1] - a1[1]) * (a1[0] - b1[0])) / denominator
+
+    return 0 <= ua <= 1 and 0 <= ub <= 1
+
+
+def do_polygons_intersect(box: BoundingBox, polygon: list[Point]) -> bool:
+    box_polygon: list[Point] = [
+        (box[0], box[1]),  # Top-left
+        (box[2], box[1]),  # Top-right
+        (box[2], box[3]),  # Bottom-right
+        (box[0], box[3]),  # Bottom-left
+        (box[0], box[1]),  # Close the polygon
+    ]
+
+    # Check if any point of one polygon is inside the other
+    if any(is_point_in_polygon(point, polygon) for point in box_polygon):
+        return True
+
+    if any(
+        point[0] >= box[0] and point[0] <= box[2] and point[1] >= box[1] and point[1] <= box[3]
+        for point in polygon
+    ):
+        return True
+
+    # Check if any lines intersect
+    return any(
+        do_lines_intersect(box_polygon[i], box_polygon[i + 1], polygon[j], polygon[j + 1])
+        for i in range(len(box_polygon) - 1)
+        for j in range(len(polygon) - 1)
+    )
+
+
+def convert_zone_points_to_pixels(zone: DetectionZone, detection: Detection) -> DetectionZone:
+    orig_width, orig_height = detection["origWidth"], detection["origHeight"]
+    converted_points = [(x * orig_width / 100, y * orig_height / 100) for x, y in zone["points"]]
+
+    if converted_points and not points_equal(converted_points[0], converted_points[-1]):
+        converted_points.append(converted_points[0])
+
+    return {
+        **zone,
+        "points": converted_points,
+    }
+
+
+def convert_bounding_box_to_pixels(detection: Detection) -> Detection:
+    orig_width = detection["origWidth"]
+    orig_height = detection["origHeight"]
+    input_width = detection["inputWidth"]
+    input_height = detection["inputHeight"]
+    xmin, ymin, xmax, ymax = detection["boundingBox"]
+
+    xmin = (xmin * orig_width) / input_width
+    ymin = (ymin * orig_height) / input_height
+    xmax = (xmax * orig_width) / input_width
+    ymax = (ymax * orig_height) / input_height
+
+    xmin = max(0, min(xmin, orig_width))
+    ymin = max(0, min(ymin, orig_height))
+    xmax = max(0, min(xmax, orig_width))
+    ymax = max(0, min(ymax, orig_height))
+
+    return {
+        **detection,
+        "boundingBox": (xmin, ymin, xmax, ymax),
+    }
+
+
+def are_detections_valid(
+    detections: list[Detection], zones: list[DetectionZone], activity_settings: CameraDetectionSettings
+) -> list[Detection]:
+    if not zones:
+        return []
+
+    if not detections:
+        return []
+
+    valid_detections: list[Detection] = []
+
+    valid_classes = set[ObjectClass]()
+    for zone in zones:
+        if zone["classes"]:
+            for cls in zone["classes"]:
+                valid_classes.add(cls)
+
+    for detection in detections:
+        # Check if detection confidence is above threshold
+        if detection["confidence"] < activity_settings["object"]["confidence"]:
+            continue
+
+        # Check if detection class is valid
+        if detection["label"] not in valid_classes:
+            continue
+
+        converted_detection = convert_bounding_box_to_pixels(detection)
+
+        # Check if detection is inside a privacy mask
+        is_in_privacy_mask = False
+        for zone in zones:
+            if not zone["isPrivacyMask"]:
+                continue
+
+            converted_zone = convert_zone_points_to_pixels(zone, converted_detection)
+            if is_box_contained_in_polygon(converted_detection["boundingBox"], converted_zone["points"]):
+                is_in_privacy_mask = True
+                break
+
+        if is_in_privacy_mask:
+            continue
+
+        has_include_zone = False
+        is_valid_for_include = False
+        should_exclude = False
+
+        for zone in zones:
+            # Skip privacy masks
+            if zone["isPrivacyMask"]:
+                continue
+
+            # Skip zones that do not include the detection class
+            if detection["label"] not in zone["classes"]:
+                continue
+
+            converted_zone = convert_zone_points_to_pixels(zone, converted_detection)
+            intersects = do_polygons_intersect(converted_detection["boundingBox"], converted_zone["points"])
+            contained = is_box_contained_in_polygon(
+                converted_detection["boundingBox"], converted_zone["points"]
+            )
+
+            if zone["filter"] == "include":  # include
+                has_include_zone = True
+                if (zone["type"] == "contain" and contained) or (
+                    zone["type"] == "intersect" and (intersects or contained)
+                ):
+                    is_valid_for_include = True
+                    break
+            else:  # exclude
+                if zone["type"] == "contain":
+                    if not is_box_completely_outside_polygon(
+                        converted_detection["boundingBox"], converted_zone["points"]
+                    ):
+                        should_exclude = True
+                        break
+                else:  # intersect/contained
+                    if intersects or contained:
+                        should_exclude = True
+                        break
+
+        # Skip detection if it should be excluded
+        if should_exclude:
+            continue
+
+        # If there are no include zones and the detection has not been blocked by a matching exclude zone
+        # or privacy mask, it is valid
+        if not has_include_zone or is_valid_for_include:
+            valid_detections.append(detection)
+
+    return valid_detections

package/dist/server/src/camera/python/typings.py

@@ -0,0 +1,50 @@
+from typing import Callable, TypeVar, Union
+
+from _camera_ui_tools.camera_ui_types import (
+    AudioSetEvent,
+    AudioState,
+    BatterySetEvent,
+    BatteryState,
+    DoorbellSetEvent,
+    DoorbellState,
+    FrameData,
+    FrameMetadata,
+    LightSetEvent,
+    LightState,
+    MotionSetEvent,
+    MotionState,
+    ObjectSetEvent,
+    ObjectState,
+    SirenSetEvent,
+    SirenState,
+)
+
+
+class FrameState(MotionState):
+    frameData: "FrameData"
+    metadata: "FrameMetadata"
+
+
+SetValue = Union[
+    LightSetEvent,
+    MotionSetEvent,
+    AudioSetEvent,
+    DoorbellSetEvent,
+    SirenSetEvent,
+    BatterySetEvent,
+    ObjectSetEvent,
+]
+
+
+T = TypeVar(
+    "T",
+    LightState,
+    MotionState,
+    AudioState,
+    DoorbellState,
+    SirenState,
+    ObjectState,
+    BatteryState,
+)
+
+OnChangeCallback = Callable[[T, T], None]

package/dist/server/src/camera/python/video_frame.py

@@ -0,0 +1,143 @@
+from typing import Literal, Optional
+
+from PIL import Image
+
+from _camera_ui_tools.camera_ui_rpc_client import ClientInstance
+from _camera_ui_tools.camera_ui_types import (
+    DecoderFormat,
+    FrameBuffer,
+    FrameData,
+    FrameImage,
+    FrameMetadata,
+    ImageInformation,
+    ImageOptions,
+    MotionSetEvent,
+)
+from _camera_ui_tools.camera_ui_types import (
+    VideoFrame as VideoFrameInterface,
+)
+from decoder.python.typings import FrameWorkerInterface
+
+
+class VideoFrame(VideoFrameInterface):
+    def __init__(
+        self,
+        frame_data: FrameData,
+        subject: str,
+        proxy: ClientInstance,
+        motion: Optional[MotionSetEvent] = None,
+    ):
+        self.__data = frame_data["data"]
+        self.__metadata = frame_data["metadata"]
+        self.__info = frame_data["info"]
+        self.__timestamp = frame_data["timestamp"]
+        self.__motion = motion
+
+        self.__subject = subject
+        self.__proxy = proxy
+
+    @property
+    def data(self) -> bytes:
+        return self.__data
+
+    @property
+    def metadata(self) -> FrameMetadata:
+        return self.__metadata
+
+    @property
+    def info(self) -> ImageInformation:
+        return self.__info
+
+    @property
+    def timestamp(self) -> int:
+        return self.__timestamp
+
+    @property
+    def motion(self) -> Optional[MotionSetEvent]:
+        return self.__motion
+
+    @property
+    def input_width(self) -> int:
+        return self.__metadata["width"]
+
+    @property
+    def input_height(self) -> int:
+        return self.__metadata["height"]
+
+    @property
+    def input_format(self) -> DecoderFormat:
+        return self.__metadata["format"]
+
+    async def to_buffer(self, options: Optional[ImageOptions] = None) -> FrameBuffer:
+        processed_frame: FrameBuffer = {
+            "image": self.data,
+            "info": self.info,
+        }
+
+        if options and len(options) > 0:
+            options = self.__validate_options("buffer", options or {})
+            frame_worker_proxy = await self.__frame_worker_proxy()
+            processed_frame = await frame_worker_proxy.processFrame(
+                self.data,
+                self.input_width,
+                self.input_height,
+                self.input_format,
+                options,
+            )
+
+        return {
+            "image": processed_frame["image"],
+            "info": processed_frame["info"],
+        }
+
+    async def to_image(self, options: Optional[ImageOptions] = None) -> FrameImage:
+        processed_frame: FrameBuffer = {
+            "image": self.data,
+            "info": self.info,
+        }
+
+        if options and len(options) > 0:
+            options = self.__validate_options("image", options or {})
+            frame_worker_proxy = await self.__frame_worker_proxy()
+            processed_frame = await frame_worker_proxy.processFrame(
+                self.data,
+                self.input_width,
+                self.input_height,
+                self.input_format,
+                options,
+            )
+
+        mode = (
+            "RGBA"
+            if processed_frame["info"]["channels"] == 4
+            else "L"
+            if processed_frame["info"]["channels"] == 1
+            else "RGB"
+        )
+
+        return {
+            "image": Image.frombytes(
+                mode,
+                (processed_frame["info"]["width"], processed_frame["info"]["height"]),
+                processed_frame["image"],
+            ),
+            "info": processed_frame["info"],
+        }
+
+    async def save(self, path: str, options: Optional[ImageOptions] = None) -> None:
+        image = await self.to_image(options or {})
+        image["image"].save(path)
+        image["image"].close()
+
+    async def __frame_worker_proxy(self) -> FrameWorkerInterface:
+        return await self.__proxy.create_rpc_proxy(FrameWorkerInterface, self.__subject)
+
+    def __validate_options(
+        self,
+        target: Literal["buffer", "image"],
+        options: ImageOptions = {},
+    ) -> ImageOptions:
+        if "format" not in options and target == "image":
+            options["format"] = {"to": "rgb"}
+
+        return options

package/dist/server/src/camera/session.d.ts

@@ -0,0 +1,37 @@
+import { Subscribed } from '@camera.ui/common/utils';
+import { ReplaySubject } from 'rxjs';
+import type { CameraDeviceSource } from '@camera.ui/types';
+import type { CameraController } from './controller.js';
+export declare class Session {
+    private camera;
+    private source;
+    readonly onStarted: ReplaySubject<void>;
+    readonly onError: ReplaySubject<Error>;
+    readonly onConnected: ReplaySubject<void>;
+    readonly onCallEnded: ReplaySubject<void>;
+    private client;
+    private isStopped;
+    private hasEnded;
+    constructor(camera: CameraController, source: CameraDeviceSource);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    private bindConnection;
+    private callError;
+    private callEnded;
+}
+export declare class SessionController extends Subscribed {
+    private camera;
+    private readonly sessions;
+    private killed;
+    private restartTimer;
+    constructor(camera: CameraController);
+    start(sources: CameraDeviceSource[]): void;
+    stop(): void;
+    private setupEventListeners;
+    private startSession;
+    private setupEventListener;
+    private removeOrphanedSessions;
+    private createNewSessions;
+    private stopSessions;
+    private canStartSession;
+}