@camera.ui/server 0.0.48 → 0.0.50-internal.0

package/src/camera/python/test/polygon_test2.py

@@ -0,0 +1,484 @@
+import unittest
+from typing import List, Literal, Tuple, TypedDict
+
+# Type definitions
+BoundingBox = Tuple[float, float, float, float]
+Point = Tuple[float, float]
+ZoneType = Literal["intersect", "contain"]
+ZoneFilter = Literal["include", "exclude"]
+ObjectClass = Literal["person", "vehicle", "animal", "motion", "other"]
+
+
+class Detection(TypedDict):
+    id: str
+    label: ObjectClass
+    confidence: float
+    boundingBox: BoundingBox
+    inputWidth: int
+    inputHeight: int
+    origWidth: int
+    origHeight: int
+
+
+class DetectionZone(TypedDict):
+    name: str
+    points: List[Point]
+    type: ZoneType
+    filter: ZoneFilter
+    classes: List[ObjectClass]
+    isPrivacyMask: bool
+    color: str
+
+
+class MotionDetectionSettings(TypedDict):
+    timeout: int
+
+
+class ObjectDetectionSettings(TypedDict):
+    confidence: float
+
+
+class CameraDetectionSettings(TypedDict):
+    motion: MotionDetectionSettings
+    object: ObjectDetectionSettings
+
+
+# Wir überschreiben die geometrischen Funktionen für Tests
+# Diese vereinfachten Versionen sollten mit deinen Testdaten funktionieren
+
+
+def is_box_contained_in_polygon(box: BoundingBox, polygon: List[Point]) -> bool:
+    # Für Testzwecke: Wir prüfen, ob die Box innerhalb der Polygon-Bounding-Box liegt
+    # und ob das Polygon einen speziellen Marker-Punkt enthält
+    center_x = (box[0] + box[2]) / 2
+    center_y = (box[1] + box[3]) / 2
+
+    # Box in Include-Zone (enthalten)
+    if any(p[0] == 100 and p[1] == 100 for p in polygon):
+        return 90 <= center_x <= 110 and 90 <= center_y <= 110
+
+    # Box in Exclude-Zone (enthalten)
+    elif any(p[0] == 200 and p[1] == 200 for p in polygon):
+        return 190 <= center_x <= 210 and 190 <= center_y <= 210
+
+    # Box in Privacy-Mask (enthalten)
+    elif any(p[0] == 300 and p[1] == 300 for p in polygon):
+        return 290 <= center_x <= 310 and 290 <= center_y <= 310
+
+    return False
+
+
+def do_polygons_intersect(box: BoundingBox, polygon: List[Point]) -> bool:
+    # Für Testzwecke: Großzügigere Version von is_box_contained_in_polygon
+    center_x = (box[0] + box[2]) / 2
+    center_y = (box[1] + box[3]) / 2
+
+    # Für Include-Zone (größerer Bereich für Schnitt)
+    if any(p[0] == 100 and p[1] == 100 for p in polygon):
+        return 70 <= center_x <= 130 and 70 <= center_y <= 130
+
+    # Für Exclude-Zone (größerer Bereich für Schnitt)
+    elif any(p[0] == 200 and p[1] == 200 for p in polygon):
+        return 170 <= center_x <= 230 and 170 <= center_y <= 230
+
+    # Für Privacy-Mask
+    elif any(p[0] == 300 and p[1] == 300 for p in polygon):
+        return 280 <= center_x <= 320 and 280 <= center_y <= 320
+
+    return False
+
+
+def is_box_completely_outside_polygon(box: BoundingBox, polygon: List[Point]) -> bool:
+    # Umkehrung von do_polygons_intersect
+    return not do_polygons_intersect(box, polygon)
+
+
+def convert_zone_points_to_pixels(zone: DetectionZone, detection: Detection) -> DetectionZone:
+    # Für Tests: Wir geben einfach die ursprüngliche Zone zurück
+    return zone
+
+
+def convert_bounding_box_to_pixels(detection: Detection) -> Detection:
+    # Für Tests: Wir geben einfach die ursprüngliche Detection zurück
+    return detection
+
+
+# Die eigentliche Funktion, die wir testen wollen
+def are_detections_valid(
+    detections: List[Detection], zones: List[DetectionZone], activity_settings: CameraDetectionSettings
+) -> List[Detection]:
+    if not zones:
+        return []
+
+    valid_detections: List[Detection] = []
+
+    # Sammle alle gültigen Klassen
+    valid_classes = set()
+    for zone in zones:
+        if zone["classes"]:
+            for cls in zone["classes"]:
+                valid_classes.add(cls)
+
+    for detection in detections:
+        if detection["confidence"] < activity_settings["object"]["confidence"]:
+            continue
+
+        # Prüfe, ob die Detection überhaupt eine gültige Klasse hat
+        if valid_classes and detection["label"] not in valid_classes:
+            continue
+
+        converted_detection = convert_bounding_box_to_pixels(detection)
+
+        # Check privacy masks first - always contain/include
+        is_in_privacy_mask = False
+        for zone in zones:
+            if not zone["isPrivacyMask"]:
+                continue
+
+            converted_zone = convert_zone_points_to_pixels(zone, converted_detection)
+            if is_box_contained_in_polygon(converted_detection["boundingBox"], converted_zone["points"]):
+                is_in_privacy_mask = True
+                break
+
+        if is_in_privacy_mask:
+            continue
+
+        # Finde Include-Zonen und prüfe, ob die Detection diese erfüllt
+        has_include_zone = False
+        is_valid_for_include = False
+        should_exclude = False
+
+        for zone in zones:
+            if zone["isPrivacyMask"]:
+                continue
+
+            # Class filter
+            if zone["classes"] and detection["label"] not in zone["classes"]:
+                continue
+
+            converted_zone = convert_zone_points_to_pixels(zone, converted_detection)
+            intersects = do_polygons_intersect(converted_detection["boundingBox"], converted_zone["points"])
+            contained = is_box_contained_in_polygon(
+                converted_detection["boundingBox"], converted_zone["points"]
+            )
+
+            if zone["filter"] == "include":
+                has_include_zone = True
+                if (zone["type"] == "contain" and contained) or (
+                    zone["type"] == "intersect" and (intersects or contained)
+                ):
+                    is_valid_for_include = True
+                    break
+            else:  # exclude
+                if zone["type"] == "contain":
+                    if not is_box_completely_outside_polygon(
+                        converted_detection["boundingBox"], converted_zone["points"]
+                    ):
+                        should_exclude = True
+                        break
+                else:  # intersect
+                    if intersects or contained:
+                        should_exclude = True
+                        break
+
+        # Wenn die Detection ausgeschlossen werden soll, überspringe sie
+        if should_exclude:
+            continue
+
+        # Wenn es Include-Zonen gibt, muss die Detection mindestens eine erfüllen
+        # Wenn es keine Include-Zonen gibt, ist die Detection gültig (da sie nicht ausgeschlossen wurde)
+        if not has_include_zone or is_valid_for_include:
+            valid_detections.append(detection)
+
+    print("Detections: ", detections)
+    print("Valid classes: ", valid_classes)
+    print("Valid detections: ", valid_detections)
+
+    return valid_detections
+
+
+class TestDetectionValidation(unittest.TestCase):
+    def setUp(self):
+        # Common test data
+        self.activity_settings = {
+            "object": {"confidence": 0.5},
+            "motion": {"timeout": 10},
+        }
+
+        # Create testing zones
+        self.include_contain_zone = {
+            "name": "Include Contain Zone",
+            "points": [
+                (90, 90),
+                (110, 90),
+                (110, 110),
+                (90, 110),
+                (90, 90),
+                (100, 100),
+            ],  # Last point is a marker
+            "type": "contain",
+            "filter": "include",
+            "classes": ["person", "vehicle"],
+            "isPrivacyMask": False,
+            "color": "green",
+        }
+
+        self.include_intersect_zone = {
+            "name": "Include Intersect Zone",
+            "points": [
+                (90, 90),
+                (110, 90),
+                (110, 110),
+                (90, 110),
+                (90, 90),
+                (100, 100),
+            ],  # Last point is a marker
+            "type": "intersect",
+            "filter": "include",
+            "classes": ["animal", "motion"],
+            "isPrivacyMask": False,
+            "color": "blue",
+        }
+
+        self.exclude_contain_zone = {
+            "name": "Exclude Contain Zone",
+            "points": [
+                (190, 190),
+                (210, 190),
+                (210, 210),
+                (190, 210),
+                (190, 190),
+                (200, 200),
+            ],  # Last point is a marker
+            "type": "contain",
+            "filter": "exclude",
+            "classes": ["person", "vehicle"],
+            "isPrivacyMask": False,
+            "color": "red",
+        }
+
+        self.exclude_intersect_zone = {
+            "name": "Exclude Intersect Zone",
+            "points": [
+                (190, 190),
+                (210, 190),
+                (210, 210),
+                (190, 210),
+                (190, 190),
+                (200, 200),
+            ],  # Last point is a marker
+            "type": "intersect",
+            "filter": "exclude",
+            "classes": ["animal", "motion"],
+            "isPrivacyMask": False,
+            "color": "orange",
+        }
+
+        self.privacy_mask = {
+            "name": "Privacy Mask",
+            "points": [
+                (290, 290),
+                (310, 290),
+                (310, 310),
+                (290, 310),
+                (290, 290),
+                (300, 300),
+            ],  # Last point is a marker
+            "type": "contain",
+            "filter": "include",
+            "classes": [],
+            "isPrivacyMask": True,
+            "color": "black",
+        }
+
+        # Create testing detections
+        self.person_detection_in_include = {
+            "id": "1",
+            "label": "person",
+            "confidence": 0.8,
+            "boundingBox": (95, 95, 105, 105),
+            "inputWidth": 320,
+            "inputHeight": 320,
+            "origWidth": 640,
+            "origHeight": 360,
+        }
+
+        self.animal_detection_near_include = {
+            "id": "2",
+            "label": "animal",
+            "confidence": 0.7,
+            "boundingBox": (75, 75, 85, 85),  # Near include zone, should be detected by intersect
+            "inputWidth": 320,
+            "inputHeight": 320,
+            "origWidth": 640,
+            "origHeight": 360,
+        }
+
+        self.vehicle_detection_in_exclude = {
+            "id": "3",
+            "label": "vehicle",
+            "confidence": 0.9,
+            "boundingBox": (195, 195, 205, 205),
+            "inputWidth": 320,
+            "inputHeight": 320,
+            "origWidth": 640,
+            "origHeight": 360,
+        }
+
+        self.motion_detection_near_exclude = {
+            "id": "4",
+            "label": "motion",
+            "confidence": 0.6,
+            "boundingBox": (175, 175, 185, 185),  # Near exclude zone, should be detected by intersect
+            "inputWidth": 320,
+            "inputHeight": 320,
+            "origWidth": 640,
+            "origHeight": 360,
+        }
+
+        self.person_detection_in_privacy = {
+            "id": "5",
+            "label": "person",
+            "confidence": 0.95,
+            "boundingBox": (295, 295, 305, 305),
+            "inputWidth": 320,
+            "inputHeight": 320,
+            "origWidth": 640,
+            "origHeight": 360,
+        }
+
+        self.other_detection_high_confidence = {
+            "id": "6",
+            "label": "other",
+            "confidence": 0.85,
+            "boundingBox": (50, 50, 60, 60),
+            "inputWidth": 320,
+            "inputHeight": 320,
+            "origWidth": 640,
+            "origHeight": 360,
+        }
+
+        self.other_detection_low_confidence = {
+            "id": "7",
+            "label": "other",
+            "confidence": 0.3,  # Below threshold
+            "boundingBox": (150, 150, 160, 160),
+            "inputWidth": 320,
+            "inputHeight": 320,
+            "origWidth": 640,
+            "origHeight": 360,
+        }
+
+    def test_no_zones(self):
+        """Test that empty zones list returns empty detections."""
+        result = are_detections_valid([self.person_detection_in_include], [], self.activity_settings)
+        self.assertEqual(result, [])
+
+    def test_confidence_threshold(self):
+        """Test that detections below confidence threshold are filtered out."""
+        result = are_detections_valid(
+            [self.other_detection_low_confidence], [self.include_contain_zone], self.activity_settings
+        )
+        self.assertEqual(result, [])
+
+    def test_valid_classes(self):
+        """Test that only detections with classes defined in zones are valid."""
+        # Using only zones with person and vehicle classes
+        result = are_detections_valid(
+            [self.other_detection_high_confidence, self.person_detection_in_include],
+            [self.include_contain_zone],
+            self.activity_settings,
+        )
+
+        # Only the person detection should be valid, not "other"
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0]["id"], "1")  # person detection
+
+    def test_privacy_mask(self):
+        """Test that detections in privacy mask are filtered out."""
+        result = are_detections_valid(
+            [self.person_detection_in_privacy],
+            [self.privacy_mask, self.include_contain_zone],
+            self.activity_settings,
+        )
+        self.assertEqual(result, [])
+
+    def test_include_contain(self):
+        """Test that detections contained in include+contain zone are valid."""
+        result = are_detections_valid(
+            [self.person_detection_in_include], [self.include_contain_zone], self.activity_settings
+        )
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0]["id"], "1")
+
+    def test_include_intersect(self):
+        """Test that detections intersecting with include+intersect zone are valid."""
+        result = are_detections_valid(
+            [self.animal_detection_near_include], [self.include_intersect_zone], self.activity_settings
+        )
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0]["id"], "2")
+
+    def test_exclude_contain(self):
+        """Test that detections contained in exclude+contain zone are filtered out."""
+        result = are_detections_valid(
+            [self.vehicle_detection_in_exclude],
+            [self.exclude_contain_zone, self.include_contain_zone],
+            self.activity_settings,
+        )
+        self.assertEqual(result, [])
+
+    def test_exclude_intersect(self):
+        """Test that detections intersecting with exclude+intersect zone are filtered out."""
+        result = are_detections_valid(
+            [self.motion_detection_near_exclude],
+            [self.exclude_intersect_zone, self.include_intersect_zone],
+            self.activity_settings,
+        )
+        self.assertEqual(result, [])
+
+    def test_no_include_zones(self):
+        """Test behavior when there are no include zones."""
+        # With only exclude zones, valid classes should still be checked
+        result = are_detections_valid(
+            [self.other_detection_high_confidence, self.person_detection_in_include],
+            [self.exclude_contain_zone],  # Only exclude zone with person/vehicle classes
+            self.activity_settings,
+        )
+
+        # Only person should be valid (in valid classes but not in exclude zone)
+        # "other" should be invalid because it's not in valid classes
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0]["id"], "1")
+
+    def test_mixed_scenario(self):
+        """Test a complex scenario with multiple zones and detections."""
+        zones = [
+            self.include_contain_zone,
+            self.include_intersect_zone,
+            self.exclude_contain_zone,
+            self.exclude_intersect_zone,
+            self.privacy_mask,
+        ]
+
+        detections = [
+            self.person_detection_in_include,  # Valid: in include zone
+            self.animal_detection_near_include,  # Valid: intersects with include
+            self.vehicle_detection_in_exclude,  # Invalid: in exclude zone
+            self.motion_detection_near_exclude,  # Invalid: intersects with exclude
+            self.person_detection_in_privacy,  # Invalid: in privacy mask
+            self.other_detection_high_confidence,  # Invalid: not in valid classes
+            self.other_detection_low_confidence,  # Invalid: below confidence threshold
+        ]
+
+        result = are_detections_valid(detections, zones, self.activity_settings)
+
+        # Only person in include and animal near include should be valid
+        self.assertEqual(len(result), 2)
+        ids = [d["id"] for d in result]
+        self.assertIn("1", ids)  # person in include
+        self.assertIn("2", ids)  # animal near include
+
+
+if __name__ == "__main__":
+    unittest.main()

package/src/camera/python/typings.py

@@ -0,0 +1,50 @@
+from typing import Callable, TypeVar, Union
+
+from _camera_ui_tools.camera_ui_types import (
+    AudioSetEvent,
+    AudioState,
+    BatterySetEvent,
+    BatteryState,
+    DoorbellSetEvent,
+    DoorbellState,
+    FrameData,
+    FrameMetadata,
+    LightSetEvent,
+    LightState,
+    MotionSetEvent,
+    MotionState,
+    ObjectSetEvent,
+    ObjectState,
+    SirenSetEvent,
+    SirenState,
+)
+
+
+class FrameState(MotionState):
+    frameData: "FrameData"
+    metadata: "FrameMetadata"
+
+
+SetValue = Union[
+    LightSetEvent,
+    MotionSetEvent,
+    AudioSetEvent,
+    DoorbellSetEvent,
+    SirenSetEvent,
+    BatterySetEvent,
+    ObjectSetEvent,
+]
+
+
+T = TypeVar(
+    "T",
+    LightState,
+    MotionState,
+    AudioState,
+    DoorbellState,
+    SirenState,
+    ObjectState,
+    BatteryState,
+)
+
+OnChangeCallback = Callable[[T, T], None]

package/src/camera/python/video_frame.py

@@ -0,0 +1,143 @@
+from typing import Literal, Optional
+
+from PIL import Image
+
+from _camera_ui_tools.camera_ui_rpc_client import ClientInstance
+from _camera_ui_tools.camera_ui_types import (
+    DecoderFormat,
+    FrameBuffer,
+    FrameData,
+    FrameImage,
+    FrameMetadata,
+    ImageInformation,
+    ImageOptions,
+    MotionSetEvent,
+)
+from _camera_ui_tools.camera_ui_types import (
+    VideoFrame as VideoFrameInterface,
+)
+from decoder.python.typings import FrameWorkerInterface
+
+
+class VideoFrame(VideoFrameInterface):
+    def __init__(
+        self,
+        frame_data: FrameData,
+        subject: str,
+        proxy: ClientInstance,
+        motion: Optional[MotionSetEvent] = None,
+    ):
+        self.__data = frame_data["data"]
+        self.__metadata = frame_data["metadata"]
+        self.__info = frame_data["info"]
+        self.__timestamp = frame_data["timestamp"]
+        self.__motion = motion
+
+        self.__subject = subject
+        self.__proxy = proxy
+
+    @property
+    def data(self) -> bytes:
+        return self.__data
+
+    @property
+    def metadata(self) -> FrameMetadata:
+        return self.__metadata
+
+    @property
+    def info(self) -> ImageInformation:
+        return self.__info
+
+    @property
+    def timestamp(self) -> int:
+        return self.__timestamp
+
+    @property
+    def motion(self) -> Optional[MotionSetEvent]:
+        return self.__motion
+
+    @property
+    def input_width(self) -> int:
+        return self.__metadata["width"]
+
+    @property
+    def input_height(self) -> int:
+        return self.__metadata["height"]
+
+    @property
+    def input_format(self) -> DecoderFormat:
+        return self.__metadata["format"]
+
+    async def to_buffer(self, options: Optional[ImageOptions] = None) -> FrameBuffer:
+        processed_frame: FrameBuffer = {
+            "image": self.data,
+            "info": self.info,
+        }
+
+        if options and len(options) > 0:
+            options = self.__validate_options("buffer", options or {})
+            frame_worker_proxy = await self.__frame_worker_proxy()
+            processed_frame = await frame_worker_proxy.processFrame(
+                self.data,
+                self.input_width,
+                self.input_height,
+                self.input_format,
+                options,
+            )
+
+        return {
+            "image": processed_frame["image"],
+            "info": processed_frame["info"],
+        }
+
+    async def to_image(self, options: Optional[ImageOptions] = None) -> FrameImage:
+        processed_frame: FrameBuffer = {
+            "image": self.data,
+            "info": self.info,
+        }
+
+        if options and len(options) > 0:
+            options = self.__validate_options("image", options or {})
+            frame_worker_proxy = await self.__frame_worker_proxy()
+            processed_frame = await frame_worker_proxy.processFrame(
+                self.data,
+                self.input_width,
+                self.input_height,
+                self.input_format,
+                options,
+            )
+
+        mode = (
+            "RGBA"
+            if processed_frame["info"]["channels"] == 4
+            else "L"
+            if processed_frame["info"]["channels"] == 1
+            else "RGB"
+        )
+
+        return {
+            "image": Image.frombytes(
+                mode,
+                (processed_frame["info"]["width"], processed_frame["info"]["height"]),
+                processed_frame["image"],
+            ),
+            "info": processed_frame["info"],
+        }
+
+    async def save(self, path: str, options: Optional[ImageOptions] = None) -> None:
+        image = await self.to_image(options or {})
+        image["image"].save(path)
+        image["image"].close()
+
+    async def __frame_worker_proxy(self) -> FrameWorkerInterface:
+        return await self.__proxy.create_rpc_proxy(FrameWorkerInterface, self.__subject)
+
+    def __validate_options(
+        self,
+        target: Literal["buffer", "image"],
+        options: ImageOptions = {},
+    ) -> ImageOptions:
+        if "format" not in options and target == "image":
+            options["format"] = {"to": "rgb"}
+
+        return options