@camera.ui/server 0.0.49 → 0.0.50-internal.0

package/src/decoder/python/test/test2.py

@@ -0,0 +1,451 @@
+import os
+import time
+from typing import Any, Optional
+
+import cv2
+import numpy as np
+import pyopencl as cl
+
+BLUR_RADIUS = 6
+MOTION_THRESHOLD = 0.04
+DILATION_SIZE = 6
+AREA_THRESHOLD = 400
+ALPHA = 0.2
+
+video_path = "motion2.mp4"
+image1_path = "test1_resized.jpg"
+image2_path = "test2_resized.jpg"
+fps = 20
+num_frames = fps * 15
+
+
+def get_contour_detections(
+    mask: np.ndarray[Any, Any], area_threshold: int
+) -> list[tuple[float, float, float, float]]:
+    cnts, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    detections: list[tuple[float, float, float, float]] = []
+    for cnt in cnts:
+        x, y, w, h = cv2.boundingRect(cnt)
+        area = w * h
+        if area > area_threshold:
+            detections.append((x, y, x + w, y + h))
+    return detections
+
+
+def create_gaussian_kernel(radius: int) -> np.ndarray[Any, Any]:
+    sigma = radius / 3.0
+    x = np.arange(-radius, radius + 1)
+    kernel = np.exp(-(x**2) / (2 * sigma**2))
+    kernel = kernel / np.sum(kernel)
+    return kernel.astype(np.float32)
+
+
+def CreateContext() -> tuple[Any, Any] | None:
+    platforms: Any = cl.get_platforms()
+    if len(platforms) == 0:
+        print("Failed to find any OpenCL platforms.")
+        return None
+
+    devices = platforms[0].get_devices(cl.device_type.GPU)
+    if len(devices) == 0:
+        print("Could not find GPU device, trying CPU...")
+        devices = platforms[0].get_devices(cl.device_type.CPU)
+        if len(devices) == 0:
+            print("Could not find OpenCL GPU or CPU device.")
+            return None
+
+    context: Any = cl.Context([devices[0]])
+    return context, devices[0]
+
+
+def CreateProgram(context: Any, device: Any, filename: str) -> Any:
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    full_path = os.path.join(current_dir, filename)
+
+    with open(full_path) as kernelFile:
+        kernelStr = kernelFile.read()
+
+    program = cl.Program(context, kernelStr)
+    program.build(devices=[device])
+
+    return program
+
+
+class OpenCLMotionDetector:
+    def __init__(self, width: int, height: int):
+        ocl = CreateContext()
+
+        if ocl is None:
+            raise RuntimeError("Failed to create OpenCL context.")
+
+        self.ctx: Any = ocl[0]
+        self.device: Any = ocl[1]
+        self.queue: Any = cl.CommandQueue(self.ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
+        self.program: Any = CreateProgram(self.ctx, self.device, "opencl.cl")
+
+        print(f"Device OpenCL version: {self.device.version}")
+        print(f"Selected device: {self.device.name}")
+
+        self.width = width
+        self.height = height
+
+        mf: Any = cl.mem_flags
+
+        # Erstelle die Images und Puffer
+        self.gray_image: Any = cl.Image(
+            self.ctx,
+            mf.READ_ONLY,
+            format=cl.ImageFormat(cl.channel_order.R, cl.channel_type.UNORM_INT8),
+            shape=(self.width, self.height),
+        )
+        self.temp_image: Any = cl.Image(
+            self.ctx,
+            mf.READ_WRITE,
+            format=cl.ImageFormat(cl.channel_order.R, cl.channel_type.FLOAT),
+            shape=(self.width, self.height),
+        )
+        self.blurred_image: Any = cl.Image(
+            self.ctx,
+            mf.READ_WRITE,
+            format=cl.ImageFormat(cl.channel_order.R, cl.channel_type.FLOAT),
+            shape=(self.width, self.height),
+        )
+        self.mask_image: Any = cl.Image(
+            self.ctx,
+            mf.READ_WRITE,
+            format=cl.ImageFormat(cl.channel_order.R, cl.channel_type.FLOAT),
+            shape=(self.width, self.height),
+        )
+        self.dilated_image: Any = cl.Image(
+            self.ctx,
+            mf.WRITE_ONLY,
+            format=cl.ImageFormat(cl.channel_order.R, cl.channel_type.FLOAT),
+            shape=(self.width, self.height),
+        )
+        self.background_model_buf: Any = cl.Buffer(
+            self.ctx,
+            mf.READ_WRITE,
+            size=self.width * self.height * np.dtype(np.float32).itemsize,
+        )
+
+        # Erstelle den Gaussian-Kernel
+        self.kernel = create_gaussian_kernel(BLUR_RADIUS)
+        self.kernel_buf: Any = cl.Buffer(self.ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=self.kernel)
+        self.kernel_size = np.int32(len(self.kernel))
+
+        self.first_frame = True
+
+    def process_frame(
+        self,
+        gray_frame: np.ndarray[Any, Any],
+        motion_threshold: float,
+        dilation_size: int,
+        area_threshold: int,
+        alpha: float,
+        dispay: bool = True,
+    ) -> tuple[
+        Optional[np.ndarray[Any, Any]],
+        Optional[np.ndarray[Any, Any]],
+        list[tuple[float, float, float, float]],
+    ]:
+        # Kopiere das Graustufenbild in das OpenCL-Image
+        cl.enqueue_copy(
+            self.queue,
+            self.gray_image,
+            gray_frame,
+            origin=(0, 0),
+            region=(self.width, self.height),
+        )
+
+        # Definiere globale und lokale Workgroup-Größen
+        global_size = (self.width, self.height)
+
+        self.program.gaussian_blur_combined(
+            self.queue,
+            (self.width, self.height),
+            None,
+            self.gray_image,
+            self.blurred_image,
+            self.kernel_buf,
+            np.int32(self.kernel_size),
+        )
+
+        if not self.first_frame:
+            # Hintergrundsubtraktion und Aktualisierung des Hintergrundmodells
+            self.program.background_subtraction(
+                self.queue,
+                global_size,
+                None,
+                self.blurred_image,
+                self.background_model_buf,
+                self.mask_image,
+                np.int32(self.width),
+                np.int32(self.height),
+                np.float32(alpha),
+                np.float32(motion_threshold),
+            )
+
+            # Dilatiere die Maske
+            self.program.dilate(
+                self.queue,
+                global_size,
+                None,
+                self.mask_image,
+                self.dilated_image,
+                np.int32(dilation_size),
+            )
+
+            blur_image_cv = None
+
+            if dispay:
+                # Kopiere blur_image zur CPU für Anzeige und Verarbeitung
+                blurred_image = np.empty((self.height, self.width), dtype=np.float32)
+                cl.enqueue_copy(
+                    self.queue,
+                    blurred_image,
+                    self.blurred_image,
+                    origin=(0, 0),
+                    region=(self.width, self.height),
+                )
+
+                blur_image_cv = (blurred_image * 255).astype(np.uint8)
+
+            # Kopiere motion_mask zur CPU für Anzeige und Verarbeitung
+            motion_mask = np.empty((self.height, self.width), dtype=np.float32)
+            cl.enqueue_copy(
+                self.queue,
+                motion_mask,
+                self.dilated_image,
+                origin=(0, 0),
+                region=(self.width, self.height),
+            )
+
+            motion_mask_cv = (motion_mask * 255).astype(np.uint8)
+            detections = get_contour_detections(motion_mask_cv, area_threshold)
+
+            return blur_image_cv, motion_mask_cv, detections
+        else:
+            # Initialisiere das Hintergrundmodell
+            background_model_np = np.empty((self.height, self.width), dtype=np.float32)
+            cl.enqueue_copy(
+                self.queue,
+                background_model_np,
+                self.blurred_image,
+                origin=(0, 0),
+                region=(self.width, self.height),
+            )
+            background_model_np = background_model_np.flatten()
+            cl.enqueue_copy(self.queue, self.background_model_buf, background_model_np)
+            self.first_frame = False
+            return None, None, []
+
+
+def process_video(video_path: str, display: bool = True):
+    cap = cv2.VideoCapture(video_path)
+
+    if not cap.isOpened():
+        print(f"Error opening video file {video_path}")
+        return
+
+    width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+
+    detector = OpenCLMotionDetector(width, height)
+
+    frame_count = 0
+    total_time = 0
+    total_gpu_time = 0
+
+    if display:
+        cv2.namedWindow("Motion Detection", cv2.WINDOW_NORMAL)
+        cv2.namedWindow("Motion Mask", cv2.WINDOW_NORMAL)
+
+    while True:
+        start_time = time.perf_counter()
+
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+
+        gpu_time = time.perf_counter()
+        blurred_image, motion_mask, boxes = detector.process_frame(
+            frame_gray,
+            MOTION_THRESHOLD,
+            DILATION_SIZE,
+            AREA_THRESHOLD,
+            ALPHA,
+            display,
+        )
+        end_time = time.perf_counter()
+
+        gpu_processing_time = end_time - gpu_time
+        total_gpu_time += gpu_processing_time
+
+        processing_time = end_time - start_time
+        total_time += processing_time
+
+        frame_count += 1
+
+        if display and motion_mask is not None and blurred_image is not None:
+            debug_frame = frame.copy()
+
+            cv2.putText(
+                debug_frame,
+                f"Frame: {frame_count}",
+                (10, 30),
+                cv2.FONT_HERSHEY_SIMPLEX,
+                1,
+                (0, 255, 0),
+                2,
+            )
+            cv2.putText(
+                debug_frame,
+                f"Boxes: {len(boxes)}",
+                (10, 70),
+                cv2.FONT_HERSHEY_SIMPLEX,
+                1,
+                (0, 255, 0),
+                2,
+            )
+
+            for box in boxes:
+                x1, y1, x2, y2 = map(int, box)
+                cv2.rectangle(debug_frame, (x1, y1), (x2, y2), (0, 255, 0), 2)
+
+            fps_text = f"FPS: {1/processing_time:.2f}"
+            cv2.putText(
+                debug_frame,
+                fps_text,
+                (10, 110),
+                cv2.FONT_HERSHEY_SIMPLEX,
+                1,
+                (0, 255, 0),
+                2,
+            )
+
+            cv2.imshow("Motion Mask", motion_mask)
+            cv2.imshow("Motion Detection", debug_frame)
+            cv2.imshow("Blurred Image", blurred_image)
+
+            if cv2.waitKey(1) & 0xFF == ord("q"):
+                break
+
+    cap.release()
+    if display:
+        cv2.destroyAllWindows()
+
+    avg_gpu_time = total_gpu_time / frame_count if frame_count > 0 else 0
+    avg_time = total_time / frame_count if frame_count > 0 else 0
+    avg_fps = 1 / avg_time if avg_time > 0 else 0
+
+    print(f"Processed {frame_count} frames")
+    print(f"Average total processing time per frame: {avg_time:.4f} seconds")
+    print(f"Average GPU processing time per frame: {avg_gpu_time:.4f} seconds")
+    print(f"Average frames per second: {avg_fps:.2f}")
+
+
+def simulate_stream(img1: np.ndarray[Any, Any], img2: np.ndarray[Any, Any], display: bool = True):
+    height, width = img1.shape
+    detector = OpenCLMotionDetector(width, height)
+
+    frame_time = 1 / fps
+    total_time = 0
+    total_gpu_time = 0
+    detected_motions = 0
+
+    if display:
+        cv2.namedWindow("Motion Detection", cv2.WINDOW_NORMAL)
+        cv2.namedWindow("Motion Mask", cv2.WINDOW_NORMAL)
+
+    max_time = time.perf_counter()
+
+    for frame_count in range(num_frames):
+        start_time = time.perf_counter()
+
+        current_frame = img1 if frame_count % 2 == 0 else img2
+
+        gpu_time = time.perf_counter()
+        blurred_image, motion_mask, boxes = detector.process_frame(
+            current_frame,
+            MOTION_THRESHOLD,
+            DILATION_SIZE,
+            AREA_THRESHOLD,
+            ALPHA,
+        )
+        end_time = time.perf_counter()
+
+        gpu_processing_time = end_time - gpu_time
+        total_gpu_time += gpu_processing_time
+
+        processing_time = end_time - start_time
+        total_time += processing_time
+
+        if boxes:
+            detected_motions += 1
+
+        if display and motion_mask is not None and blurred_image is not None:
+            debug_frame = cv2.cvtColor(current_frame, cv2.COLOR_GRAY2BGR)
+
+            cv2.putText(
+                debug_frame,
+                f"Frame: {frame_count + 1}",
+                (10, 30),
+                cv2.FONT_HERSHEY_SIMPLEX,
+                1,
+                (0, 255, 0),
+                2,
+            )
+            cv2.putText(
+                debug_frame, f"Boxes: {len(boxes)}", (10, 70), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2
+            )
+            cv2.putText(
+                debug_frame,
+                f"FPS: {1/processing_time:.2f}",
+                (10, 110),
+                cv2.FONT_HERSHEY_SIMPLEX,
+                1,
+                (0, 255, 0),
+                2,
+            )
+
+            for box in boxes:
+                x1, y1, x2, y2 = map(int, box)
+                cv2.rectangle(debug_frame, (x1, y1), (x2, y2), (0, 255, 0), 2)
+
+            cv2.imshow("Motion Mask", motion_mask)
+            cv2.imshow("Motion Detection", debug_frame)
+            cv2.imshow("Blurred Image", blurred_image)
+
+            if cv2.waitKey(1) & 0xFF == ord("q"):
+                break
+
+        # Simulate frame rate
+        time_to_wait = frame_time - processing_time
+        if time_to_wait > 0:
+            time.sleep(time_to_wait)
+
+    max_time = time.perf_counter() - max_time
+
+    print(f"Max time: {max_time:.4f} seconds")
+
+    if display:
+        cv2.destroyAllWindows()
+
+    avg_gpu_time = total_gpu_time / num_frames
+    avg_time = total_time / num_frames
+    avg_fps = 1 / avg_time if avg_time > 0 else 0
+
+    print(f"Processed {num_frames} frames")
+    print(f"Average total processing time per frame: {avg_time:.4f} seconds")
+    print(f"Average GPU processing time per frame: {avg_gpu_time:.4f} seconds")
+    print(f"Average frames per second: {avg_fps:.2f}")
+    print(f"Frames with detected motion: {detected_motions}")
+    print(f"Percentage of frames with motion: {(detected_motions/num_frames)*100:.2f}%")
+
+
+if __name__ == "__main__":
+    img1 = cv2.imread(image1_path, cv2.IMREAD_GRAYSCALE)
+    img2 = cv2.imread(image2_path, cv2.IMREAD_GRAYSCALE)
+    simulate_stream(img1, img2, display=False)

package/src/decoder/python/test/utils.py

@@ -0,0 +1,50 @@
+import os
+from typing import Any
+
+import cv2
+import numpy as np
+import pyopencl as cl
+
+
+def CreateContext() -> tuple[Any, Any] | None:
+    platforms: Any = cl.get_platforms()
+    if len(platforms) == 0:
+        print("Failed to find any OpenCL platforms.")
+        return None
+
+    devices = platforms[0].get_devices(cl.device_type.GPU)
+    if len(devices) == 0:
+        print("Could not find GPU device, trying CPU...")
+        devices = platforms[0].get_devices(cl.device_type.CPU)
+        if len(devices) == 0:
+            print("Could not find OpenCL GPU or CPU device.")
+            return None
+
+    context: Any = cl.Context([devices[0]])
+    return context, devices[0]
+
+
+def CreateProgram(context: Any, device: Any, filename: str) -> Any:
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    full_path = os.path.join(current_dir, filename)
+
+    with open(full_path) as kernelFile:
+        kernelStr = kernelFile.read()
+
+    program = cl.Program(context, kernelStr)
+    program.build(devices=[device])
+
+    return program
+
+
+def get_contour_detections(
+    mask: np.ndarray[Any, Any], area_threshold: int
+) -> list[tuple[float, float, float, float]]:
+    cnts, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    detections: list[tuple[float, float, float, float]] = []
+    for cnt in cnts:
+        x, y, w, h = cv2.boundingRect(cnt)
+        area = w * h
+        if area > area_threshold:
+            detections.append((x, y, x + w, y + h))
+    return detections

package/src/decoder/python/typings.py

@@ -0,0 +1,57 @@
+from collections.abc import AsyncGenerator
+from typing import Any, Optional, Protocol, Union, runtime_checkable
+
+from typing_extensions import Literal, TypedDict
+
+from _camera_ui_tools.camera_ui_types import (
+    CameraFrameWorkerDecoder,
+    CameraFrameWorkerResolution,
+    DecoderFormat,
+    FrameBuffer,
+    FrameData,
+    ImageOptions,
+)
+
+
+class AuthConfig(TypedDict):
+    user: str
+    password: str
+
+
+PixelFormat = Literal["yuv420p", "rgb24", "nv12"]
+
+
+class BaseFrameWorkerConfig(TypedDict):
+    cameraId: str
+    cameraName: str
+    cameraSource: str
+    ffmpegPath: str
+    auth: AuthConfig
+    proxyEndpoints: list[str]
+    fps: int
+    resolution: CameraFrameWorkerResolution
+    decoder: CameraFrameWorkerDecoder
+
+
+class WorkerToMainMessage(TypedDict):
+    message: Literal["started"]
+    data: dict[str, Any]
+
+
+@runtime_checkable
+class FrameWorkerInterface(Protocol):
+    def generateFrames(self, options: Optional[ImageOptions] = None) -> AsyncGenerator[FrameData, None]: ...
+    async def processFrame(
+        self,
+        frame: Union[bytes, bytearray],
+        width: int,
+        height: int,
+        format: Union[DecoderFormat, Literal["rgba", "gray"]],
+        options: ImageOptions,
+    ) -> FrameBuffer: ...
+    async def getPrebufferedFrame(
+        self,
+        timestamp: int,
+        options: Optional[ImageOptions] = None,
+        max_difference: Optional[int] = 100,
+    ) -> Optional[FrameData]: ...

package/src/decoder/python/utils.py

@@ -0,0 +1,70 @@
+import asyncio
+from dataclasses import dataclass
+from typing import Any, Optional
+
+
+@dataclass
+class TCPConnectionDetails:
+    server: asyncio.Server
+    server_url: str
+    server_address: str
+    server_port: int
+    connection_promise: asyncio.Future[tuple[asyncio.StreamReader, asyncio.StreamWriter]]
+
+    def terminate(self) -> None:
+        self.server.close()
+
+
+async def create_single_connection_server(
+    bind_address: str = "127.0.0.1", config: Optional[dict[str, Any]] = None, wait_duration: float = 30.0
+) -> TCPConnectionDetails:
+    if not config:
+        config = {}
+
+    if not bind_address or bind_address == "0.0.0.0":
+        bind_address = "127.0.0.1"
+
+    connection_future: asyncio.Future[tuple[asyncio.StreamReader, asyncio.StreamWriter]] = asyncio.Future()
+
+    async def client_connected(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
+        if not connection_future.done():
+            connection_future.set_result((reader, writer))
+
+    server = await asyncio.start_server(
+        client_connected,
+        host=bind_address,
+        port=0,
+        **config,
+    )
+
+    server_port = 0
+    for sock in server.sockets:
+        _, server_port = sock.getsockname()
+        break
+
+    if server_port == 0:
+        raise RuntimeError("Konnte keinen Server-Port zuweisen")
+
+    def on_timeout() -> None:
+        if not connection_future.done():
+            connection_future.set_exception(
+                TimeoutError(f"Verbindungs-Timeout nach {wait_duration} Sekunden")
+            )
+            server.close()
+
+    timeout_handle = asyncio.get_running_loop().call_later(wait_duration, on_timeout)
+
+    def cancel_timeout(future: asyncio.Future[tuple[asyncio.StreamReader, asyncio.StreamWriter]]) -> None:
+        timeout_handle.cancel()
+        if future.exception() and server.is_serving():
+            server.close()
+
+    connection_future.add_done_callback(cancel_timeout)
+
+    return TCPConnectionDetails(
+        server=server,
+        server_url=f"tcp://{bind_address}:{server_port}",
+        server_address=bind_address,
+        server_port=server_port,
+        connection_promise=connection_future,
+    )