@camera.ui/server 0.0.48 → 0.0.50-internal.0

package/src/decoder/python/image_utils.py

@@ -0,0 +1,196 @@
+from typing import Any, Literal, Union
+
+import numpy as np
+from PIL import Image
+
+from _camera_ui_tools.camera_ui_common import to_thread
+from _camera_ui_tools.camera_ui_types import (
+    DecoderFormat,
+    ImageCrop,
+    ImageFormat,
+    ImageOptions,
+    ImageResize,
+)
+
+_YUV2RGB_MATRIX = np.array([[1.0, 0.0, 1.402], [1.0, -0.344136, -0.714136], [1.0, 1.772, 0.0]]).T
+
+
+async def process_image_threaded(
+    image_data: bytes,
+    input_width: int,
+    input_height: int,
+    input_format: Union[DecoderFormat, Literal["rgba"], Literal["gray"]],
+    options: ImageOptions,
+) -> bytes:
+    return await to_thread(
+        lambda: process_image(image_data, input_width, input_height, input_format, options)
+    )
+
+
+def process_image(
+    image_data: bytes,
+    input_width: int,
+    input_height: int,
+    input_format: Union[DecoderFormat, Literal["rgba"], Literal["gray"]],
+    options: ImageOptions,
+) -> bytes:
+    image: Union[Image.Image, None] = None
+
+    crop = options.get("crop", None)
+    resize = options.get("resize", None)
+    format = options.get("format", None)
+
+    if format and format["to"] == "gray":
+        if input_format == "yuv420p":
+            gray = yuv420_to_gray(image_data, input_width, input_height)
+            image = Image.fromarray(gray)
+        elif input_format == "nv12":
+            gray = nv12_to_gray(image_data, input_width, input_height)
+            image = Image.fromarray(gray)
+        elif input_format == "rgb24":
+            image = Image.frombytes("RGB", (input_width, input_height), image_data).convert("L")
+        elif input_format == "rgba":
+            image = Image.frombytes("RGBA", (input_width, input_height), image_data).convert("L")
+        else:
+            image = Image.frombytes("L", (input_width, input_height), image_data)
+
+        format = None
+    elif format and format["to"] == "rgba":
+        if input_format == "yuv420p":
+            rgba = yuv420_to_rgba(image_data, input_width, input_height)
+            image = Image.fromarray(rgba)
+        elif input_format == "nv12":
+            rgba = nv12_to_rgba(image_data, input_width, input_height)
+            image = Image.fromarray(rgba)
+        elif input_format == "rgb24":
+            image = Image.frombytes("RGB", (input_width, input_height), image_data).convert("RGBA")
+        elif input_format == "gray":
+            image = Image.frombytes("L", (input_width, input_height), image_data).convert("RGBA")
+        else:
+            image = Image.frombytes("RGBA", (input_width, input_height), image_data)
+
+        format = None
+    else:
+        if input_format == "yuv420p":
+            rgb = yuv420_to_rgb(image_data, input_width, input_height)
+            image = Image.fromarray(rgb)
+        elif input_format == "nv12":
+            rgb = nv12_to_rgb(image_data, input_width, input_height)
+            image = Image.fromarray(rgb)
+        elif input_format == "rgb24":
+            image = Image.frombytes("RGB", (input_width, input_height), image_data)
+        elif input_format == "gray":
+            image = Image.frombytes("L", (input_width, input_height), image_data)
+        else:
+            image = Image.frombytes("RGBA", (input_width, input_height), image_data)
+
+    if crop:
+        image = crop_image(image, crop)
+        input_width = crop["width"]
+        input_height = crop["height"]
+
+    if resize:
+        image = resize_image(image, input_width, input_height, resize)
+        input_width = resize["width"]
+        input_height = resize["height"]
+
+    if format:
+        image = format_image(image, format)
+
+    return image.tobytes()
+
+
+def crop_image(image: Image.Image, crop: ImageCrop) -> Image.Image:
+    crop_box = (
+        int(crop["left"]),
+        int(crop["top"]),
+        int(crop["left"] + crop["width"]),
+        int(crop["top"] + crop["height"]),
+    )
+
+    return image.crop(crop_box)
+
+
+def resize_image(image: Image.Image, input_width: int, input_height: int, resize: ImageResize) -> Image.Image:
+    width = resize["width"]
+    height = resize["height"]
+
+    if width and height and (width != input_width or height != input_height):
+        image = image.resize((int(width), int(height)), resample=Image.Resampling.BILINEAR)
+
+    return image
+
+
+def format_image(image: Image.Image, format: ImageFormat) -> Image.Image:
+    if format["to"] == "rgba":
+        image = image.convert("RGBA")
+    elif format["to"] == "gray":
+        image = image.convert("L")
+
+    return image
+
+
+def yuv420_to_rgb(yuv_bytes: bytes, width: int, height: int) -> np.ndarray[Any, Any]:
+    frame_size = width * height
+    uv_size = frame_size // 4
+
+    raw_data = np.frombuffer(yuv_bytes, dtype=np.uint8)
+
+    y = raw_data[:frame_size].reshape((height, width)).astype(np.float32) - 16.0
+    u = (
+        raw_data[frame_size : frame_size + uv_size].reshape((height // 2, width // 2)).astype(np.float32)
+        - 128.0
+    )
+    v = raw_data[frame_size + uv_size :].reshape((height // 2, width // 2)).astype(np.float32) - 128.0
+
+    u = u.repeat(2, axis=0).repeat(2, axis=1)
+    v = v.repeat(2, axis=0).repeat(2, axis=1)
+
+    r = y + 1.402 * v
+    g = y - 0.344136 * u - 0.714136 * v
+    b = y + 1.772 * u
+
+    rgb = np.stack((r, g, b), axis=-1)
+    return np.clip(rgb, 0, 255).astype(np.uint8)
+
+
+def yuv420_to_rgba(yuv_bytes: bytes, width: int, height: int) -> np.ndarray[Any, Any]:
+    rgb = yuv420_to_rgb(yuv_bytes, width, height)
+    alpha = np.full((height, width, 1), 255, dtype=np.uint8)
+    rgba = np.concatenate((rgb, alpha), axis=-1)
+    return rgba
+
+
+def yuv420_to_gray(yuv_bytes: bytes, width: int, height: int) -> np.ndarray[Any, Any]:
+    frame_size = width * height
+    raw_data = np.frombuffer(yuv_bytes, dtype=np.uint8)
+    return raw_data[:frame_size].reshape((height, width)).astype(np.uint8)
+
+
+def nv12_to_gray(nv12_bytes: bytes, width: int, height: int) -> np.ndarray[Any, Any]:
+    frame_size = width * height
+    raw_data = np.frombuffer(nv12_bytes, dtype=np.uint8)
+    return raw_data[:frame_size].reshape((height, width))
+
+
+def nv12_to_rgb(nv12_bytes: bytes, width: int, height: int) -> np.ndarray[Any, Any]:
+    frame_size = width * height
+
+    raw_data = np.frombuffer(nv12_bytes, dtype=np.uint8)
+    y = raw_data[:frame_size].reshape((height, width)).astype(np.float32) - 16.0
+    uv_plane = raw_data[frame_size:].reshape((height // 2, width)).astype(np.float32) - 128.0
+
+    u = uv_plane[:, 0::2].repeat(2, axis=0).repeat(2, axis=1)
+    v = uv_plane[:, 1::2].repeat(2, axis=0).repeat(2, axis=1)
+
+    yuv = np.stack((y, u, v), axis=-1)
+
+    rgb = np.dot(yuv, _YUV2RGB_MATRIX)
+
+    return np.clip(rgb, 0, 255).astype(np.uint8)
+
+
+def nv12_to_rgba(nv12_bytes: bytes, width: int, height: int) -> np.ndarray[Any, Any]:
+    rgb = nv12_to_rgb(nv12_bytes, width, height)
+    alpha = np.full((height, width, 1), 255, dtype=np.uint8)
+    return np.concatenate((rgb, alpha), axis=-1)

package/src/decoder/python/pool.py

@@ -0,0 +1,45 @@
+import weakref
+
+
+class FramePool:
+    def __init__(self, frame_size: int, max_pool_size: int = 4):
+        self._frame_size = frame_size
+        self._max_pool_size = max_pool_size
+        self._pool: list[memoryview] = []
+        self._in_use: dict[int, weakref.ref[memoryview]] = {}
+
+        # Initial pool filling
+        for _ in range(max_pool_size):
+            self._pool.append(memoryview(bytearray(frame_size)))
+
+    def get(self) -> memoryview:
+        self._cleanup_expired_refs()
+        frame = self._pool.pop() if self._pool else memoryview(bytearray(self._frame_size))
+        frame_id = id(frame)
+
+        def callback(ref: weakref.ref[memoryview]) -> None:
+            return self._on_frame_collected(frame_id)
+
+        self._in_use[frame_id] = weakref.ref(frame, callback)
+
+        return frame
+
+    def release(self, frame: memoryview) -> None:
+        if frame.nbytes != self._frame_size:
+            return  # Ignore invalid frames
+
+        frame_id = id(frame)
+        if frame_id in self._in_use:
+            del self._in_use[frame_id]
+
+            if len(self._pool) < self._max_pool_size:
+                self._pool.append(frame)
+
+    def _cleanup_expired_refs(self) -> None:
+        expired = [frame_id for frame_id, ref in self._in_use.items() if ref() is None]
+        for frame_id in expired:
+            del self._in_use[frame_id]
+
+    def _on_frame_collected(self, frame_id: int) -> None:
+        if frame_id in self._in_use:
+            del self._in_use[frame_id]

package/src/decoder/python/test/group.py

@@ -0,0 +1,62 @@
+def __get_optimal_work_group_size1():
+    width = 640
+    height = 360
+
+    max_work_group_size = 256
+    compute_units = 8
+
+    optimal_group_count = compute_units * 4
+
+    for size in [32, 16, 8, 4]:
+        work_groups_per_dimension = (width // size) * (height // size)
+        if (
+            width % size == 0
+            and height % size == 0
+            and size * size <= max_work_group_size
+            and work_groups_per_dimension >= optimal_group_count
+        ):
+            return (size, size)
+
+    return (min(max_work_group_size, width), 1)
+
+
+def get_optimal_work_group_size2():
+    width = 640
+    height = 360
+
+    max_work_group_size = 256
+    compute_units = 8
+
+    # Prüfe zunächst auf quadratische Größen, die Potenzen von zwei sind
+    for size in [32, 16, 8, 4]:
+        if width % size == 0 and height % size == 0 and size * size <= max_work_group_size:
+            return (size, size)
+
+    # Wenn keine geeignete quadratische Größe gefunden wurde, suche nach der besten rechteckigen Größe
+    for size in range(min(max_work_group_size, width), 0, -1):
+        if width % size == 0:
+            return (size, 1)
+
+    # Wenn keine passende Workgroup-Größe gefunden wurde, Fallback auf eine sehr kleine Größe
+    return (1, 1)
+
+
+def get_optimal_work_group_size3():
+    width = 640
+    height = 360
+
+    max_work_group_size = 256
+    compute_units = 8
+
+    # Priorisiere größere Workgroup-Größen, um die GPU-Auslastung zu maximieren
+    for size in [16, 8, 4]:
+        if width % size == 0 and height % size == 0 and size * size <= max_work_group_size:
+            return (size, size)
+
+    # Fallback auf eine kleinere rechteckige Workgroup-Größe, falls notwendig
+    return (min(max_work_group_size, width), 1)
+
+
+print(__get_optimal_work_group_size1())
+print(get_optimal_work_group_size2())
+print(get_optimal_work_group_size3())

package/src/decoder/python/test/opencl.cl

@@ -0,0 +1,70 @@
+__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;
+
+__kernel void process_frame(
+    __read_only image2d_t input,
+    __global float *background_model,
+    __global float *temp_buffer,
+    __write_only image2d_t output,
+    __constant float *kernel_values,
+    int kernel_size,
+    int width,
+    int height,
+    float alpha,
+    float threshold,
+    int dilation_size,
+    int first_frame
+) {
+    int x = get_global_id(0);
+    int y = get_global_id(1);
+    if (x >= width || y >= height) return;
+    int idx = y * width + x;
+
+    // Gaussian blur
+    int radius = (kernel_size - 1) / 2;
+    float blurred_value = 0.0f;
+    for (int i = -radius; i <= radius; i++) {
+        for (int j = -radius; j <= radius; j++) {
+            int2 samplePos = (int2)(clamp(x + i, 0, width - 1), clamp(y + j, 0, height - 1));
+            float4 pixel = read_imagef(input, sampler, samplePos);
+            float weight = kernel_values[i + radius] * kernel_values[j + radius];
+            blurred_value += pixel.x * weight;
+        }
+    }
+
+    // Store blurred value in temp buffer
+    temp_buffer[idx] = blurred_value;
+
+    if (first_frame) {
+        // Initialize background model
+        background_model[idx] = blurred_value;
+        write_imagef(output, (int2)(x, y), (float4)(0.0f, 0.0f, 0.0f, 1.0f));
+    } else {
+        // Background subtraction
+        float bg_pixel = background_model[idx];
+
+        // Update background model
+        float updated_bg = bg_pixel * (1.0f - alpha) + blurred_value * alpha;
+        background_model[idx] = updated_bg;
+
+        // Compute difference and threshold
+        float diff = fabs(blurred_value - bg_pixel);
+        float mask_value = (diff > threshold) ? 1.0f : 0.0f;
+
+        // Dilation
+        float maxValue = mask_value;
+        for (int dy = -dilation_size; dy <= dilation_size; dy++) {
+            int sy = clamp(y + dy, 0, height - 1);
+            for (int dx = -dilation_size; dx <= dilation_size; dx++) {
+                int sx = clamp(x + dx, 0, width - 1);
+                int neighbor_idx = sy * width + sx;
+                float neighbor_pixel = background_model[neighbor_idx];
+                float neighbor_blurred = temp_buffer[neighbor_idx];
+                float neighbor_diff = fabs(neighbor_blurred - neighbor_pixel);
+                float neighbor_mask = (neighbor_diff > threshold) ? 1.0f : 0.0f;
+                maxValue = fmax(maxValue, neighbor_mask);
+            }
+        }
+
+        write_imagef(output, (int2)(x, y), (float4)(maxValue, maxValue, maxValue, 1.0f));
+    }
+}

package/src/decoder/python/test/opencl2.cl

@@ -0,0 +1,130 @@
+__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;
+
+__kernel void gaussian_blur_combined(
+    __read_only image2d_t input,
+    __write_only image2d_t output,
+    __constant float *kernel_values,
+    int kernel_size
+) {
+    int x = get_global_id(0);
+    int y = get_global_id(1);
+    int width = get_image_width(input);
+    int height = get_image_height(input);
+    
+    if (x >= width || y >= height) return;
+    
+    int radius = (kernel_size - 1) / 2;
+    float sum = 0.0f;
+
+    // Kombinierter horizontaler und vertikaler Blur
+    for (int i = -radius; i <= radius; i++) {
+        for (int j = -radius; j <= radius; j++) {
+            int2 samplePos = (int2)(clamp(x + i, 0, width - 1), clamp(y + j, 0, height - 1));
+            float4 pixel = read_imagef(input, sampler, samplePos);
+            float weight = kernel_values[i + radius] * kernel_values[j + radius];
+            sum += pixel.x * weight;
+        }
+    }
+
+    write_imagef(output, (int2)(x, y), (float4)(sum, 0.0f, 0.0f, 1.0f));
+}
+
+__kernel void gaussian_blur_horizontal(
+    __read_only image2d_t input,
+    __write_only image2d_t output,
+    __global float *kernel_values,
+    int kernel_size
+) {
+    int x = get_global_id(0);
+    int y = get_global_id(1);
+
+    int width = get_image_width(input);
+    int radius = (kernel_size - 1) / 2;
+
+    if (x >= width) return;
+
+    float sum = 0.0f;
+    for (int k = -radius; k <= radius; k++) {
+        int sx = clamp(x + k, 0, width - 1);
+        float4 pixel = read_imagef(input, sampler, (int2)(sx, y));
+        float weight = kernel_values[k + radius];
+        sum += pixel.x * weight;
+    }
+
+    write_imagef(output, (int2)(x, y), (float4)(sum, 0.0f, 0.0f, 1.0f));
+}
+
+__kernel void gaussian_blur_vertical(
+    __read_only image2d_t input,
+    __write_only image2d_t output,
+    __global float *kernel_values,
+    int kernel_size
+) {
+    int x = get_global_id(0);
+    int y = get_global_id(1);
+
+    int height = get_image_height(input);
+    int radius = (kernel_size - 1) / 2;
+
+    if (y >= height) return;
+
+    float sum = 0.0f;
+    for (int k = -radius; k <= radius; k++) {
+        int sy = clamp(y + k, 0, height - 1);
+        float4 pixel = read_imagef(input, sampler, (int2)(x, sy));
+        float weight = kernel_values[k + radius];
+        sum += pixel.x * weight;
+    }
+
+    write_imagef(output, (int2)(x, y), (float4)(sum, 0.0f, 0.0f, 1.0f));
+}
+
+__kernel void background_subtraction(
+    __read_only image2d_t current_frame,
+    __global float *background_model,
+    __write_only image2d_t mask,
+    int width,
+    int height,
+    float alpha,
+    float threshold
+) {
+    int x = get_global_id(0);
+    int y = get_global_id(1);
+    if (x >= width || y >= height) return;
+    int idx = y * width + x;
+
+    float current_pixel = read_imagef(current_frame, sampler, (int2)(x, y)).x;
+    float bg_pixel = background_model[idx];
+
+    // Aktualisiere Hintergrundmodell
+    float updated_bg = bg_pixel * (1.0f - alpha) + current_pixel * alpha;
+    background_model[idx] = updated_bg;
+
+    // Berechne Differenz
+    float diff = fabs(current_pixel - bg_pixel);
+    float result = (diff > threshold) ? 1.0f : 0.0f;
+    write_imagef(mask, (int2)(x, y), (float4)(result, 0.0f, 0.0f, 1.0f));
+}
+
+__kernel void dilate(
+    __read_only image2d_t input,
+    __write_only image2d_t output,
+    int size
+) {
+    int x = get_global_id(0);
+    int y = get_global_id(1);
+    int width = get_image_width(input);
+    int height = get_image_height(input);
+    if (x >= width || y >= height) return;
+
+    float maxValue = 0.0f;
+    for (int dy = -size; dy <= size; dy++) {
+        int sy = clamp(y + dy, 0, height - 1);
+        for (int dx = -size; dx <= size; dx++) {
+            int sx = clamp(x + dx, 0, width - 1);
+            float pixel = read_imagef(input, sampler, (int2)(sx, sy)).x;
+            maxValue = fmax(maxValue, pixel);
+        }
+    }
+    write_imagef(output, (int2)(x, y), (float4)(maxValue, 0.0f, 0.0f, 1.0f));
+}