@camera.ui/server 0.0.48 → 0.0.50-internal.0

package/src/camera/interfaces.ts

@@ -0,0 +1,95 @@
+import { rpc } from '@camera.ui/rpc-common';
+import { CameraDeviceCapability } from '@camera.ui/types';
+
+import { NamespaceManager } from '../rpc/namespaces.js';
+
+import type { Promisify, RPCConnection } from '@camera.ui/rpc-common';
+import type { PTZCommand, PTZInterface, SnapshotInterface, StreamingInterface } from '@camera.ui/types';
+import type { BehaviorSubject } from 'rxjs';
+import type { CameraInterfaces, PluginCapabilities } from './types.js';
+
+interface ProxyInstance {
+  createRPCConnection(namespace: string): Promise<RPCConnection>;
+}
+
+export class CameraInterface implements CameraInterfaces {
+  public interfaces?: PTZInterface | SnapshotInterface | StreamingInterface;
+
+  #proxy: ProxyInstance;
+  #capabilities: BehaviorSubject<PluginCapabilities[]>;
+  #cameraId: string;
+  #pluginId?: string;
+  #rpcConnections: { namespace: string; connection: RPCConnection }[] = [];
+
+  constructor(proxy: ProxyInstance, capabilities: BehaviorSubject<PluginCapabilities[]>, cameraId: string, pluginId?: string) {
+    this.#proxy = proxy;
+    this.#capabilities = capabilities;
+    this.#cameraId = cameraId;
+    this.#pluginId = pluginId;
+  }
+
+  @rpc({ public: true })
+  public async streamUrl(sourceName: string): Promise<string> {
+    const capability = this.#capabilities.getValue().find((cap) => cap.capabilities.some((c) => c === CameraDeviceCapability.StreamUrl));
+    if (!capability) {
+      throw new Error('No capability found for streamUrl');
+    }
+
+    if (this.interfaces && 'streamUrl' in this.interfaces) {
+      return this.interfaces.streamUrl(sourceName);
+    } else {
+      const cameraProxy = await this.#cameraProxy(capability.pluginId);
+      return cameraProxy.streamUrl(sourceName);
+    }
+  }
+
+  @rpc({ public: true })
+  public async snapshot(sourceId: string, forceNew?: boolean): Promise<ArrayBuffer | undefined> {
+    const capability = this.#capabilities.getValue().find((cap) => cap.capabilities.some((c) => c === CameraDeviceCapability.Snapshot));
+    if (!capability) {
+      throw new Error('No capability found for snapshot');
+    }
+
+    if (this.interfaces && 'snapshot' in this.interfaces) {
+      return this.interfaces.snapshot(sourceId, forceNew);
+    } else {
+      const cameraProxy = await this.#cameraProxy(capability.pluginId);
+      return cameraProxy.snapshot(sourceId, forceNew);
+    }
+  }
+
+  @rpc({ public: true })
+  public async ptzCommand(command: PTZCommand): Promise<void> {
+    const capability = this.#capabilities.getValue().find((cap) => cap.capabilities.some((c) => c === CameraDeviceCapability.PTZ));
+    if (!capability) {
+      throw new Error('No capability found for PTZ');
+    }
+
+    if (this.interfaces && 'ptzCommand' in this.interfaces) {
+      return this.interfaces.ptzCommand(command);
+    } else {
+      const cameraProxy = await this.#cameraProxy(capability.pluginId);
+      return cameraProxy.ptzCommand(command);
+    }
+  }
+
+  public async disconnect(): Promise<void> {
+    await Promise.allSettled(this.#rpcConnections.map((rpc) => rpc.connection.disconnect()));
+    this.#rpcConnections = [];
+  }
+
+  async #cameraProxy(pluginId: string): Promise<Promisify<CameraInterfaces>> {
+    if (pluginId === this.#pluginId) {
+      throw new Error('Can not create camera proxy for the same plugin');
+    }
+
+    const namespace = NamespaceManager.pluginCameraNamespaces(pluginId, this.#cameraId).cameraInterfacesRpc;
+    const rpcConnection = await this.#proxy.createRPCConnection(namespace);
+
+    if (!this.#rpcConnections.some((rpc) => rpc.namespace === namespace)) {
+      this.#rpcConnections.push({ namespace, connection: rpcConnection });
+    }
+
+    return rpcConnection.createProxy<CameraInterfaces>();
+  }
+}

package/src/camera/iou.ts

@@ -0,0 +1,153 @@
+import type { BoundingBox, Detection } from '@camera.ui/types';
+
+// Simple spatial index using grid-based bucketing
+class SpatialIndex {
+  private cells = new Map<string, Detection[]>();
+  private cellSize: number;
+
+  constructor(cellSize = 100) {
+    this.cellSize = cellSize;
+  }
+
+  private getCellsForBox(box: BoundingBox, margin = 0): string[] {
+    const minCellX = Math.floor((box[0] - margin) / this.cellSize);
+    const minCellY = Math.floor((box[1] - margin) / this.cellSize);
+    const maxCellX = Math.floor((box[2] + margin) / this.cellSize);
+    const maxCellY = Math.floor((box[3] + margin) / this.cellSize);
+
+    const cells: string[] = [];
+    for (let x = minCellX; x <= maxCellX; x++) {
+      for (let y = minCellY; y <= maxCellY; y++) {
+        cells.push(`${x},${y}`);
+      }
+    }
+    return cells;
+  }
+
+  insert(detection: Detection): void {
+    const box = detection.boundingBox;
+    const cells = this.getCellsForBox(box);
+
+    for (const cell of cells) {
+      if (!this.cells.has(cell)) {
+        this.cells.set(cell, []);
+      }
+      this.cells.get(cell)!.push(detection);
+    }
+  }
+
+  search(box: BoundingBox, margin = 0): Detection[] {
+    const cells = this.getCellsForBox(box, margin);
+    const result = new Set<Detection>();
+
+    for (const cell of cells) {
+      const detections = this.cells.get(cell) ?? [];
+      for (const detection of detections) {
+        result.add(detection);
+      }
+    }
+
+    return Array.from(result);
+  }
+}
+
+// Helper functions
+function calculateIoU(box1: BoundingBox, box2: BoundingBox): number {
+  const intersectionX1 = Math.max(box1[0], box2[0]);
+  const intersectionY1 = Math.max(box1[1], box2[1]);
+  const intersectionX2 = Math.min(box1[2], box2[2]);
+  const intersectionY2 = Math.min(box1[3], box2[3]);
+
+  if (intersectionX2 < intersectionX1 || intersectionY2 < intersectionY1) {
+    return 0;
+  }
+
+  const intersectionArea = (intersectionX2 - intersectionX1) * (intersectionY2 - intersectionY1);
+  const box1Area = (box1[2] - box1[0]) * (box1[3] - box1[1]);
+  const box2Area = (box2[2] - box2[0]) * (box2[3] - box2[1]);
+
+  return intersectionArea / (box1Area + box2Area - intersectionArea);
+}
+
+function mergeBoundingBoxes(boxes: BoundingBox[]): BoundingBox {
+  const minX = Math.max(0, Math.round(Math.min(...boxes.map((box) => box[0]))));
+  const minY = Math.max(0, Math.round(Math.min(...boxes.map((box) => box[1]))));
+  const maxX = Math.min(1000, Math.round(Math.max(...boxes.map((box) => box[2]))));
+  const maxY = Math.min(1000, Math.round(Math.max(...boxes.map((box) => box[3]))));
+
+  return [minX, minY, maxX, maxY];
+}
+
+function generateId(): string {
+  return Math.random().toString(36).slice(2, 9);
+}
+
+function isValidBox(box: BoundingBox): boolean {
+  return box[0] < box[2] && box[1] < box[3];
+}
+
+export function mergeDetections(detections: Detection[], iouThreshold = 0.01, closeThreshold = 100): Detection[] {
+  if (detections.length === 0) {
+    return [];
+  }
+
+  // Create spatial index with cell size based on closeThreshold
+  const spatialIndex = new SpatialIndex(closeThreshold);
+  const processedIds = new Set<string>();
+  const mergedDetections: Detection[] = [];
+
+  // Prepare detections with IDs
+  const preparedDetections = detections.map((d) => ({
+    ...d,
+    id: d.id ?? generateId(),
+  }));
+
+  // Load spatial index
+  for (const detection of preparedDetections) {
+    spatialIndex.insert(detection);
+  }
+
+  for (const detection of preparedDetections) {
+    if (processedIds.has(detection.id)) continue;
+
+    const currentDetection = { ...detection };
+    let maxConfidence = currentDetection.confidence;
+    const overlappingBoxes: BoundingBox[] = [currentDetection.boundingBox];
+
+    // Find potential overlaps using spatial index
+    const searchMargin = Math.max(closeThreshold, 10);
+    const potentialOverlaps = spatialIndex.search(currentDetection.boundingBox, searchMargin);
+
+    for (const overlap of potentialOverlaps) {
+      if (overlap.id === currentDetection.id || processedIds.has(overlap.id!) || overlap.label !== currentDetection.label) {
+        continue;
+      }
+
+      const iou = calculateIoU(currentDetection.boundingBox, overlap.boundingBox);
+      const isClose =
+        Math.abs(currentDetection.boundingBox[0] - overlap.boundingBox[0]) <= closeThreshold &&
+        Math.abs(currentDetection.boundingBox[1] - overlap.boundingBox[1]) <= closeThreshold;
+
+      if (iou > iouThreshold || isClose) {
+        overlappingBoxes.push(overlap.boundingBox);
+        processedIds.add(overlap.id!);
+        maxConfidence = Math.max(maxConfidence, overlap.confidence);
+      }
+    }
+
+    if (overlappingBoxes.length > 1) {
+      const mergedBox = mergeBoundingBoxes(overlappingBoxes);
+      if (isValidBox(mergedBox)) {
+        currentDetection.boundingBox = mergedBox;
+        currentDetection.confidence = maxConfidence;
+      }
+    }
+
+    if (isValidBox(currentDetection.boundingBox)) {
+      mergedDetections.push(currentDetection);
+    }
+    processedIds.add(currentDetection.id);
+  }
+
+  return mergedDetections;
+}

package/src/camera/polygon.ts

@@ -0,0 +1,226 @@
+import type { BoundingBox, CameraDetectionSettings, Detection, DetectionZone, ObjectClass, Point } from '@camera.ui/types';
+
+function pointsEqual(p1: Point, p2: Point): boolean {
+  return p1[0] === p2[0] && p1[1] === p2[1];
+}
+
+function isBoxContainedInPolygon(box: BoundingBox, polygon: Point[]): boolean {
+  const corners: Point[] = [
+    [box[0], box[1]], // Top-left
+    [box[2], box[1]], // Top-right
+    [box[2], box[3]], // Bottom-right
+    [box[0], box[3]], // Bottom-left
+  ];
+
+  return corners.every((point) => isPointInPolygon(point, polygon));
+}
+
+function isBoxCompletelyOutsidePolygon(box: BoundingBox, polygon: Point[]): boolean {
+  const corners: Point[] = [
+    [box[0], box[1]], // Top-left
+    [box[2], box[1]], // Top-right
+    [box[2], box[3]], // Bottom-right
+    [box[0], box[3]], // Bottom-left
+  ];
+
+  return !doPolygonsIntersect(box, polygon) && corners.every((point) => !isPointInPolygon(point, polygon));
+}
+
+function isPointInPolygon(point: Point, polygon: Point[]): boolean {
+  const [x, y] = point;
+  let inside = false;
+
+  // Check if point is on the edge of the polygon
+  for (let i = 0; i < polygon.length; i++) {
+    const j = (i + 1) % polygon.length;
+    const [xi, yi] = polygon[i];
+    const [xj, yj] = polygon[j];
+
+    // Point lies on a horizontal or vertical edge
+    if (x >= Math.min(xi, xj) && x <= Math.max(xi, xj) && y >= Math.min(yi, yj) && y <= Math.max(yi, yj)) {
+      // Verticale Linie
+      if (Math.abs(xi - xj) < 0.000001) {
+        if (Math.abs(x - xi) < 0.000001) {
+          return true;
+        }
+      } else {
+        // Other lines
+        const m = (yj - yi) / (xj - xi);
+        const b = yi - m * xi;
+        const yOnLine = m * x + b;
+        if (Math.abs(y - yOnLine) < 0.000001) {
+          return true;
+        }
+      }
+    }
+
+    // Standard Ray-Casting Test
+    // eslint-disable-next-line @stylistic/no-mixed-operators
+    if (yi > y !== yj > y && x < ((xj - xi) * (y - yi)) / (yj - yi) + xi) {
+      inside = !inside;
+    }
+  }
+
+  return inside;
+}
+
+function doLinesIntersect(a1: Point, a2: Point, b1: Point, b2: Point): boolean {
+  const denominator = (b2[1] - b1[1]) * (a2[0] - a1[0]) - (b2[0] - b1[0]) * (a2[1] - a1[1]);
+  if (denominator === 0) return false;
+
+  const ua = ((b2[0] - b1[0]) * (a1[1] - b1[1]) - (b2[1] - b1[1]) * (a1[0] - b1[0])) / denominator;
+  const ub = ((a2[0] - a1[0]) * (a1[1] - b1[1]) - (a2[1] - a1[1]) * (a1[0] - b1[0])) / denominator;
+
+  return ua >= 0 && ua <= 1 && ub >= 0 && ub <= 1;
+}
+
+function doPolygonsIntersect(box: BoundingBox, polygon: Point[]): boolean {
+  const boxPolygon: Point[] = [
+    [box[0], box[1]], // Top-left
+    [box[2], box[1]], // Top-right
+    [box[2], box[3]], // Bottom-right
+    [box[0], box[3]], // Bottom-left
+    [box[0], box[1]], // Close the polygon
+  ];
+
+  // Check if any of the box corners are inside the polygon
+  if (boxPolygon.some((point) => isPointInPolygon(point, polygon))) {
+    return true;
+  }
+
+  if (polygon.some((point) => point[0] >= box[0] && point[0] <= box[2] && point[1] >= box[1] && point[1] <= box[3])) {
+    return true;
+  }
+
+  // Check if any of the polygon corners intersect with any of the box edges
+  for (let i = 0; i < boxPolygon.length - 1; i++) {
+    for (let j = 0; j < polygon.length - 1; j++) {
+      if (doLinesIntersect(boxPolygon[i], boxPolygon[i + 1], polygon[j], polygon[j + 1])) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+export const convertZonePointsToPixels = (zone: DetectionZone, detection: Detection): DetectionZone => {
+  const { origWidth, origHeight } = detection;
+  const convertedPoints: Point[] = zone.points.map(([x, y]) => [(x * origWidth) / 100, (y * origHeight) / 100]);
+
+  if (convertedPoints.length > 0 && !pointsEqual(convertedPoints[0], convertedPoints[convertedPoints.length - 1])) {
+    convertedPoints.push(convertedPoints[0]);
+  }
+
+  return {
+    ...zone,
+    points: convertedPoints,
+  };
+};
+
+export const convertBoundingBoxToPixels = (detection: Detection): Detection => {
+  const { origWidth, origHeight, inputWidth, inputHeight } = detection;
+  let [xmin, ymin, xmax, ymax] = detection.boundingBox;
+
+  xmin = (xmin * origWidth) / inputWidth;
+  ymin = (ymin * origHeight) / inputHeight;
+  xmax = (xmax * origWidth) / inputWidth;
+  ymax = (ymax * origHeight) / inputHeight;
+
+  xmin = Math.max(0, Math.min(xmin, origWidth));
+  ymin = Math.max(0, Math.min(ymin, origHeight));
+  xmax = Math.max(0, Math.min(xmax, origWidth));
+  ymax = Math.max(0, Math.min(ymax, origHeight));
+
+  return {
+    ...detection,
+    boundingBox: [xmin, ymin, xmax, ymax],
+  };
+};
+
+export const areDetectionsValid = (detections: Detection[], zones: DetectionZone[], activitySettings: CameraDetectionSettings): Detection[] => {
+  if (zones.length === 0) {
+    return [];
+  }
+
+  if (detections.length === 0) {
+    return [];
+  }
+
+  const validClasses = new Set<ObjectClass>();
+  for (const zone of zones) {
+    if (zone.classes.length > 0) {
+      zone.classes.forEach((cls) => validClasses.add(cls));
+    }
+  }
+
+  return detections.filter((detection) => {
+    // Check if detection confidence is above threshold
+    if (detection.confidence < activitySettings.object.confidence) {
+      return false;
+    }
+
+    // Check if detection class is valid
+    if (!validClasses.has(detection.label)) {
+      return false;
+    }
+
+    const convertedDetection = convertBoundingBoxToPixels(detection);
+
+    // Check if detection is inside a privacy mask
+    for (const zone of zones) {
+      if (!zone.isPrivacyMask) continue;
+
+      const convertedZone = convertZonePointsToPixels(zone, convertedDetection);
+      if (isBoxContainedInPolygon(convertedDetection.boundingBox, convertedZone.points)) {
+        return false;
+      }
+    }
+
+    let hasIncludeZone = false;
+    let isValid = false;
+
+    for (const zone of zones) {
+      // Skip privacy masks
+      if (zone.isPrivacyMask) continue;
+
+      // Skip zones that do not include the detection class
+      if (!zone.classes.includes(detection.label)) {
+        continue;
+      }
+
+      const convertedZone = convertZonePointsToPixels(zone, convertedDetection);
+      const intersects = doPolygonsIntersect(convertedDetection.boundingBox, convertedZone.points);
+      const contained = isBoxContainedInPolygon(convertedDetection.boundingBox, convertedZone.points);
+
+      if (zone.filter === 'include') {
+        // include
+        hasIncludeZone = true;
+        if ((zone.type === 'contain' && contained) || (zone.type === 'intersect' && (intersects || contained))) {
+          isValid = true;
+          break;
+        }
+      } else {
+        // exclude
+        if (zone.type === 'contain') {
+          if (!isBoxCompletelyOutsidePolygon(convertedDetection.boundingBox, convertedZone.points)) {
+            return false;
+          }
+        } else {
+          // intersect/contained
+          if (intersects || contained) {
+            return false;
+          }
+        }
+      }
+    }
+
+    // If there are no include zones and the detection has not been blocked by a matching exclude zone
+    // or privacy mask, it is valid
+    if (!hasIncludeZone) {
+      return true;
+    }
+
+    return isValid;
+  });
+};

package/src/camera/python/classes.py

@@ -0,0 +1,41 @@
+from _camera_ui_tools.camera_ui_types import Detection, ObjectClass
+
+
+def categorize_coco_class(class_name: str) -> ObjectClass:
+    if class_name == "person":
+        return "person"
+
+    if class_name == "motion":
+        return "motion"
+
+    animals = {
+        "bird",
+        "cat",
+        "dog",
+        "horse",
+        "sheep",
+        "cow",
+        "elephant",
+        "bear",
+        "zebra",
+        "giraffe",
+        "teddy bear",
+    }
+    if class_name in animals:
+        return "animal"
+
+    vehicles = {"bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat"}
+    if class_name in vehicles:
+        return "vehicle"
+
+    return "other"
+
+
+def transform_labels(detections: list[Detection]) -> list[Detection]:
+    transformed_detections: list[Detection] = []
+    for detection in detections:
+        new_detection = detection.copy()
+        new_detection["label"] = categorize_coco_class(detection["label"])
+        transformed_detections.append(new_detection)
+
+    return transformed_detections

package/src/camera/python/interfaces.py

@@ -0,0 +1,130 @@
+import asyncio
+from contextlib import suppress
+from typing import Optional, TypedDict, Union, cast
+
+from reactivex.subject.behaviorsubject import BehaviorSubject
+
+from _camera_ui_tools.camera_ui_rpc_client import ClientInstance
+from _camera_ui_tools.camera_ui_rpc_client.base.rpc import RPCConnection
+from _camera_ui_tools.camera_ui_rpc_client.utils.rpc import rpc
+from _camera_ui_tools.camera_ui_types import (
+    CameraDeviceCapability,
+    PTZCommand,
+    PTZInterface,
+    SnapshotInterface,
+    StreamingInterface,
+)
+from plugins.runtime.python.namespaces import NamespaceManager
+from plugins.runtime.python.typings import CameraInterfaces, PluginCapabilities
+
+
+class RPCConnectionInfo(TypedDict):
+    namespace: str
+    connection: RPCConnection
+
+
+class CameraInterface(CameraInterfaces):
+    def __init__(
+        self,
+        proxy: ClientInstance,
+        capabilities: BehaviorSubject[list[PluginCapabilities]],
+        camera_id: str,
+        plugin_id: Optional[str] = None,
+    ):
+        self.__proxy = proxy
+        self.__capabilities = capabilities
+        self.__camera_id = camera_id
+        self.__plugin_id = plugin_id
+
+        self.__rpc_connection: list[RPCConnectionInfo] = []
+        self.interfaces: Optional[Union[PTZInterface, SnapshotInterface, StreamingInterface]] = None
+
+    @rpc(public=True)
+    async def streamUrl(self, source_name: str) -> str:
+        capability = next(
+            (
+                cap
+                for cap in self.__capabilities.value
+                if any(c == CameraDeviceCapability.StreamUrl for c in cap["capabilities"])
+            ),
+            None,
+        )
+
+        if not capability:
+            raise RuntimeError("No capability found for streamUrl")
+
+        if self.interfaces and hasattr(self.interfaces, "streamUrl"):
+            interfaces = cast(StreamingInterface, self.interfaces)
+            return await interfaces.streamUrl(source_name)
+        else:
+            camera_proxy = await self.__camera_proxy(capability["pluginId"])
+            return await camera_proxy.streamUrl(source_name)
+
+    @rpc(public=True)
+    async def snapshot(self, source_id: str, force_new: Optional[bool] = None) -> Optional[bytes]:
+        capability = next(
+            (
+                cap
+                for cap in self.__capabilities.value
+                if any(c == CameraDeviceCapability.Snapshot for c in cap["capabilities"])
+            ),
+            None,
+        )
+
+        if not capability:
+            raise RuntimeError("No capability found for snapshot")
+
+        if self.interfaces and hasattr(self.interfaces, "snapshot"):
+            interfaces = cast(SnapshotInterface, self.interfaces)
+            return await interfaces.snapshot(source_id, force_new)
+        else:
+            camera_proxy = await self.__camera_proxy(capability["pluginId"])
+            return await camera_proxy.snapshot(source_id, force_new)
+
+    @rpc(public=True)
+    async def ptzCommand(self, command: PTZCommand) -> None:
+        capability = next(
+            (
+                cap
+                for cap in self.__capabilities.value
+                if any(c == CameraDeviceCapability.PTZ for c in cap["capabilities"])
+            ),
+            None,
+        )
+
+        if not capability:
+            raise RuntimeError("No capability found for PTZ")
+
+        if self.interfaces and hasattr(self.interfaces, "ptzCommand"):
+            interfaces = cast(PTZInterface, self.interfaces)
+            return await interfaces.ptzCommand(command)
+        else:
+            camera_proxy = await self.__camera_proxy(capability["pluginId"])
+            return await camera_proxy.ptzCommand(command)
+
+    async def disconnect(self) -> None:
+        with suppress(Exception):
+            await asyncio.gather(*[rpc["connection"].disconnect() for rpc in self.__rpc_connection])
+
+    async def __camera_proxy(self, plugin_id: str) -> CameraInterfaces:
+        if plugin_id == self.__plugin_id:
+            raise RuntimeError("Can not create camera proxy for the same plugin")
+
+        namespace = NamespaceManager.plugin_camera_namespaces(plugin_id, self.__camera_id)[
+            "cameraInterfacesRpc"
+        ]
+        rpc_connection = await self.__proxy.create_rpc_connection(namespace)
+
+        rpc_connection_exists = next(
+            (rpc for rpc in self.__rpc_connection if rpc["namespace"] == namespace), None
+        )
+
+        if not rpc_connection_exists:
+            self.__rpc_connection.append(
+                {
+                    "namespace": namespace,
+                    "connection": rpc_connection,
+                }
+            )
+
+        return rpc_connection.create_proxy(CameraInterfaces)