@camera.ui/camera-ui-coreml 0.0.7

package/CHANGELOG.md

@@ -0,0 +1,8 @@
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [X.X.X] - ???
+
+- Initial Release

package/CONTRIBUTING.md

@@ -0,0 +1 @@
+# Contributing

package/LICENSE.md

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2023-2024 seydx <dev@seydx.com>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1 @@
+# @camera.ui/camera-ui-coreml

package/config.schema.json

@@ -0,0 +1,43 @@
+{
+  "schema": {
+    "model": {
+      "type": "string",
+      "title": "Model",
+      "description": "Model to use for object detection",
+      "required": true,
+      "store": true,
+      "defaultValue": "yolov9m_320",
+      "enum": [
+        "yolo3-tinyu",
+        "yolo3-tinyu_320",
+        "yolov5nu",
+        "yolov5nu_320",
+        "yolov5mu",
+        "yolov5mu_320",
+        "yolov8n",
+        "yolov8n_320",
+        "yolov8s",
+        "yolov8s_320",
+        "yolov8m",
+        "yolov8m_320",
+        "yolov9t",
+        "yolov9t_320",
+        "yolov9s",
+        "yolov9s_320",
+        "yolov9m",
+        "yolov9m_320",
+        "yolov9c",
+        "yolov9c_320"
+      ]
+    },
+    "precision": {
+      "type": "string",
+      "title": "Precision",
+      "description": "Precision of the model",
+      "required": true,
+      "store": true,
+      "defaultValue": "FP16",
+      "enum": ["INT8", "FP16"]
+    }
+  }
+}

package/package.json

@@ -0,0 +1,45 @@
+{
+  "displayName": "CoreML",
+  "name": "@camera.ui/camera-ui-coreml",
+  "version": "0.0.7",
+  "description": "camera.ui coreml plugin",
+  "author": "seydx (https://github.com/seydx/camera.ui)",
+  "main": "./src/main.py",
+  "type": "module",
+  "scripts": {
+    "format": "ruff format",
+    "install-updates": "npm i --save",
+    "lint": "ruff check --fix",
+    "update": "updates --update ./",
+    "prepublishOnly": "npm i --package-lock-only && npm run format && npm run lint"
+  },
+  "dependencies": {},
+  "devDependencies": {
+    "updates": "^16.4.0"
+  },
+  "bugs": {
+    "url": "https://github.com/seydx/camera.ui/issues"
+  },
+  "engines": {
+    "camera.ui": ">=0.0.1",
+    "node": ">=18.12.0"
+  },
+  "homepage": "https://github.com/seydx/camera.ui#readme",
+  "keywords": [
+    "camera-ui-plugin",
+    "motion",
+    "object",
+    "coreml",
+    "detection"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/seydx/camera.ui.git"
+  },
+  "camera.ui": {
+    "pythonVersion": "3.11",
+    "extension": "objectDetection",
+    "dependencies": []
+  }
+}

package/requirements.txt

@@ -0,0 +1,4 @@
+coremltools==7.2
+camera-ui-python-types==0.1.81
+requests==2.32.3
+Pillow==10.4.0

package/src/camera_detector.py

@@ -0,0 +1,158 @@
+import asyncio
+from typing import Any, Optional, cast
+from uuid import uuid4
+
+from camera_ui_python_types import (
+    CameraDevice,
+    CameraStorage,
+    Detection,
+    MotionState,
+    ObjectClass,
+    PluginAPI,
+    PluginLogger,
+)
+from object_detector import ObjectDetector
+from plugin_typings import CameraStorageValues
+
+background_tasks = set[asyncio.Task[Any]]()
+
+
+class CameraDetector:
+    def __init__(
+        self,
+        camera: CameraDevice,
+        object_detector: ObjectDetector,
+        api: PluginAPI,
+        logger: PluginLogger,
+        has_motion: list[str],
+    ) -> None:
+        super().__init__()
+
+        self.camera = camera
+        self.api = api
+        self.logger = logger
+        self.object_detector = object_detector
+
+        self.has_motion = has_motion
+        self.started = False
+        self.closed = False
+        self.frame_generation_task: Optional[asyncio.Task[Any]] = None
+
+        self.camera_storage: CameraStorage[CameraStorageValues] = self.__create_camera_storage()
+        self.camera.on_connected.subscribe(lambda connected: self.start() if connected else self.close())
+        self.camera.on_motion_detected.subscribe(self.__on_motion_detected)
+
+    def start(self):
+        if not self.started:
+            self.started = True
+            self.closed = False
+
+            self.logger.log(f"Starting object detection for camera {self.camera.name}...")
+
+            self.frame_generation_task = asyncio.create_task(
+                self.__detect(), name=f"ObjectDetection-{self.camera.name}"
+            )
+
+            background_tasks.add(self.frame_generation_task)
+            self.frame_generation_task.add_done_callback(
+                lambda _: background_tasks.remove(self.frame_generation_task)
+                if self.frame_generation_task in background_tasks
+                else None
+            )
+
+    def close(self):
+        if self.started and not self.closed:
+            self.started = False
+            self.closed = True
+
+            self.logger.log(f"Stopping object detection for camera {self.camera.name}...")
+
+            if self.frame_generation_task is not None:
+                self.frame_generation_task.cancel()
+                self.frame_generation_task = None
+
+        if self.camera.name in self.has_motion:
+            self.has_motion.remove(self.camera.name)
+
+    def restart(self):
+        self.logger.log(f"Restarting object detection for camera {self.camera.name}...")
+
+        self.close()
+
+        if self.camera.connected:
+            self.start()
+
+    async def __detect(self):
+        try:
+            async for frame in self.camera.get_motion_frames():
+                frame_image = await frame.to_image(
+                    {
+                        "format": {"to": "rgb"},
+                        "resize": {
+                            "width": self.object_detector.input_width,
+                            "height": self.object_detector.input_height,
+                        },
+                    }
+                )
+
+                detections: list[Detection] = []
+
+                class_ids, scores, boxes = await self.object_detector.detect(frame_image["image"])
+
+                for class_id, score, box in zip(class_ids, scores, boxes):
+                    x1, y1, x2, y2 = box
+
+                    detection: Detection = {
+                        "id": str(uuid4()),
+                        "label": cast(ObjectClass, self.object_detector.labels[class_id]),
+                        "confidence": score,
+                        "boundingBox": (float(x1), float(y1), float(x2), float(y2)),
+                        "inputWidth": self.object_detector.input_width,
+                        "inputHeight": self.object_detector.input_height,
+                        "origWidth": frame.metadata["origWidth"],
+                        "origHeight": frame.metadata["origHeight"],
+                    }
+
+                    detections.append(detection)
+
+                await self.camera.update_state("object", {"detections": detections})
+        except asyncio.CancelledError:
+            pass
+        except Exception as error:
+            self.logger.error(self.camera.name, "Error generating motion frames", error)
+        finally:
+            if not self.closed:
+                self.restart()
+
+    def __on_motion_detected(self, state: MotionState) -> None:
+        motion_detected = bool(state.get("state", False) or len(state["detections"]) > 0)
+
+        if motion_detected:
+            if self.camera.name not in self.has_motion:
+                self.has_motion.append(self.camera.name)
+        elif self.camera.name in self.has_motion:
+            self.has_motion.remove(self.camera.name)
+
+    def __create_camera_storage(self) -> CameraStorage[CameraStorageValues]:
+        camera_storage: Optional[CameraStorage[CameraStorageValues]] = (
+            self.api.storage_controller.get_camera_storage(self.camera.id)
+        )
+
+        if camera_storage is not None:
+            return camera_storage
+
+        return self.api.storage_controller.create_camera_storage(
+            self,
+            self.camera.id,
+            {
+                "active_object_detections": {
+                    "type": "string",
+                    "key": "active_object_detections",
+                    "title": "Active Object Detections",
+                    "description": "Active object detections",
+                    "readonly": True,
+                    "enum": [],
+                    "onGet": lambda: self.has_motion,
+                }
+            },
+        )

package/src/detector_defaults.py

@@ -0,0 +1,30 @@
+available_models = [
+    "yolo3-tinyu",
+    "yolo3-tinyu_320",
+    "yolov5nu",
+    "yolov5nu_320",
+    "yolov5mu",
+    "yolov5mu_320",
+    "yolov8n",
+    "yolov8n_320",
+    "yolov8s",
+    "yolov8s_320",
+    "yolov8m",
+    "yolov8m_320",
+    "yolov9t",
+    "yolov9t_320",
+    "yolov9s",
+    "yolov9s_320",
+    "yolov9m",
+    "yolov9m_320",
+    "yolov9c",
+    "yolov9c_320",
+    # "yolov10n",
+    # "yolov10n_320",
+    # "yolov10s",
+    # "yolov10s_320",
+    # "yolov10m",
+    # "yolov10m_320",
+]
+
+available_precision = ["INT8", "FP16"]

package/src/main.py

@@ -0,0 +1,128 @@
+from typing import Any, Union
+
+from camera_detector import CameraDetector
+from camera_ui_python_types import (
+    CameraDevice,
+    CameraExtension,
+    FormSubmitResponse,
+    ImageMetadata,
+    ObjectDetectionPlugin,
+    ObjectDetectionPluginResponse,
+    PluginAPI,
+    PluginLogger,
+    RootSchema,
+)
+from detector_defaults import available_models, available_precision
+from object_detector import ObjectDetector
+
+
+class CoreML(ObjectDetectionPlugin):
+    def __init__(self, logger: PluginLogger, api: PluginAPI):
+        self.api = api
+        self.logger = logger
+        self.camera_devices: dict[str, CameraDevice] = {}
+        self.detectors: dict[str, CameraDetector] = {}
+        self.object_detector = ObjectDetector(self.api, self.logger)
+        self.single_image_detectors: dict[str, ObjectDetector] = {}
+
+        self.has_motion: list[str] = []
+
+        self.api.on("finishLaunching", self.start)
+        self.api.on("shutdown", self.stop)
+
+        self.api.device_manager.on("cameraSelected", self.camera_selected)
+        self.api.device_manager.on("cameraDeselected", self.camera_deselected)
+
+    async def camera_selected(self, camera: CameraDevice, extension: CameraExtension) -> None:
+        self.logger.log(f"New camera added ({extension}): {camera.name}")
+        self.camera_devices[camera.id] = camera
+        self.create_detector(camera)
+
+    def camera_deselected(self, id: str, extension: CameraExtension) -> None:
+        camera = self.camera_devices.get(id)
+
+        if camera:
+            self.logger.log(f"Camera removed ({extension}): {camera.name}")
+            self.remove_detector(self.camera_devices[id])
+            del self.camera_devices[id]
+
+    def start(self) -> None:
+        self.logger.log("Plugin Started")
+
+        self.object_detector.initialize()
+
+        for camera_device in self.camera_devices.values():
+            self.create_detector(camera_device)
+
+    def stop(self) -> None:
+        self.logger.log("Plugin Stopped")
+
+        detectors_copy = list(self.detectors.values())
+
+        for detector in detectors_copy:
+            self.remove_detector(detector.camera)
+
+    async def onFormSubmit(self, action_id: str, payload: Any) -> Union[FormSubmitResponse, None]:
+        print("Form submitted", action_id, payload)
+
+    def configure_cameras(self, cameras: list[CameraDevice]) -> None:
+        for camera in cameras:
+            self.camera_devices[camera.id] = camera
+
+    def create_detector(self, camera: CameraDevice) -> None:
+        detector = CameraDetector(camera, self.object_detector, self.api, self.logger, self.has_motion)
+        self.detectors[camera.id] = detector
+
+    def remove_detector(self, camera: CameraDevice) -> None:
+        detector = self.detectors[camera.id]
+
+        if detector:
+            detector.close()
+            del self.detectors[camera.id]
+
+    def interfaceSchema(self) -> RootSchema | None:
+        return {
+            "schema": {
+                "models": {
+                    "type": "string",
+                    "title": "Model",
+                    "description": "Model to use for object detection",
+                    "required": True,
+                    "defaultValue": "yolov9m_320",
+                    "enum": available_models,
+                },
+                "precision": {
+                    "type": "string",
+                    "title": "Precision",
+                    "description": "Precision of the model",
+                    "required": True,
+                    "defaultValue": "FP16",
+                    "enum": available_precision,
+                },
+            }
+        }
+
+    async def detectObjects(
+        self, image_path: str, metadata: ImageMetadata, config: dict[str, Any]
+    ) -> ObjectDetectionPluginResponse:
+        model = config.get("models")
+        precision = config.get("precision")
+
+        identifier = f"{model}_{precision}"
+        single_image_detector = self.single_image_detectors.get(identifier)
+
+        if not single_image_detector:
+            single_image_detector = ObjectDetector(self.api, self.logger)
+            self.single_image_detectors[identifier] = single_image_detector
+
+        single_image_detector.initialize(model, precision)
+
+        detections = await single_image_detector.detect_single(image_path, metadata)
+
+        return {
+            "detections": detections,
+        }
+
+
+def __main__():
+    return CoreML

package/src/object_detector.py

@@ -0,0 +1,190 @@
+import asyncio
+import os
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Optional, cast
+from uuid import uuid4
+
+import coremltools as ct
+import numpy as np
+import requests
+from camera_ui_python_types import Detection, ImageMetadata, ObjectClass, PluginAPI, PluginLogger
+from detector_defaults import available_models, available_precision
+from PIL import Image
+from utils import parse_labels
+
+
+class ObjectDetector:
+    def __init__(
+        self,
+        api: PluginAPI,
+        logger: PluginLogger,
+    ):
+        self.api = api
+        self.logger = logger
+        self.config = self.api.config_service.all()
+
+        model_version = "v1"
+        self.model_path = os.path.join(f"{self.api.storage_path}/models/{model_version}")
+
+        self.initialized = False
+        self.minThreshold = 0.2
+        self.executor = ThreadPoolExecutor(max_workers=8)
+
+        self.selected_model = None
+        self.selected_precision = None
+
+    @property
+    def input_width(self) -> int:
+        return self.input_size[0]
+
+    @property
+    def input_height(self) -> int:
+        return self.input_size[1]
+
+    def initialize(self, model: Optional[str] = None, precision: Optional[str] = None) -> None:
+        if not self.initialized:
+            self.initialized = True
+
+            self.logger.log("Initializing CoreML Object Detector...")
+
+            self.selected_model = model if model else self.config.get("model", "yolov9m_320")
+            self.selected_precision = precision if precision else self.config.get("precision", "FP16")
+
+            if self.selected_model not in available_models:
+                self.selected_model = "yolov9m_320"
+
+            if self.selected_precision not in available_precision:
+                self.selected_precision = "FP16"
+
+            yolo_model_name = self.selected_model.split("_")[0]
+            model_file = None
+
+            files = [
+                f"coreml_{self.selected_model}/{self.selected_precision}/{yolo_model_name}.mlpackage/Data/com.apple.CoreML/weights/weight.bin",
+                f"coreml_{self.selected_model}/{self.selected_precision}/{yolo_model_name}.mlpackage/Data/com.apple.CoreML/model.mlmodel",
+                f"coreml_{self.selected_model}/{self.selected_precision}/{yolo_model_name}.mlpackage/Manifest.json",
+            ]
+
+            for f in files:
+                p = self.__downloadFile(
+                    f"https://raw.githubusercontent.com/seydx/models/main/models/coreml/{f}", f
+                )
+                model_file = os.path.dirname(p)
+
+            self.model = ct.models.MLModel(model=model_file, compute_units=ct.ComputeUnit.ALL)
+            self.input_name: str = self.model.get_spec().description.input[0].name  # type: ignore
+
+            modelspec: Any = self.model.get_spec()
+            inputdesc = modelspec.description.input[0]  # type: ignore
+            inputheight: Any = inputdesc.type.imageType.height
+            inputwidth: Any = inputdesc.type.imageType.width
+
+            self.input_size: tuple[int, int] = (inputwidth, inputheight)
+            self.labels = parse_labels(modelspec.description.metadata.userDefined)  # type: ignore
+
+            self.logger.log("CoreML Object Detector initialized")
+
+    async def detect(self, image: Image.Image) -> tuple[list[int], list[float], np.ndarray[Any, Any]]:
+        if not self.initialized:
+            raise Exception("CoreML Object Detector not initialized")
+
+        out_dict = await asyncio.get_event_loop().run_in_executor(self.executor, self.__model_predict, image)
+
+        coordinatesList = out_dict["coordinates"]
+        confidenceList = out_dict["confidence"]
+
+        indices = np.argmax(confidenceList, axis=1)
+        maxConfidences = confidenceList[np.arange(len(confidenceList)), indices]
+        valid = maxConfidences >= self.minThreshold
+
+        if not np.any(valid):
+            return [], [], np.array([])
+
+        coordinates = coordinatesList[valid]
+        class_ids = indices[valid]
+        scores = maxConfidences[valid]
+
+        x_center = self.__torelative(coordinates[:, 0], self.input_width)
+        y_center = self.__torelative(coordinates[:, 1], self.input_height)
+        width = self.__torelative(coordinates[:, 2], self.input_width)
+        height = self.__torelative(coordinates[:, 3], self.input_height)
+
+        xmin = np.maximum(0, x_center - width / 2)
+        ymin = np.maximum(0, y_center - height / 2)
+        xmax = np.minimum(self.input_width, x_center + width / 2)
+        ymax = np.minimum(self.input_height, y_center + height / 2)
+
+        boxes = np.vstack([xmin, ymin, xmax, ymax]).T
+        # dets = np.hstack([boxes, scores.reshape(-1, 1), class_ids.reshape(-1, 1)])
+
+        return class_ids.tolist(), scores.tolist(), boxes
+
+    async def detect_single(self, image_path: str, metadata: ImageMetadata) -> list[Detection]:
+        image = Image.open(image_path)
+        image = image.convert("RGB").resize((self.input_width, self.input_height))
+
+        detections: list[Detection] = []
+
+        class_ids, scores, boxes = await self.detect(image)
+
+        for class_id, score, box in zip(class_ids, scores, boxes):
+            x1, y1, x2, y2 = box
+
+            detection: Detection = {
+                "id": str(uuid4()),
+                "label": cast(ObjectClass, self.labels[class_id]),
+                "confidence": score,
+                "boundingBox": (float(x1), float(y1), float(x2), float(y2)),
+                "inputWidth": self.input_width,
+                "inputHeight": self.input_height,
+                "origWidth": metadata["width"],
+                "origHeight": metadata["height"],
+            }
+
+            detections.append(detection)
+
+        image.close()
+
+        return detections
+
+    def __torelative(self, values: int, size: int) -> int:
+        return values * size
+
+    def __model_predict(self, image: Image.Image) -> Any:
+        return self.model.predict({self.input_name: image})  # type: ignore
+
+    def __downloadFile(self, url: str, filename: str) -> str:
+        try:
+            fullpath = os.path.join(self.model_path, filename)
+
+            if os.path.isfile(fullpath):
+                return fullpath
+
+            tmp = fullpath + ".tmp"
+
+            self.logger.debug("Creating directory for", tmp)
+
+            os.makedirs(os.path.dirname(fullpath), exist_ok=True)
+
+            self.logger.debug("Downloading", url)
+
+            response = requests.get(url, stream=True)
+
+            if response.status_code < 200 or response.status_code >= 300:
+                raise Exception(f"Error downloading {url}: {response.status_code}")
+
+            read = 0
+
+            with open(tmp, "wb") as f:
+                for chunk in response.iter_content(chunk_size=1024 * 1024):
+                    if chunk:
+                        read += len(chunk)
+                        self.logger.debug("Downloaded", read, "bytes")
+                        f.write(chunk)
+
+            os.rename(tmp, fullpath)
+
+            return fullpath
+        except Exception as e:
+            self.logger.error("Error downloading", url, e)
+            raise

package/src/plugin_typings.py

@@ -0,0 +1,5 @@
+from typing import TypedDict
+
+
+class CameraStorageValues(TypedDict):
+    active_object_detections: list[str]

package/src/utils.py

@@ -0,0 +1,44 @@
+import ast
+import re
+from typing import Any
+
+
+def parse_label_contents(contents: str) -> dict[int, Any]:
+    lines = contents.split(",")
+    ret: dict[int, Any] = {}
+
+    for row_number, content in enumerate(lines):
+        pair = re.split(r"[:\s]+", content.strip(), maxsplit=1)
+        if len(pair) == 2 and pair[0].strip().isdigit():
+            ret[int(pair[0])] = pair[1].strip()
+        else:
+            ret[row_number] = content.strip()
+
+    return ret
+
+
+def parse_labels(userDefined: dict[str, Any]) -> list[str]:
+    yolo = userDefined.get("names") or userDefined.get("yolo.names")
+
+    if yolo:
+        j = ast.literal_eval(yolo)
+        ret = []
+        for _, v in j.items():
+            ret.append(v)
+
+        return ret
+
+    classes = userDefined.get("classes")
+
+    # print("classes", classes)
+
+    if not classes:
+        raise Exception("no classes found in model metadata")
+
+    cont = parse_label_contents(classes)
+    ret: list[str] = []
+
+    for _, v in cont.items():
+        ret.append(v)
+
+    return ret