@camera.ui/camera-ui-coreml 0.0.21 → 0.0.23

package/package.json

@@ -1,28 +1,29 @@
 {
   "displayName": "CoreML",
   "name": "@camera.ui/camera-ui-coreml",
-  "version": "0.0.21",
+  "version": "0.0.23",
   "description": "camera.ui coreml plugin",
   "author": "seydx (https://github.com/seydx/camera.ui)",
-  "main": "./src/main.py",
-  "type": "module",
+  "main": "./dist/main.py",
+  "type": "commonjs",
   "scripts": {
+    "bundle": "npm run format && npm run lint && cui bundle",
     "format": "ruff format",
     "install-updates": "npm i --save",
-    "lint": "ruff check --fix",
-    "update": "updates --update ./",
-    "prepublishOnly": "npm i --package-lock-only && npm run format && npm run lint"
-  },
-  "dependencies": {},
-  "devDependencies": {
-    "updates": "^16.4.0"
+    "lint": "ruff check",
+    "lint:fix": "ruff check --fix",
+    "prepublishOnly": "node -e \"if(!process.env.SAFE_PUBLISH){console.error('Error: Please use @camera.ui/cli to publish the plugin:\\n  npm run publish:alpha\\n  npm run publish:beta\\n  npm run publish:latest\\n');process.exit(1)}\"",
+    "publish:alpha": "cui publish --alpha",
+    "publish:beta": "cui publish --beta",
+    "publish:latest": "cui publish --latest",
+    "update": "updates --update ./"
   },
   "bugs": {
     "url": "https://github.com/seydx/camera.ui/issues"
   },
   "engines": {
-    "camera.ui": ">=0.0.34-alpha.1",
-    "node": ">=20.17.0"
+    "camera.ui": ">=0.0.34-alpha.2",
+    "node": ">=20.18.0"
   },
   "homepage": "https://github.com/seydx/camera.ui#readme",
   "keywords": [
@@ -40,6 +41,7 @@
   "camera.ui": {
     "pythonVersion": "3.11",
     "extension": "objectDetection",
-    "dependencies": []
+    "dependencies": [],
+    "bundled": true
   }
-}
+}

package/CHANGELOG.md

@@ -1,8 +0,0 @@
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [X.X.X] - ???
-
-- Initial Release

package/CONTRIBUTING.md

@@ -1 +0,0 @@
-# Contributing

package/LICENSE.md

@@ -1,22 +0,0 @@
-MIT License
-
-Copyright (c) 2023-2024 seydx <dev@seydx.com>
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -1 +0,0 @@
-# @camera.ui/camera-ui-coreml

package/config.schema.json

@@ -1,43 +0,0 @@
-{
-  "schema": {
-    "model": {
-      "type": "string",
-      "title": "Model",
-      "description": "Model to use for object detection",
-      "required": true,
-      "store": true,
-      "defaultValue": "yolov9m_320",
-      "enum": [
-        "yolo3-tinyu",
-        "yolo3-tinyu_320",
-        "yolov5nu",
-        "yolov5nu_320",
-        "yolov5mu",
-        "yolov5mu_320",
-        "yolov8n",
-        "yolov8n_320",
-        "yolov8s",
-        "yolov8s_320",
-        "yolov8m",
-        "yolov8m_320",
-        "yolov9t",
-        "yolov9t_320",
-        "yolov9s",
-        "yolov9s_320",
-        "yolov9m",
-        "yolov9m_320",
-        "yolov9c",
-        "yolov9c_320"
-      ]
-    },
-    "precision": {
-      "type": "string",
-      "title": "Precision",
-      "description": "Precision of the model",
-      "required": true,
-      "store": true,
-      "defaultValue": "FP16",
-      "enum": ["INT8", "FP16"]
-    }
-  }
-}

package/requirements.txt

@@ -1,4 +0,0 @@
-coremltools==7.2
-camera-ui-python-types==0.1.93
-requests==2.32.3
-Pillow==10.4.0

package/src/camera_detector.py

@@ -1,162 +0,0 @@
-import asyncio
-from typing import Any, Optional, cast
-from uuid import uuid4
-
-from camera_ui_python_types import (
-    CameraDevice,
-    CameraStorage,
-    Detection,
-    LoggerService,
-    MotionState,
-    ObjectClass,
-    PluginAPI,
-)
-from object_detector import ObjectDetector
-from plugin_typings import CameraStorageValues
-
-background_tasks = set[asyncio.Task[Any]]()
-
-
-class CameraDetector:
-    def __init__(
-        self,
-        camera: CameraDevice,
-        object_detector: ObjectDetector,
-        api: PluginAPI,
-        logger: LoggerService,
-        has_motion: list[str],
-    ) -> None:
-        super().__init__()
-
-        self.camera = camera
-        self.api = api
-        self.logger = logger
-        self.object_detector = object_detector
-
-        self.has_motion = has_motion
-        self.started = False
-        self.closed = False
-        self.frame_generation_task: Optional[asyncio.Task[Any]] = None
-
-        self.camera_storage: CameraStorage[CameraStorageValues] = self.__create_camera_storage()
-        self.camera.on_connected.subscribe(lambda connected: self.start() if connected else self.close())
-        self.camera.on_motion_detected.subscribe(self.__on_motion_detected)
-
-    def start(self):
-        if not self.started:
-            self.started = True
-            self.closed = False
-
-            self.logger.log(f"Starting object detection for camera {self.camera.name}...")
-
-            self.frame_generation_task = asyncio.create_task(
-                self.__detect(), name=f"ObjectDetection-{self.camera.name}"
-            )
-
-            background_tasks.add(self.frame_generation_task)
-            self.frame_generation_task.add_done_callback(
-                lambda _: background_tasks.remove(self.frame_generation_task)
-                if self.frame_generation_task in background_tasks
-                else None
-            )
-
-    def close(self):
-        if self.started and not self.closed:
-            self.started = False
-            self.closed = True
-
-            self.logger.log(f"Stopping object detection for camera {self.camera.name}...")
-
-            if self.frame_generation_task is not None:
-                self.frame_generation_task.cancel()
-                self.frame_generation_task = None
-
-        if self.camera.name in self.has_motion:
-            self.has_motion.remove(self.camera.name)
-
-    def restart(self):
-        self.logger.log(f"Restarting object detection for camera {self.camera.name}...")
-
-        self.close()
-
-        if self.camera.connected:
-            self.start()
-
-    async def __detect(self):
-        while not self.closed:
-            try:
-                async for frame in self.camera.get_motion_frames():
-                    if self.closed:
-                        break
-
-                    frame_image = await frame.to_image(
-                        {
-                            "format": {"to": "rgb"},
-                            "resize": {
-                                "width": self.object_detector.input_width,
-                                "height": self.object_detector.input_height,
-                            },
-                        }
-                    )
-
-                    detections: list[Detection] = []
-
-                    class_ids, scores, boxes = await self.object_detector.detect(frame_image["image"])
-
-                    for class_id, score, box in zip(class_ids, scores, boxes):
-                        x1, y1, x2, y2 = box
-
-                        detection: Detection = {
-                            "id": str(uuid4()),
-                            "label": cast(ObjectClass, self.object_detector.labels[class_id]),
-                            "confidence": score,
-                            "boundingBox": (float(x1), float(y1), float(x2), float(y2)),
-                            "inputWidth": self.object_detector.input_width,
-                            "inputHeight": self.object_detector.input_height,
-                            "origWidth": frame.metadata["origWidth"],
-                            "origHeight": frame.metadata["origHeight"],
-                        }
-
-                        detections.append(detection)
-
-                    await self.camera.update_state("object", {"detections": detections})
-            except asyncio.CancelledError:
-                break
-            except Exception as error:
-                self.logger.error(self.camera.name, "Error generating motion frames", error)
-                self.logger.log(self.camera.name, "Restarting object detection in 5 seconds...")
-                await asyncio.sleep(5)
-                continue
-
-    def __on_motion_detected(self, state: MotionState) -> None:
-        motion_detected = bool(state.get("state", False) or len(state["detections"]) > 0)
-
-        if motion_detected:
-            if self.camera.name not in self.has_motion:
-                self.has_motion.append(self.camera.name)
-        elif self.camera.name in self.has_motion:
-            self.has_motion.remove(self.camera.name)
-
-    def __create_camera_storage(self) -> CameraStorage[CameraStorageValues]:
-        camera_storage: Optional[CameraStorage[CameraStorageValues]] = (
-            self.api.storage_controller.get_camera_storage(self.camera.id)
-        )
-
-        if camera_storage is not None:
-            return camera_storage
-
-        return self.api.storage_controller.create_camera_storage(
-            self,
-            self.camera.id,
-            {
-                "active_object_detections": {
-                    "type": "string",
-                    "key": "active_object_detections",
-                    "title": "Active Object Detections",
-                    "description": "Active object detections",
-                    "readonly": True,
-                    "enum": [],
-                    "onGet": lambda: self.has_motion,
-                }
-            },
-        )

package/src/detector_defaults.py

@@ -1,32 +0,0 @@
-available_models = [
-    "yolo3-tinyu",
-    "yolo3-tinyu_320",
-    "yolov5nu",
-    "yolov5nu_320",
-    "yolov5mu",
-    "yolov5mu_320",
-    "yolov8n",
-    "yolov8n_320",
-    "yolov8s",
-    "yolov8s_320",
-    "yolov8m",
-    "yolov8m_320",
-    "yolov9t",
-    "yolov9t_320",
-    "yolov9s",
-    "yolov9s_320",
-    "yolov9m",
-    "yolov9m_320",
-    "yolov9c",
-    "yolov9c_320",
-    "yolov10n",
-    "yolov10n_320",
-    "yolov10s",
-    "yolov10s_320",
-    "yolov10m",
-    "yolov10m_320",
-    "yolov10l",
-    "yolov10l_320",
-]
-
-available_precision = ["INT8", "FP16"]

package/src/main.py

@@ -1,128 +0,0 @@
-from typing import Any, Union
-
-from camera_detector import CameraDetector
-from camera_ui_python_types import (
-    CameraDevice,
-    CameraExtension,
-    FormSubmitResponse,
-    ImageMetadata,
-    LoggerService,
-    ObjectDetectionPlugin,
-    ObjectDetectionPluginResponse,
-    PluginAPI,
-    RootSchema,
-)
-from detector_defaults import available_models, available_precision
-from object_detector import ObjectDetector
-
-
-class CoreML(ObjectDetectionPlugin):
-    def __init__(self, logger: LoggerService, api: PluginAPI):
-        self.api = api
-        self.logger = logger
-        self.camera_devices: dict[str, CameraDevice] = {}
-        self.detectors: dict[str, CameraDetector] = {}
-        self.object_detector = ObjectDetector(self.api, self.logger)
-        self.single_image_detectors: dict[str, ObjectDetector] = {}
-
-        self.has_motion: list[str] = []
-
-        self.api.on("finishLaunching", self.start)
-        self.api.on("shutdown", self.stop)
-
-        self.api.device_manager.on("cameraSelected", self.camera_selected)
-        self.api.device_manager.on("cameraDeselected", self.camera_deselected)
-
-    async def camera_selected(self, camera: CameraDevice, extension: CameraExtension) -> None:
-        self.logger.log(f"New camera added ({extension}): {camera.name}")
-        self.camera_devices[camera.id] = camera
-        self.create_detector(camera)
-
-    def camera_deselected(self, id: str, extension: CameraExtension) -> None:
-        camera = self.camera_devices.get(id)
-
-        if camera:
-            self.logger.log(f"Camera removed ({extension}): {camera.name}")
-            self.remove_detector(self.camera_devices[id])
-            del self.camera_devices[id]
-
-    def start(self) -> None:
-        self.logger.log("Plugin Started")
-
-        self.object_detector.initialize()
-
-        for camera_device in self.camera_devices.values():
-            self.create_detector(camera_device)
-
-    def stop(self) -> None:
-        self.logger.log("Plugin Stopped")
-
-        detectors_copy = list(self.detectors.values())
-
-        for detector in detectors_copy:
-            self.remove_detector(detector.camera)
-
-    async def onFormSubmit(self, action_id: str, payload: Any) -> Union[FormSubmitResponse, None]:
-        print("Form submitted", action_id, payload)
-
-    def configureCameras(self, cameras: list[CameraDevice]) -> None:
-        for camera in cameras:
-            self.camera_devices[camera.id] = camera
-
-    def create_detector(self, camera: CameraDevice) -> None:
-        detector = CameraDetector(camera, self.object_detector, self.api, self.logger, self.has_motion)
-        self.detectors[camera.id] = detector
-
-    def remove_detector(self, camera: CameraDevice) -> None:
-        detector = self.detectors[camera.id]
-
-        if detector:
-            detector.close()
-            del self.detectors[camera.id]
-
-    def interfaceSchema(self) -> RootSchema | None:
-        return {
-            "schema": {
-                "models": {
-                    "type": "string",
-                    "title": "Model",
-                    "description": "Model to use for object detection",
-                    "required": True,
-                    "defaultValue": "yolov9m_320",
-                    "enum": available_models,
-                },
-                "precision": {
-                    "type": "string",
-                    "title": "Precision",
-                    "description": "Precision of the model",
-                    "required": True,
-                    "defaultValue": "FP16",
-                    "enum": available_precision,
-                },
-            }
-        }
-
-    async def detectObjects(
-        self, image_path: str, metadata: ImageMetadata, config: dict[str, Any]
-    ) -> ObjectDetectionPluginResponse:
-        model = config.get("models")
-        precision = config.get("precision")
-
-        identifier = f"{model}_{precision}"
-        single_image_detector = self.single_image_detectors.get(identifier)
-
-        if not single_image_detector:
-            single_image_detector = ObjectDetector(self.api, self.logger)
-            self.single_image_detectors[identifier] = single_image_detector
-
-        single_image_detector.initialize(model, precision)
-
-        detections = await single_image_detector.detect_single(image_path, metadata)
-
-        return {
-            "detections": detections,
-        }
-
-
-def __main__():
-    return CoreML

package/src/object_detector.py

@@ -1,190 +0,0 @@
-import asyncio
-import os
-from concurrent.futures import ThreadPoolExecutor
-from typing import Any, Optional, cast
-from uuid import uuid4
-
-import coremltools as ct
-import numpy as np
-import requests
-from camera_ui_python_types import Detection, ImageMetadata, LoggerService, ObjectClass, PluginAPI
-from detector_defaults import available_models, available_precision
-from PIL import Image
-from utils import parse_labels
-
-
-class ObjectDetector:
-    def __init__(
-        self,
-        api: PluginAPI,
-        logger: LoggerService,
-    ):
-        self.api = api
-        self.logger = logger
-        self.config = self.api.config_service.all()
-
-        model_version = "v1"
-        self.model_path = os.path.join(f"{self.api.storage_path}/models/{model_version}")
-
-        self.initialized = False
-        self.minThreshold = 0.2
-        self.executor = ThreadPoolExecutor(max_workers=8)
-
-        self.selected_model = None
-        self.selected_precision = None
-
-    @property
-    def input_width(self) -> int:
-        return self.input_size[0]
-
-    @property
-    def input_height(self) -> int:
-        return self.input_size[1]
-
-    def initialize(self, model: Optional[str] = None, precision: Optional[str] = None) -> None:
-        if not self.initialized:
-            self.initialized = True
-
-            self.logger.log("Initializing CoreML Object Detector...")
-
-            self.selected_model = model if model else self.config.get("model", "yolov9m_320")
-            self.selected_precision = precision if precision else self.config.get("precision", "FP16")
-
-            if self.selected_model not in available_models:
-                self.selected_model = "yolov9m_320"
-
-            if self.selected_precision not in available_precision:
-                self.selected_precision = "FP16"
-
-            yolo_model_name = self.selected_model.split("_")[0]
-            model_file = None
-
-            files = [
-                f"coreml_{self.selected_model}/{self.selected_precision}/{yolo_model_name}.mlpackage/Data/com.apple.CoreML/weights/weight.bin",
-                f"coreml_{self.selected_model}/{self.selected_precision}/{yolo_model_name}.mlpackage/Data/com.apple.CoreML/model.mlmodel",
-                f"coreml_{self.selected_model}/{self.selected_precision}/{yolo_model_name}.mlpackage/Manifest.json",
-            ]
-
-            for f in files:
-                p = self.__downloadFile(
-                    f"https://raw.githubusercontent.com/seydx/models/main/models/coreml/{f}", f
-                )
-                model_file = os.path.dirname(p)
-
-            self.model = ct.models.MLModel(model=model_file, compute_units=ct.ComputeUnit.ALL)
-            self.input_name: str = self.model.get_spec().description.input[0].name  # type: ignore
-
-            modelspec: Any = self.model.get_spec()
-            inputdesc = modelspec.description.input[0]  # type: ignore
-            inputheight: Any = inputdesc.type.imageType.height
-            inputwidth: Any = inputdesc.type.imageType.width
-
-            self.input_size: tuple[int, int] = (inputwidth, inputheight)
-            self.labels = parse_labels(modelspec.description.metadata.userDefined)  # type: ignore
-
-            self.logger.log("CoreML Object Detector initialized")
-
-    async def detect(self, image: Image.Image) -> tuple[list[int], list[float], np.ndarray[Any, Any]]:
-        if not self.initialized:
-            raise Exception("CoreML Object Detector not initialized")
-
-        out_dict = await asyncio.get_event_loop().run_in_executor(self.executor, self.__model_predict, image)
-
-        coordinatesList = out_dict["coordinates"]
-        confidenceList = out_dict["confidence"]
-
-        indices = np.argmax(confidenceList, axis=1)
-        maxConfidences = confidenceList[np.arange(len(confidenceList)), indices]
-        valid = maxConfidences >= self.minThreshold
-
-        if not np.any(valid):
-            return [], [], np.array([])
-
-        coordinates = coordinatesList[valid]
-        class_ids = indices[valid]
-        scores = maxConfidences[valid]
-
-        x_center = self.__torelative(coordinates[:, 0], self.input_width)
-        y_center = self.__torelative(coordinates[:, 1], self.input_height)
-        width = self.__torelative(coordinates[:, 2], self.input_width)
-        height = self.__torelative(coordinates[:, 3], self.input_height)
-
-        xmin = np.maximum(0, x_center - width / 2)
-        ymin = np.maximum(0, y_center - height / 2)
-        xmax = np.minimum(self.input_width, x_center + width / 2)
-        ymax = np.minimum(self.input_height, y_center + height / 2)
-
-        boxes = np.vstack([xmin, ymin, xmax, ymax]).T
-        # dets = np.hstack([boxes, scores.reshape(-1, 1), class_ids.reshape(-1, 1)])
-
-        return class_ids.tolist(), scores.tolist(), boxes
-
-    async def detect_single(self, image_path: str, metadata: ImageMetadata) -> list[Detection]:
-        image = Image.open(image_path)
-        image = image.convert("RGB").resize((self.input_width, self.input_height))
-
-        detections: list[Detection] = []
-
-        class_ids, scores, boxes = await self.detect(image)
-
-        for class_id, score, box in zip(class_ids, scores, boxes):
-            x1, y1, x2, y2 = box
-
-            detection: Detection = {
-                "id": str(uuid4()),
-                "label": cast(ObjectClass, self.labels[class_id]),
-                "confidence": score,
-                "boundingBox": (float(x1), float(y1), float(x2), float(y2)),
-                "inputWidth": self.input_width,
-                "inputHeight": self.input_height,
-                "origWidth": metadata["width"],
-                "origHeight": metadata["height"],
-            }
-
-            detections.append(detection)
-
-        image.close()
-
-        return detections
-
-    def __torelative(self, values: int, size: int) -> int:
-        return values * size
-
-    def __model_predict(self, image: Image.Image) -> Any:
-        return self.model.predict({self.input_name: image})  # type: ignore
-
-    def __downloadFile(self, url: str, filename: str) -> str:
-        try:
-            fullpath = os.path.join(self.model_path, filename)
-
-            if os.path.isfile(fullpath):
-                return fullpath
-
-            tmp = fullpath + ".tmp"
-
-            self.logger.debug("Creating directory for", tmp)
-
-            os.makedirs(os.path.dirname(fullpath), exist_ok=True)
-
-            self.logger.debug("Downloading", url)
-
-            response = requests.get(url, stream=True)
-
-            if response.status_code < 200 or response.status_code >= 300:
-                raise Exception(f"Error downloading {url}: {response.status_code}")
-
-            read = 0
-
-            with open(tmp, "wb") as f:
-                for chunk in response.iter_content(chunk_size=1024 * 1024):
-                    if chunk:
-                        read += len(chunk)
-                        self.logger.debug("Downloaded", read, "bytes")
-                        f.write(chunk)
-
-            os.rename(tmp, fullpath)
-
-            return fullpath
-        except Exception as e:
-            self.logger.error("Error downloading", url, e)
-            raise

package/src/plugin_typings.py

@@ -1,5 +0,0 @@
-from typing import TypedDict
-
-
-class CameraStorageValues(TypedDict):
-    active_object_detections: list[str]

package/src/utils.py

@@ -1,44 +0,0 @@
-import ast
-import re
-from typing import Any
-
-
-def parse_label_contents(contents: str) -> dict[int, Any]:
-    lines = contents.split(",")
-    ret: dict[int, Any] = {}
-
-    for row_number, content in enumerate(lines):
-        pair = re.split(r"[:\s]+", content.strip(), maxsplit=1)
-        if len(pair) == 2 and pair[0].strip().isdigit():
-            ret[int(pair[0])] = pair[1].strip()
-        else:
-            ret[row_number] = content.strip()
-
-    return ret
-
-
-def parse_labels(userDefined: dict[str, Any]) -> list[str]:
-    yolo = userDefined.get("names") or userDefined.get("yolo.names")
-
-    if yolo:
-        j = ast.literal_eval(yolo)
-        ret = []
-        for _, v in j.items():
-            ret.append(v)
-
-        return ret
-
-    classes = userDefined.get("classes")
-
-    # print("classes", classes)
-
-    if not classes:
-        raise Exception("no classes found in model metadata")
-
-    cont = parse_label_contents(classes)
-    ret: list[str] = []
-
-    for _, v in cont.items():
-        ret.append(v)
-
-    return ret