@camera.ui/server 0.0.15 → 0.0.17

package/dist/server/src/polyglot/python/plugins/camera_storage.py

@@ -1,8 +1,6 @@
 import inspect
-import json
 from contextlib import suppress
 from copy import deepcopy
-from pathlib import Path
 from typing import Any, Optional, TypeVar, Union, cast
 
 from camera_ui_types import (
@@ -24,42 +22,16 @@ from schema import (
 )
 from typing_extensions import Literal, overload
 from utilities.object_path import ObjectPath
-from utilities.utils import merge_with
+from utilities.utils import merge, merge_with
 
 P = TypeVar("P", str, int, float, bool, dict[Any, Any], list[Any], None)
 
 
-def merge(
-    obj_value: Any,
-    src_value: Any,
-    key: Union[str, int],
-    object: Union[dict[str, Any], list[Any]],
-    source: Union[dict[str, Any], list[Any]],
-    stack: list[Any],
-) -> Any:
-    if isinstance(obj_value, list):
-        return src_value
-
-
-class JsonLmdb(Lmdb[Literal["config"], dict[str, Any]]):
-    def _pre_key(self, key: Literal["config"]) -> bytes:
-        return key.encode("utf-8")
-
-    def _post_key(self, key: bytes) -> Literal["config"]:
-        _key: Any = key.decode("utf-8")
-        return _key
-
-    def _pre_value(self, value: dict[str, Any]) -> bytes:
-        return json.dumps(value).encode("utf-8")
-
-    def _post_value(self, value: bytes) -> dict[str, Any]:
-        return json.loads(value.decode("utf-8"))
-
-
 class CameraStorage:
     def __init__(
         self,
         api: PluginAPI,
+        plugin_db: Lmdb[Literal["config"], dict[str, Any]],
         instance: Any,
         camera_id: str,
         schema: Optional[JsonSchemaForm] = None,
@@ -67,15 +39,12 @@ class CameraStorage:
         if schema is None:
             schema = {}
         self.__api = api
+        self.__plugin_db = plugin_db
         self.__instance = instance
         self.__camera_id = camera_id
         self.schema: JsonSchemaForm = schema or {}
         self.values: dict[str, Any] = {}
 
-        db_path = Path(api.storage_path) / "volume"
-        self.__db: Lmdb[Literal["config"], dict[str, Any]] = JsonLmdb.open(file=str(db_path), flag="c")
-
-        self.__api.device_manager.once("cameraDeselected", self.__deselected)
         self.__api.once("shutdown", self.__close)
 
     def initializeStorage(self) -> None:
@@ -84,17 +53,10 @@ class CameraStorage:
         This will load the configuration from the database.
         """
 
-        camera_config: dict[str, Any] = {}
-
-        try:
-            config: Any = self.__db.get("config") or {}
-            camera_config = config.get(self.__camera_id, {})
-        except:  # noqa: E722
-            pass
-
+        config: Any = self.__plugin_db.get("config") or {}
+        camera_config = config.get(self.__camera_id, {})
         schema_config = generate_json_from_root_schema({"schema": self.schema})
-
-        self.values = merge_with(schema_config, camera_config, merge, [])
+        self.values = merge_with(schema_config, camera_config, merge)
 
         self.__save_db()
 
@@ -228,7 +190,7 @@ class CameraStorage:
         elif isinstance(schema_or_path, dict) and schema is None:
             new_schema = schema_or_path
 
-            self.schema = merge_with(self.schema, new_schema, merge, [])
+            self.schema = merge_with(self.schema, new_schema, merge)
 
             await self.__resolve_on_get_functions(new_schema)
 
@@ -258,7 +220,7 @@ class CameraStorage:
             ObjectPath.delete(self.values, path)
 
             if self.__contains_storable_schema(schema):
-                self.__remove_db()
+                self.__save_db()
 
     async def changeSchema(self, path: str, new_schema: dict[str, Any]) -> None:
         """
@@ -274,7 +236,7 @@ class CameraStorage:
         schema: Optional[JsonSchema] = ObjectPath.get(self.schema, schema_path) or None
 
         if schema:
-            merge_with(schema, new_schema, merge, [])
+            merge_with(schema, new_schema, merge)
             ObjectPath.set(self.schema, schema_path, schema)
 
             await self.__resolve_on_get_functions(schema, schema_path)
@@ -362,7 +324,7 @@ class CameraStorage:
                 schema_path = config_path_to_schema_path(new_path)
                 schema_item: Optional[JsonBaseSchema] = get_value_by_path(self.schema, schema_path)
 
-                if schema_item:
+                if schema_item and schema_item["type"] != "button":
                     on_set_function = schema_item.get("onSet", None)
                     if on_set_function:
                         if inspect.iscoroutinefunction(on_set_function):
@@ -417,25 +379,13 @@ class CameraStorage:
         return False
 
     def __save_db(self) -> None:
-        config: Any = self.__db.get("config") or {}
+        config: Any = self.__plugin_db.get("config") or {}
         storable_config = self.__filter_storable_values(self.schema)
 
         config[self.__camera_id] = storable_config
 
-        self.__db.update({"config": config})
-
-    def __remove_db(self) -> None:
-        config: Any = self.__db.get("config") or {}
-
-        if self.__camera_id in config:
-            del config[self.__camera_id]
-            self.__db.update(config)
-
-    def __deselected(self, id: str) -> None:
-        if id == self.__camera_id:
-            self.__remove_db()
+        self.__plugin_db.update({"config": config})
 
     def __close(self) -> None:
         with suppress(Exception):
             self.__save_db()
-            self.__db.close()

package/dist/server/src/polyglot/python/plugins/child.py

@@ -7,11 +7,13 @@ import sys
 import traceback
 from asyncio import AbstractEventLoop
 from contextlib import suppress
+from pathlib import Path
 from types import ModuleType
-from typing import Any, Optional, cast
+from typing import Any, Literal, Optional, cast
 
 from camera_ui_types import BasePlugin, CameraDevice
 from decoder.typings import AuthConfig
+from lmdbm import Lmdb
 from message_queue import MessageQueue
 from nats.aio.msg import Msg
 from plugin_api import PluginAPI
@@ -27,6 +29,7 @@ from typings import (
     ProxyMessageStructure,
 )
 from utilities.connection import ProxyConnection
+from utilities.json_lmdb import JsonLmdb
 from utilities.packer import unpack
 from utilities.subscriptions import ProxySubscription
 from utilities.task import TaskSet
@@ -88,6 +91,7 @@ class PluginChild:
 
         self.display_name = sys.argv[1] if len(sys.argv) > 1 else "PluginChild"
         self.logger = PluginLogger(self.display_name, os.environ["LOGGER_DISABLE_TIMESTAMPS"] == "true")
+        self.plugin_db: Optional[Lmdb[Literal["config"], dict[str, Any]]] = None
 
         self.subject = f"{os.environ['WORKER_ID']}.ipc.subscriber"
         self.plugin_subject = f"{os.environ['PLUGIN_ID']}.child"
@@ -104,6 +108,7 @@ class PluginChild:
         self.plugin: Optional[BasePlugin] = None
 
         self.stopped = False
+        self.closed = False
 
     async def run(self):
         self.shutdown_event = asyncio.Event()
@@ -136,10 +141,28 @@ class PluginChild:
         plugin_info = data["plugin"]
         cameras = data["cameras"]
 
+        db_path = Path(storage_path) / "volume"
+        Path(db_path).mkdir(parents=True, exist_ok=True)
+
+        self.plugin_db = JsonLmdb.open(file=str(db_path), flag="c")
+
+        config: Any = self.plugin_db.get("config")
+        if config is None:
+            self.plugin_db.update({"config": {}})
+
         self.api = PluginAPI(
-            plugin_info, storage_path, config_file, self.logger, self.proxy_endpoints, self.auth
+            plugin_info,
+            storage_path,
+            config_file,
+            self.logger,
+            self.proxy_endpoints,
+            self.auth,
+            self.plugin_db,
         )
 
+        plugin_constructor = self.load_plugin()
+        self.plugin = cast(BasePlugin, plugin_constructor(self.logger, self.api))
+
         await asyncio.gather(
             self.api.core_manager.init(),
             self.api.device_manager.init(),
@@ -159,9 +182,6 @@ class PluginChild:
 
         await asyncio.gather(*(camera_device.init() for camera_device in cameras_devices))
 
-        plugin_constructor = self.load_plugin()
-        self.plugin = cast(BasePlugin, plugin_constructor(self.logger, self.api))
-
         self.api.device_manager.configure_cameras(cameras_devices)
         self.plugin.configure_cameras(cast(list[CameraDevice], cameras_devices))
 
@@ -175,6 +195,9 @@ class PluginChild:
 
         self.stopped = True
 
+        if self.api:
+            self.api.emit("shutdown")
+
         if self.message_queue:
             self.message_queue.abort_queue()
 
@@ -182,8 +205,10 @@ class PluginChild:
             with suppress(Exception):
                 await self.publisher.close()
 
-        if self.api:
-            self.api.emit("shutdown")
+        if self.plugin_db:
+            await asyncio.sleep(0.5)
+            with suppress(Exception):
+                self.plugin_db.close()
 
         self.shutdown_event.set()
 
@@ -268,9 +293,27 @@ class PluginChild:
 
         if "type" in message:
             if message["type"] == PLUGIN_COMMAND.START.value and "data" in message:
-                self.tasks.add(self.start(message["data"]))
+                await self.start(message["data"])
             elif message["type"] == PLUGIN_COMMAND.STOP.value:
-                self.tasks.add(self.stop())
+                await self.stop()
+
+    async def gracefully_close(self, signame: str, reason: Any = "unknown"):
+        if self.stopped or self.closed:
+            return
+
+        self.closed = True
+
+        self.logger.log(
+            f"{self.display_name} is gracefully closing from signal='{signame}' reason='{reason}'..."
+        )
+
+        try:
+            stop_task = asyncio.create_task(self.stop(), name=f"{self.display_name} Stop")
+            await asyncio.wait_for(stop_task, timeout=2)
+            os.kill(os.getpid(), signal.SIGKILL)
+        except asyncio.TimeoutError:
+            self.logger.warn("Failed to gracefully close before timeout, force quit!")
+            os.kill(os.getpid(), signal.SIGKILL)
 
     def setup_signal_handlers(self):
         loop = asyncio.get_running_loop()
@@ -279,20 +322,12 @@ class PluginChild:
         for signame in ("SIGINT", "SIGTERM"):
             loop.add_signal_handler(
                 getattr(signal, signame),
-                lambda signame=signame: self.gracefully_close(signame, getattr(signal, signame)),
+                lambda signame=signame: asyncio.create_task(
+                    self.gracefully_close(signame, getattr(signal, signame)),
+                    name=f"{self.display_name} Signal",
+                ),
             )
 
-    def gracefully_close(self, signame: str, reason: Any = "unknown"):
-        if self.stopped:
-            return
-
-        self.logger.log(
-            f"{self.display_name} is gracefully closing from signal='{signame}' reason='{reason}'..."
-        )
-
-        self.tasks.add(self.stop())
-        self.shutdown_event.set()
-
     def handle_exception(self, loop: AbstractEventLoop, context: Any):
         print("Exception context:", context)
         traceback.print_exc()
@@ -308,9 +343,10 @@ class PluginChild:
             self.logger.error(f"Caught exception: {msg}")
 
         if loop.is_running():
-            self.gracefully_close("uncaughtException", msg)
+            asyncio.create_task(self.gracefully_close("uncaughtException", msg))
 
 
 if __name__ == "__main__":
     plugin_child = PluginChild()
-    asyncio.run(plugin_child.run(), debug=True)
+    asyncio.run(plugin_child.run())
+    # asyncio.run(plugin_child.run(), debug=True)

package/dist/server/src/polyglot/python/plugins/config_service.py

@@ -163,7 +163,14 @@ class PluginConfigService:
         self.set(path_segments[0], self.__config[path_segments[0]], write)
 
     def __init(self) -> None:
-        os.makedirs(os.path.dirname(self.__config_file), exist_ok=True)
+        directory = os.path.dirname(self.__config_file)
+
+        if not os.path.exists(directory):
+            os.makedirs(directory)
+
+        if not os.path.exists(self.__config_file):
+            open(self.__config_file, "a").close()
+
         self.__read()
 
     def __read(self) -> None:

package/dist/server/src/polyglot/python/plugins/plugin_api.py

@@ -1,5 +1,8 @@
+from typing import Any, Literal
+
 from config_service import PluginConfigService
 from decoder.typings import AuthConfig
+from lmdbm import Lmdb
 from plugin_logger import PluginLogger
 from pyeecui.asyncio import AsyncIOEventEmitter
 from typings import PluginInfo
@@ -14,6 +17,7 @@ class PluginAPI(AsyncIOEventEmitter):
         logger: PluginLogger,
         proxy_endpoints: list[str],
         auth: AuthConfig,
+        plugin_db: Lmdb[Literal["config"], dict[str, Any]],
     ) -> None:
         super().__init__()
 
@@ -25,6 +29,6 @@ class PluginAPI(AsyncIOEventEmitter):
         self.config_file = config_file
 
         self.config_service = PluginConfigService(config_file)
-        self.storage_controller = StorageController(self)
+        self.storage_controller = StorageController(self, plugin_db)
         self.core_manager = CoreManager(self, logger, plugin, proxy_endpoints, auth)
         self.device_manager = DeviceManager(self, logger, plugin, proxy_endpoints, auth)

package/dist/server/src/polyglot/python/plugins/proxy/camera_device.py

@@ -10,6 +10,7 @@ from uuid import uuid4
 
 import aiohttp
 from cachetools import LRUCache, cached
+from camera.iou import merge_detections
 from camera.polygon import are_motion_detections_valid, are_object_detections_valid
 from camera.video_frame import MotionFrame, VideoFrame
 from camera_ui_types import (
@@ -68,6 +69,7 @@ from typings import (
     ClientType,
     Delegates,
     DeserializedError,
+    FrameSubscribtion,
     PluginInfo,
     ProxyMessageStructure,
 )
@@ -190,13 +192,13 @@ class CameraDeviceProxy:
         self.camera_subscriber: Optional[ProxySubscription] = None
         self.camera_plugin_subscriber: Optional[ProxySubscription] = None
         self.signaling_subscriber: Optional[ProxySubscription] = None
-        self.frame_subscriber: Optional[ProxySubscription] = None
-        self.motion_frame_subscriber: Optional[ProxySubscription] = None
+        self.frame_subscribers: list[FrameSubscribtion] = []
+        self.motion_frame_subscribers: list[FrameSubscribtion] = []
 
         self.tasks: TaskSet = TaskSet(f"{self.plugin['name']} - {self.name} - CameraDeviceProxy")
         self.message_queue: Optional[MessageQueue] = None
         self.active_requests: set[str] = set()
-        self.active_motion_requests = 0
+        self.active_motion_requests: set[str] = set()
 
         # print(("INIT camera device with clientSubject", self.subject))
 
@@ -425,24 +427,24 @@ class CameraDeviceProxy:
 
         if prebuffer_duration > 0:
             request_type = "prebuffer"
-            prebuffer_duration = 5 if prebuffer_duration > 5 else prebuffer_duration
+            prebuffer_duration = min(prebuffer_duration, 5)
 
         request_id = f"{str(uuid4())}-{request_type}"
 
         try:
             metadata = await self.__connect_to_decoder(request_id)
+            frame_subscriber = await self.frame_publisher.subscribe(self.frame_subject)
 
-            if not self.frame_subscriber:
-                self.frame_subscriber = await self.frame_publisher.subscribe(self.frame_subject)
+            self.frame_subscribers.append({"request_id": request_id, "subscriber": frame_subscriber})
 
-            async for msg in self.frame_subscriber.subscriber.messages:
-                try:
+            try:
+                async for msg in frame_subscriber.subscriber.messages:
                     if (
                         not self.initialized
                         or len(self.active_requests) == 0
                         or request_id not in self.active_requests
                     ):
-                        break
+                        return
 
                     if len(msg.data) == 0:
                         continue
@@ -450,35 +452,39 @@ class CameraDeviceProxy:
                     frame_data: FrameData = unpack(msg.data)
 
                     yield VideoFrame(self.id, self.frame_publisher, frame_data, metadata, prebuffer_duration)
-                except Exception as e:
-                    if self.initialized:
-                        self.logger.error("Error while processing frame:", e)
-                    else:
-                        break
+
+            except GeneratorExit:
+                return
 
         except asyncio.CancelledError:
-            pass
+            return
         except Exception as e:
             if self.initialized:
                 self.logger.error("Error while generating frames:", e)
+            raise
         finally:
-            self.tasks.add(self.__disconnect_from_decoder(request_id))
+            await self.__disconnect_from_decoder(request_id)
 
     async def get_motion_frames(self) -> AsyncGenerator[MotionFrame, None]:
         if not self.motion_frame_publisher:
             raise Exception("Motion Frame Publisher is not initialized")
 
-        self.active_motion_requests += 1
+        request_id = f"{str(uuid4())}-motion"
 
         try:
-            if not self.motion_frame_subscriber:
-                self.motion_frame_subscriber = await self.motion_frame_publisher.subscribe(
-                    self.motion_frame_subject
-                )
+            motion_frame_subscriber = await self.motion_frame_publisher.subscribe(self.motion_frame_subject)
 
-            async for msg in self.motion_frame_subscriber.subscriber.messages:
-                try:
-                    if not self.initialized or self.active_motion_requests <= 0:
+            self.motion_frame_subscribers.append(
+                {"request_id": request_id, "subscriber": motion_frame_subscriber}
+            )
+
+            try:
+                async for msg in motion_frame_subscriber.subscriber.messages:
+                    if (
+                        not self.initialized
+                        or len(self.active_motion_requests) == 0
+                        or request_id not in self.active_motion_requests
+                    ):
                         break
 
                     if not msg or not msg.data or len(msg.data) == 0:
@@ -499,22 +505,30 @@ class CameraDeviceProxy:
                         motion_state,
                         motion_frame["metadata"],
                     )
-                except Exception as e:
-                    if self.initialized:
-                        self.logger.error("Error while processing motion frame:", e)
-                    else:
-                        break
+
+            except GeneratorExit:
+                return
 
         except asyncio.CancelledError:
             pass
         except Exception as e:
             if self.initialized:
                 self.logger.error("Error while generating motion frames:", e)
+            raise
         finally:
-            self.active_motion_requests -= 1
+            if request_id in self.active_motion_requests:
+                self.active_motion_requests.remove(request_id)
+
+            motion_frame_subscribtion = next(
+                (sub for sub in self.motion_frame_subscribers if sub["request_id"] == request_id), None
+            )
+
+            if motion_frame_subscribtion:
+                self.motion_frame_subscribers.remove(motion_frame_subscribtion)
+                await motion_frame_subscribtion["subscriber"].close()
 
-            if self.active_motion_requests <= 0:
-                self.tasks.add(self.__close_motion_frame_subscriber())
+            if len(self.motion_frame_subscribers) == 0:
+                await self.__close_motion_frame_subscribers()
 
     @overload
     def get_value(self, state_name: Literal["light"]) -> LightState: ...
@@ -913,6 +927,8 @@ class CameraDeviceProxy:
                 det["boundingBox"] = cast(tuple[int, int, int, int], tuple(det["boundingBox"]))
                 det["confidence"] = float(det["confidence"])
 
+            motion_event["detections"] = merge_detections(motion_event["detections"])
+
             motion_event["detections"] = are_motion_detections_valid(
                 motion_event["detections"], self.activity_zones, self.activity_settings
             )
@@ -1433,37 +1449,48 @@ class CameraDeviceProxy:
             metadata = metadata_response.get("metadata")
 
             return metadata
-        except:  # noqa: E722
-            self.logger.error("Failed to get metadata! Frame Worker FFmpeg process might not be running")
+        except Exception as e:
+            self.logger.error("Failed to get metadata! Frame Worker FFmpeg process might not be running", e)
 
     async def __disconnect_from_decoder(self, request_id: str) -> None:
         if request_id in self.active_requests:
             self.active_requests.remove(request_id)
 
-        if not self.signaling_publisher:
-            raise Exception("Signaling Publisher not initialized")
+        frame_subscription = next(
+            (sub for sub in self.frame_subscribers if sub["request_id"] == request_id), None
+        )
 
-        message: SignalingRequest = {
-            "type": "disconnect",
-            "requestId": request_id,
-            "cameraId": self.id,
-            "pluginId": self.plugin["id"],
-        }
+        if frame_subscription:
+            self.frame_subscribers.remove(frame_subscription)
+            await frame_subscription["subscriber"].close()
 
-        await self.signaling_publisher.publish(self.signaling_target_subject, message)
+        if self.signaling_publisher:
+            message: SignalingRequest = {
+                "type": "disconnect",
+                "requestId": request_id,
+                "cameraId": self.id,
+                "pluginId": self.plugin["id"],
+            }
 
-    async def __close_frame_subscriber(self) -> None:
-        with suppress(Exception):
-            if self.frame_subscriber:
-                await self.frame_subscriber.close()
-                self.frame_subscriber = None
+            await self.signaling_publisher.publish(self.signaling_target_subject, message)
+
+    async def __close_frame_subscribers(self) -> None:
+        self.active_requests.clear()
+
+        for sub in self.frame_subscribers:
+            with suppress(Exception):
+                await sub["subscriber"].close()
 
-    async def __close_motion_frame_subscriber(self) -> None:
-        if self.motion_frame_subscriber:
+        self.frame_subscribers.clear()
+
+    async def __close_motion_frame_subscribers(self) -> None:
+        self.active_motion_requests.clear()
+
+        for sub in self.motion_frame_subscribers:
             with suppress(Exception):
-                await self.motion_frame_subscriber.close()
+                await sub["subscriber"].close()
 
-            self.motion_frame_subscriber = None
+        self.motion_frame_subscribers.clear()
 
     async def __request_handler(self, message: ProxyMessageStructure) -> None:
         if not self.message_queue:
@@ -1565,8 +1592,16 @@ class CameraDeviceProxy:
             if message["requestId"] in self.active_requests:
                 self.active_requests.remove(message["requestId"])
 
+            frame_subscribtion = next(
+                (sub for sub in self.frame_subscribers if sub["request_id"] == message["requestId"]), None
+            )
+
+            if frame_subscribtion:
+                self.frame_subscribers.remove(frame_subscribtion)
+                await frame_subscribtion["subscriber"].close()
+
             if len(self.active_requests) == 0:
-                self.tasks.add(self.__close_frame_subscriber())
+                self.tasks.add(self.__close_frame_subscribers())
 
     async def __respond_to_signaling_message(self, msg: Msg, message: PingMessage) -> None:
         try:

package/dist/server/src/polyglot/python/plugins/proxy/core_manager.py

@@ -41,10 +41,10 @@ class CoreManager(AsyncIOEventEmitter):
         self.cm_subject = "coreManager.subscriber"
 
         self.publisher: Optional[ProxyConnection] = None
-        self.dm_publisher: Optional[ProxyConnection] = None
+        self.cm_publisher: Optional[ProxyConnection] = None
 
         self.subscriber: Optional[ProxySubscription] = None
-        self.dm_subscriber: Optional[ProxySubscription] = None
+        self.cm_subscriber: Optional[ProxySubscription] = None
 
         self.tasks: TaskSet = TaskSet(f"{self.plugin['name']} CoreManager")
         self.message_queue: Optional[MessageQueue] = None
@@ -64,8 +64,8 @@ class CoreManager(AsyncIOEventEmitter):
         self.publisher = ProxyConnection(self.subject, servers, self.auth)
         await self.publisher.connect()
 
-        self.dm_publisher = ProxyConnection(self.cm_subject, servers, self.auth)
-        await self.dm_publisher.connect()
+        self.cm_publisher = ProxyConnection(self.cm_subject, servers, self.auth)
+        await self.cm_publisher.connect()
 
         self.subscriber = await self.publisher.subscribe(subject=self.subject)
         self.message_queue = MessageQueue(
@@ -75,8 +75,8 @@ class CoreManager(AsyncIOEventEmitter):
             self.__request_handler,
         )
 
-        self.dm_subscriber = await self.dm_publisher.subscribe(subject=self.cm_subject)
-        self.dm_subscriber.on("message", self.__on_proxy_message)
+        self.cm_subscriber = await self.cm_publisher.subscribe(subject=self.cm_subject)
+        self.cm_subscriber.on("message", self.__on_proxy_message)
 
     async def get_ffmpeg_path(self) -> str:
         return await self.__on_request("getFFmpegPath")
@@ -141,5 +141,5 @@ class CoreManager(AsyncIOEventEmitter):
             if self.publisher:
                 await self.publisher.close()
 
-            if self.dm_publisher:
-                await self.dm_publisher.close()
+            if self.cm_publisher:
+                await self.cm_publisher.close()