@camera.ui/server 2.0.4 → 2.0.5

package/dist/server/src/plugins/runtime/python/storage.py

@@ -1,26 +1,16 @@
+from __future__ import annotations
+
 import asyncio
 import inspect
-from collections.abc import Awaitable, Callable
-from contextlib import suppress
 from copy import deepcopy
-from typing import Any, Literal, cast
+from typing import TYPE_CHECKING, Any, ClassVar, cast
 
 from deepdiff.diff import DeepDiff
-from plugins.runtime.python.config_db import PluginConfigDb
 
 from _camera_ui_tools.camera_ui_common import ObjectPath, merge, merge_with
-from _camera_ui_tools.camera_ui_rpc import CloseHandler, RPCClient, RPCMethod
-from _camera_ui_tools.camera_ui_sdk import (
-    API_EVENT,
-    FormSubmitResponse,
-    JsonSchema,
-    PluginAPI,
-    SchemaConfig,
-)
+from _camera_ui_tools.camera_ui_rpc import RPCMethod
 from _camera_ui_tools.camera_ui_sdk import DeviceStorage as DeviceStorageInterface
-from plugins.runtime.python.namespaces import (
-    NamespaceManager,
-)
+from plugins.runtime.python.namespaces import NamespaceManager
 from plugins.runtime.python.schema import (
     generate_config_from_schemas,
     get_value_by_key,
@@ -28,13 +18,28 @@ from plugins.runtime.python.schema import (
     is_submit_type,
     remove_callbacks_from_schemas,
 )
-from plugins.runtime.python.typings import PluginInfo
+from plugins.runtime.python.store import (
+    PluginLocation,
+    SensorLocation,
+    delete_location,
+    read_location,
+    validate_store_value,
+    write_location,
+)
 
-ConfigDict = dict[str, Any]
+if TYPE_CHECKING:
+    from _camera_ui_tools.camera_ui_rpc import CloseHandler, RPCClient
+    from _camera_ui_tools.camera_ui_sdk import (
+        FormSubmitResponse,
+        JsonSchema,
+        PluginAPI,
+        SchemaConfig,
+    )
+    from plugins.runtime.python.config_db import PluginConfigDb
+    from plugins.runtime.python.store import StoreLocation
+    from plugins.runtime.python.typings import PluginInfo
 
-OnSetCallback = Callable[[Any, Any], Awaitable[None] | None]
-OnGetCallback = Callable[[], Awaitable[Any] | Any]
-OnClickCallback = Callable[[Any], Awaitable[FormSubmitResponse | None] | FormSubmitResponse | None]
+ConfigDict = dict[str, Any]
 
 
 def merge2(
@@ -53,17 +58,19 @@ def merge2(
 
 
 class DeviceStorage(DeviceStorageInterface):
+    # Strong refs to detached onSet tasks: the event loop only holds weak
+    # references, so an unreferenced task could be garbage-collected mid-flight.
+    __detached_tasks: ClassVar[set[asyncio.Task[None]]] = set()
+
     def __init__(
         self,
         api: PluginAPI,
         proxy: RPCClient,
         plugin: PluginInfo,
         plugin_db: PluginConfigDb,
-        device_id: str,
+        location: StoreLocation,
         schemas: list[JsonSchema] | None = [],
-        is_plugin_storage: bool = False,
         sensor_id: str | None = None,
-        camera_id: str | None = None,
     ) -> None:
         if schemas is None:
             schemas = []
@@ -75,32 +82,31 @@ class DeviceStorage(DeviceStorageInterface):
         self.__proxy = proxy
         self.__plugin = plugin
         self.__plugin_db = plugin_db
-        self.__device_id = device_id
-        self.__is_plugin_storage = is_plugin_storage
+        self.__location = location
         self.__sensor_id = sensor_id
-        self.__camera_id = camera_id
         self.__close_proxy: CloseHandler | None = None
+        self.__dirty = False
 
-        if self.__sensor_id is not None and self.__camera_id is not None:
+        if isinstance(location, SensorLocation):
+            if not self.__sensor_id:
+                raise ValueError(f"sensor storage for {plugin['id']} requires a sensor_id")
             sensor_ns = NamespaceManager.plugin_sensor_namespaces(
-                plugin["id"], self.__camera_id, self.__sensor_id
+                plugin["id"], location.camera_id, self.__sensor_id
             )
             self.__storage_namespace = sensor_ns.sensor_storage_rpc
-        elif self.__is_plugin_storage:
+        elif isinstance(location, PluginLocation):
             plugin_ns = NamespaceManager.plugin_namespaces(plugin["id"])
             self.__storage_namespace = plugin_ns.plugin_storage_rpc
         else:
-            camera_ns = NamespaceManager.plugin_camera_namespaces(plugin["id"], self.__device_id)
+            camera_ns = NamespaceManager.plugin_camera_namespaces(plugin["id"], location.camera_id)
             self.__storage_namespace = camera_ns.camera_storage_rpc
 
-        self.__api.once(API_EVENT.SHUTDOWN, self.__close)
-
     @RPCMethod
     async def getValue(self, key: str, default_value: Any | None = None) -> Any:
         schema: Any = next((schema for schema in self.schemas if schema["key"] == key), None)
         config_value = ObjectPath.get(self.values, key)
         schema_default_value = (
-            schema["defaultValue"] if "defaultValue" in schema and schema["defaultValue"] else None
+            schema["defaultValue"] if schema and "defaultValue" in schema and schema["defaultValue"] else None
         )
 
         if schema:
@@ -132,23 +138,24 @@ class DeviceStorage(DeviceStorageInterface):
         schema: Any = next((schema for schema in self.schemas if schema["key"] == key), None)
 
         if schema:
+            validate_store_value(key, new_value)
             old_value = ObjectPath.get(self.values, key)
-            ObjectPath.set(self.values, key, new_value)
-
-            on_set_function = schema.get("onSet", None)
-            if on_set_function:
-                if inspect.iscoroutinefunction(on_set_function):
-                    await on_set_function(new_value, old_value)
-                else:
-                    res = on_set_function(new_value, old_value)
-
-                    if inspect.iscoroutine(res):
-                        await res
-                    else:
-                        on_set_function(new_value, old_value)
-
-            if self.__contains_storable_schema(schema):
-                self.save()
+            if new_value is None:
+                unchanged = old_value is None
+                ObjectPath.delete(self.values, key)
+            else:
+                unchanged = old_value is not None and DeepDiff(old_value, new_value) == {}
+                stored = (  # pyright: ignore[reportUnknownVariableType]
+                    deepcopy(new_value)  # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType]
+                    if isinstance(new_value, (dict, list))
+                    else new_value
+                )
+                ObjectPath.set(self.values, key, stored)
+
+            if self.__contains_storable_schema(schema) and (not unchanged or self.__dirty):
+                await self.save()
+
+            self.__run_on_set_detached(schema.get("onSet", None), key, new_value, old_value)
 
     @RPCMethod
     async def submitValue(self, key: str, new_value: Any) -> FormSubmitResponse | None:
@@ -170,9 +177,20 @@ class DeviceStorage(DeviceStorageInterface):
         return None
 
     @RPCMethod
-    def setInternalValue(self, key: str, value: Any) -> None:
-        ObjectPath.set(self.values, key, value)
-        self.save()
+    async def setInternalValue(self, key: str, value: Any) -> None:  # type: ignore[override]
+        validate_store_value(key, value)
+        old_value = ObjectPath.get(self.values, key)
+        if value is None:
+            unchanged = old_value is None
+            ObjectPath.delete(self.values, key)
+        else:
+            unchanged = old_value is not None and DeepDiff(old_value, value) == {}
+            stored = deepcopy(value) if isinstance(value, (dict, list)) else value  # type: ignore
+            ObjectPath.set(self.values, key, stored)
+
+        if unchanged and not self.__dirty:
+            return
+        await self.save()
 
     @RPCMethod
     def hasValue(self, key: str) -> bool:
@@ -188,52 +206,67 @@ class DeviceStorage(DeviceStorageInterface):
 
     @RPCMethod
     async def setConfig(self, new_config: dict[str, Any]) -> None:
+        validate_store_value("config", new_config)
         old_config = deepcopy(self.values)
         self.values = merge_with(self.values, new_config, merge)
 
-        await self.__trigger_on_set_for_changes(old_config, self.values)
-        self.save()
+        await self.save()
+        self.__trigger_on_set_for_changes(old_config, self.values)
 
     @RPCMethod
     async def addSchema(self, schema: JsonSchema) -> None:
-        should_save = False
-
-        schema_exist = next((schema for schema in self.schemas if schema["key"] == schema["key"]), None)
-
-        if schema_exist:
+        if self.hasSchema(schema["key"]):
             raise Exception(f"Schema with key {schema['key']} already exists")
-        else:
-            self.schemas.append(schema)
 
-        await self.__resolve_on_get_functions(schema)
+        self.schemas.append(schema)
 
-        if self.__contains_storable_schema(schema):
-            should_save = True
+        old_value = ObjectPath.get(self.values, schema["key"])
+        await self.__resolve_on_get_functions(schema)
 
-        if should_save:
-            self.save()
+        new_value = ObjectPath.get(self.values, schema["key"])
+        if (
+            self.__contains_storable_schema(schema)
+            and DeepDiff(old_value, new_value, ignore_order=True) != {}
+        ):
+            await self.save()
 
     @RPCMethod
-    def removeSchema(self, key: str) -> None:
+    async def removeSchema(self, key: str) -> None:  # type: ignore[override]
         schema = next((schema for schema in self.schemas if schema["key"] == key), None)
+        had_value = ObjectPath.get(self.values, key) is not None
         self.schemas = [s for s in self.schemas if s["key"] != key]
         ObjectPath.delete(self.values, key)
 
-        if schema and self.__contains_storable_schema(schema):
-            self.save()
+        if schema and self.__contains_storable_schema(schema) and had_value:
+            await self.save()
 
     @RPCMethod
     async def changeSchema(self, key: str, new_schema: dict[str, Any]) -> None:
         new_schema["key"] = key
-        schema = next((schema for schema in self.schemas if schema["key"] == new_schema["key"]), None)
+        index = next(
+            (i for i, schema in enumerate(self.schemas) if schema["key"] == key),
+            None,
+        )
+        if index is None:
+            return
 
-        if schema:
-            merge_with(schema, new_schema, merge)
+        was_storable = self.__contains_storable_schema(self.schemas[index])
+        self.schemas[index] = cast(JsonSchema, new_schema)
 
-            await self.__resolve_on_get_functions(schema)
+        # Storable-ness flipped: the next write must persist even if the
+        # value compares unchanged, otherwise the flip never becomes durable.
+        if self.__contains_storable_schema(self.schemas[index]) != was_storable:
+            self.__dirty = True
 
-            if self.__contains_storable_schema(schema):
-                self.save()
+        old_value = ObjectPath.get(self.values, key)
+        await self.__resolve_on_get_functions(self.schemas[index])
+
+        new_value = ObjectPath.get(self.values, key)
+        if (
+            self.__contains_storable_schema(self.schemas[index])
+            and DeepDiff(old_value, new_value, ignore_order=True) != {}
+        ):
+            await self.save()
 
     @RPCMethod
     def getSchema(self, key: str) -> JsonSchema | None:
@@ -245,42 +278,50 @@ class DeviceStorage(DeviceStorageInterface):
 
     @RPCMethod
     async def destroy(self) -> None:
-        self.__api.removeListener(API_EVENT.SHUTDOWN, self.__close)
-        config: Any = cast(dict[str, Any], self.__plugin_db.get("config") or {})
-        if self.__device_id in config:
-            del config[self.__device_id]
-            self.values = {}
-            self.__plugin_db.update({"config": config})
+        config = self.__plugin_db.get("config") or {}
+        delete_location(config, self.__location)
+        self.values = {}
+        await self.__plugin_db.put("config", config)
 
-    def save(self) -> None:
-        config: Any = cast(dict[str, Any], self.__plugin_db.get("config") or {})
+    async def save(self) -> None:  # type: ignore[override]
+        config = self.__plugin_db.get("config") or {}
 
         if len(self.schemas) > 0:
             storable_config = self.__filter_storable_values(self.schemas)
         else:
             storable_config = self.values
 
-        config[self.__device_id] = storable_config
-        self.__plugin_db.update({"config": config})
+        write_location(config, self.__location, storable_config)
+        try:
+            await self.__plugin_db.put("config", config)
+        except Exception:
+            self.__dirty = True
+            raise
+        self.__dirty = False
 
     def defineSchemas(self, schemas: list[JsonSchema]) -> None:
         self.schemas = schemas
         self.__initializeStorage()
 
-    # Internal method
     def update_schema(self, schemas: list[JsonSchema] = []) -> None:
         self.schemas = schemas
         self.__initializeStorage()
 
-    # Internal method
     async def register_storage(self) -> None:
         self.__close_proxy = await self.__proxy.register_handler(self.__storage_namespace, self)
 
-    # Internal method
     async def unregister_storage(self) -> None:
         if self.__close_proxy:
             await self.__close_proxy()
 
+    # Internal method to close the storage
+    async def close(self) -> None:
+        try:
+            await self.save()
+            await self.unregister_storage()
+        except Exception as error:
+            cast(Any, self.__api).logger.error(f"store: close save failed: {error}")
+
     async def __resolve_on_get_functions(
         self,
         schemas: JsonSchema | list[JsonSchema],
@@ -294,46 +335,40 @@ class DeviceStorage(DeviceStorageInterface):
                 if schema_value is not None:
                     ObjectPath.set(self.values, schema["key"], schema_value)
 
-    async def __trigger_on_set_for_changes(
-        self, old_config: dict[str, Any], new_config: dict[str, Any]
-    ) -> None:
-        tasks: list[Awaitable[None | Any]] = []
-
+    def __trigger_on_set_for_changes(self, old_config: dict[str, Any], new_config: dict[str, Any]) -> None:
         for config_key in list(new_config.keys()):
             old_value: Any = get_value_by_key(old_config, config_key) or {}
             new_value: Any = new_config[config_key]
 
             if DeepDiff(old_value, new_value, ignore_order=True) != {}:
-                schema = next((schema for schema in self.schemas if schema["key"] == config_key), None)
+                schema = next(
+                    (schema for schema in self.schemas if schema["key"] == config_key),
+                    None,
+                )
                 if schema and schema["type"] != "submit" and "onSet" in schema:
-                    on_set_function: Any = schema.get("onSet", None)
-                    if on_set_function:
-
-                        async def process_on_set(
-                            func: Callable[[Any, Any], Awaitable[None | Any]]
-                            | Callable[[Any, Any], None | Any],
-                            new_val: Any,
-                            old_val: Any,
-                        ) -> None:
-                            if inspect.iscoroutinefunction(func):
-                                await func(new_val, old_val)
-                            else:
-                                res = func(new_val, old_val)
-                                if inspect.iscoroutine(res):
-                                    await res
-
-                        tasks.append(process_on_set(on_set_function, new_value, old_value))
-
-        if tasks:
-            await asyncio.gather(*tasks)
+                    self.__run_on_set_detached(schema.get("onSet", None), config_key, new_value, old_value)
+
+    def __run_on_set_detached(self, on_set_function: Any, key: str, new_value: Any, old_value: Any) -> None:
+        if not on_set_function:
+            return
+
+        async def _run() -> None:
+            try:
+                res = on_set_function(new_value, old_value)
+                if inspect.iscoroutine(res):
+                    await res
+            except Exception as error:
+                cast(Any, self.__api).logger.error(f'onSet handler for "{key}" failed: {error}')
+
+        task = asyncio.create_task(_run())
+        self.__detached_tasks.add(task)
+        task.add_done_callback(self.__detached_tasks.discard)
 
     def __filter_storable_values(
         self,
         schemas: list[JsonSchema],
-        result: ConfigDict | None = None,
     ) -> ConfigDict:
-        if result is None:
-            result = {}
+        result: ConfigDict = {}
 
         for schema in schemas:
             if is_button_type(schema) or is_submit_type(schema):
@@ -343,9 +378,10 @@ class DeviceStorage(DeviceStorageInterface):
                 config_value = get_value_by_key(self.values, schema["key"])
                 ObjectPath.set(result, schema["key"], config_value)
 
-        # Preserve internal values (prefixed with '_') that have no schema
+        # Keys without any schema are internal values and always persist;
+        # schema-covered keys persist only via their store flag above.
         for key, value in self.values.items():
-            if key.startswith("_"):
+            if not any(schema["key"] == key or schema["key"].startswith(f"{key}.") for schema in schemas):
                 result[key] = value
 
         return result
@@ -360,12 +396,7 @@ class DeviceStorage(DeviceStorageInterface):
         return bool("store" in schema and schema["store"])
 
     def __initializeStorage(self) -> None:
-        config: Any = cast(dict[str, Any], self.__plugin_db.get("config") or {})
-        device_config = config.get(self.__device_id, {})
+        config = self.__plugin_db.get("config") or {}
+        device_config = read_location(config, self.__location) or {}
         schema_config = generate_config_from_schemas(self.schemas)
         self.values = merge_with(schema_config, device_config, merge2)
-
-    async def __close(self) -> None:
-        with suppress(Exception):
-            self.save()
-            await self.unregister_storage()

package/dist/server/src/plugins/runtime/python/storage_controller.py

@@ -1,18 +1,19 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Literal, overload
+from typing import TYPE_CHECKING, Literal, overload
 
-from plugins.runtime.python.config_db import PluginConfigDb
-
-from _camera_ui_tools.camera_ui_rpc import RPCClient
-from _camera_ui_tools.camera_ui_sdk import JsonSchema, SensorType
 from plugins.runtime.python.storage import DeviceStorage
-from plugins.runtime.python.typings import PluginInfo
+from plugins.runtime.python.store import CameraLocation, PluginLocation, SensorLocation
 
 if TYPE_CHECKING:
+    from _camera_ui_tools.camera_ui_rpc import RPCClient
+    from _camera_ui_tools.camera_ui_sdk import JsonSchema, SensorType
+    from plugins.runtime.python.config_db import PluginConfigDb
     from plugins.runtime.python.plugin_api import PluginAPI
+    from plugins.runtime.python.typings import PluginInfo
 
 
+# Stable storage key for sensor instance storage.
 def sensor_storage_key(camera_id: str, sensor_type: str, plugin_id: str, sensor_name: str) -> str:
     return f"{camera_id}:sensor:{sensor_type}:{plugin_id}:{sensor_name}"
 
@@ -39,7 +40,12 @@ class StorageController:
 
         if not camera_storage:
             camera_storage = DeviceStorage(
-                self.__api, self.__proxy, self.__plugin, self.__plugin_db, cameraId, schemas, False
+                self.__api,
+                self.__proxy,
+                self.__plugin,
+                self.__plugin_db,
+                CameraLocation(camera_id=cameraId),
+                schemas,
             )
             self.__storages[cameraId] = camera_storage
         else:
@@ -55,7 +61,12 @@ class StorageController:
 
         if not plugin_storage:
             plugin_storage = DeviceStorage(
-                self.__api, self.__proxy, self.__plugin, self.__plugin_db, "storage", schemas, True
+                self.__api,
+                self.__proxy,
+                self.__plugin,
+                self.__plugin_db,
+                PluginLocation(),
+                schemas,
             )
             self.__storages["storage"] = plugin_storage
         else:
@@ -82,16 +93,17 @@ class StorageController:
         storage = self.__storages.get(storage_key)
 
         if not storage:
+            # storage_key is only the in-memory registry key; persistence addresses
+            # the canonical sensors.<camId>.<type>.<name> path. sensor_id is the
+            # runtime UUID for the RPC namespace.
             storage = DeviceStorage(
                 self.__api,
                 self.__proxy,
                 self.__plugin,
                 self.__plugin_db,
-                storage_key,
+                SensorLocation(camera_id=camera_id, sensor_type=sensor_type.value, sensor_name=sensor_name),
                 schemas,
-                False,
                 sensor_id,
-                camera_id,
             )
             self.__storages[storage_key] = storage
             storage.update_schema(schemas)
@@ -106,7 +118,6 @@ class StorageController:
         storage_key = sensor_storage_key(camera_id, sensor_type.value, plugin_id, sensor_name)
         return self.__storages.get(storage_key)
 
-    # Internal method
     @overload
     async def createStorage(self, type_: Literal["camera"], device_id: str) -> DeviceStorage: ...
     @overload
@@ -150,7 +161,6 @@ class StorageController:
         await storage.register_storage()
         return storage
 
-    # Internal method
     @overload
     async def removeStorage(self, type_: Literal["camera"], device_id: str) -> None: ...
     @overload
@@ -194,3 +204,8 @@ class StorageController:
             await device_storage.unregister_storage()
             del self.__storages[storage_key]
             return
+
+    # Internal method to close all storages
+    async def close(self) -> None:
+        for storage in self.__storages.values():
+            await storage.close()