@camera.ui/server 2.2.5-beta.4 → 2.2.6

package/dist/server/src/_camera_ui_tools/camera_ui_sdk/plugin/assistant_models.py

@@ -0,0 +1,300 @@
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from typing import Any, Literal, NotRequired, Protocol, TypedDict, runtime_checkable
+
+
+class AssistantModelSpec(TypedDict):
+    """One model a plugin offers to the assistant. The user picks it under
+    Settings, Assistant like any other model; the host never sees a key or an
+    endpoint of it.
+
+    Example:
+        ```python
+        on_device: AssistantModelSpec = {
+            "id": "apple-on-device",
+            "name": "Apple on-device",
+            "contextTokens": 8192,
+            "vision": True,
+            "toolCalling": True,
+            "structuredOutput": True,
+            "note": "Runs on this Mac, nothing leaves the house.",
+        }
+        ```
+    """
+
+    id: str
+    """Model id, unique within the plugin and stable across restarts: the settings store it."""
+
+    name: str
+    """Name shown in the model list."""
+
+    contextTokens: int
+    """Hard context limit of the model in tokens. The host plans prompt, tools and history with it."""
+
+    vision: bool
+    """The model reads images. Without it the host sends text only."""
+
+    toolCalling: bool
+    """The model calls tools. Without it the assistant answers from the conversation alone."""
+
+    structuredOutput: bool
+    """The model honors a JSON schema. Without it the host asks for free text and parses defensively."""
+
+    toolRouting: NotRequired[bool]
+    """The host picks the tools for a question with a short call of its own and puts only
+    those in front of the model. For a small model that calls the tools it sees but
+    does not work through a catalog of fifty."""
+
+    note: NotRequired[str]
+    """One line shown next to the model, for example where it runs or what it costs."""
+
+
+class AssistantModelText(TypedDict):
+    """Text part of a message."""
+
+    type: Literal["text"]
+    """Always ``text``."""
+
+    text: str
+    """The text itself."""
+
+
+class AssistantModelImage(TypedDict):
+    """Image part of a message. Arrives only for a model whose spec says
+    ``vision``, and only when the instance allows pictures to reach the model."""
+
+    type: Literal["image"]
+    """Always ``image``."""
+
+    data: str
+    """Base64-encoded image bytes."""
+
+    mimeType: Literal["image/jpeg", "image/png", "image/webp"]
+    """MIME type of ``data``."""
+
+
+AssistantModelContent = AssistantModelText | AssistantModelImage
+"""Part of a message: text or an image."""
+
+
+class AssistantModelToolCall(TypedDict):
+    """A tool call: which tool with which arguments."""
+
+    id: str
+    """Id the host uses to match the result, unique within the request."""
+
+    name: str
+    """Name of the tool as declared in ``AssistantModelRequest["tools"]``."""
+
+    arguments: dict[str, Any]
+    """Arguments as parsed JSON, matching the tool's input schema."""
+
+
+class AssistantModelMessage(TypedDict):
+    """One message of the conversation. ``tool`` messages carry the result of
+    the call with the matching ``toolCallId``."""
+
+    role: Literal["user", "assistant", "tool"]
+    """Who wrote it."""
+
+    content: list[AssistantModelContent]
+    """Text and images of the message."""
+
+    toolCalls: NotRequired[list[AssistantModelToolCall]]
+    """Tool calls the assistant asked for, on ``assistant`` messages."""
+
+    toolCallId: NotRequired[str]
+    """Call this message answers, on ``tool`` messages."""
+
+
+class AssistantModelTool(TypedDict):
+    """A tool the model may call in this request."""
+
+    name: str
+    """Name the model uses in its call."""
+
+    description: str
+    """What the tool does, written for the model."""
+
+    inputSchema: dict[str, Any]
+    """JSON Schema object of the arguments."""
+
+
+class AssistantModelRequest(TypedDict):
+    """One request to a plugin model: the conversation so far plus what the
+    model may do with it."""
+
+    modelId: str
+    """Model id from ``AssistantModelSpec["id"]``."""
+
+    system: list[str]
+    """Instructions that precede the conversation."""
+
+    messages: list[AssistantModelMessage]
+    """The conversation, oldest first. The host has already trimmed it to ``contextTokens``."""
+
+    tools: list[AssistantModelTool]
+    """Tools the model may call. Empty when the run allows none."""
+
+    outputSchema: NotRequired[dict[str, Any]]
+    """JSON Schema the answer has to match. Only set for models whose spec says ``structuredOutput``."""
+
+    maxOutputTokens: NotRequired[int]
+    """Upper bound for the answer in tokens, when the host has one."""
+
+
+class AssistantModelTextChunk(TypedDict):
+    """A piece of the answer text."""
+
+    type: Literal["text"]
+    """Always ``text``."""
+
+    delta: str
+    """The text generated since the last chunk."""
+
+
+class AssistantModelToolCallChunk(TypedDict):
+    """A tool call the model asks for, with complete arguments."""
+
+    type: Literal["tool_call"]
+    """Always ``tool_call``."""
+
+    call: AssistantModelToolCall
+    """The call itself."""
+
+
+class AssistantModelUsageChunk(TypedDict):
+    """What the answer consumed."""
+
+    type: Literal["usage"]
+    """Always ``usage``."""
+
+    promptTokens: int
+    """Tokens the request consumed."""
+
+    completionTokens: int
+    """Tokens the answer consumed."""
+
+
+class AssistantModelDoneChunk(TypedDict):
+    """End of the answer, yielded exactly once."""
+
+    type: Literal["done"]
+    """Always ``done``."""
+
+    finish: Literal["stop", "tool_calls", "length", "error"]
+    """Why the model stopped."""
+
+    message: NotRequired[str]
+    """What went wrong, with ``error``."""
+
+
+AssistantModelChunk = (
+    AssistantModelTextChunk | AssistantModelToolCallChunk | AssistantModelUsageChunk | AssistantModelDoneChunk
+)
+"""A piece of the answer: text, a tool call, token counts, or the end."""
+
+
+class AssistantModelStatus(TypedDict):
+    """Whether the plugin can answer right now. A model that has to be
+    downloaded or loaded first reports ``ready: False`` with a line the settings
+    page shows, and a ``progress`` while it knows one.
+
+    Example:
+        ```python
+        {"ready": False, "message": "macOS is downloading the model", "progress": 0.62}
+        ```
+    """
+
+    ready: bool
+    """The models of this plugin can answer right now."""
+
+    message: NotRequired[str]
+    """One line for the user: what the plugin is waiting for, or what it is doing."""
+
+    progress: NotRequired[float]
+    """How far the loading got, between 0 and 1."""
+
+
+class AssistantModelContext(TypedDict):
+    """Who the answer is for and when to stop writing it."""
+
+    userId: NotRequired[str]
+    """ID of the user the answer is for. Absent for scheduled runs and plugin requests."""
+
+    language: str
+    """BCP 47 language tag the answer should use."""
+
+    timeoutMs: int
+    """Milliseconds the host waits before it drops the answer."""
+
+
+@runtime_checkable
+class AssistantModelProvider(Protocol):
+    """Implemented by plugins that list ``PluginInterface.AssistantModels`` in
+    their contract. Such a plugin is a bridge to a model the host cannot reach
+    itself: one without an HTTP API, one behind an unusual authentication, or
+    one that lives on the machine the plugin runs on. The plugin holds no
+    conversation state, every request carries the whole conversation.
+
+    Example:
+        ```python
+        class ApplePlugin(BasePlugin):
+            async def assistantModels(self) -> list[AssistantModelSpec]:
+                return [
+                    {
+                        "id": "apple-on-device",
+                        "name": "Apple on-device",
+                        "contextTokens": 8192,
+                        "vision": True,
+                        "toolCalling": True,
+                        "structuredOutput": True,
+                    }
+                ]
+
+            async def assistantGenerate(
+                self, request: AssistantModelRequest, ctx: AssistantModelContext
+            ) -> AsyncIterator[AssistantModelChunk]:
+                async for delta in self.session.stream(request, ctx["timeoutMs"]):
+                    yield {"type": "text", "delta": delta}
+                yield {"type": "done", "finish": "stop"}
+        ```
+    """
+
+    async def assistantModels(self) -> list[AssistantModelSpec]:
+        """List the models this plugin offers. Called after the plugin is ready
+        and whenever the settings page is opened.
+
+        Returns:
+            The specs, empty while the plugin has no usable model (no hardware,
+            no permission yet).
+        """
+        ...
+
+    async def assistantModelStatus(self) -> AssistantModelStatus:
+        """Report whether the models can answer right now. The host asks before
+        it offers them and while the settings page is open, so a model that is
+        still downloading shows up as busy instead of missing. Leave it out when
+        the models are usable as soon as the plugin runs.
+
+        Returns:
+            Readiness, with a line and a progress while it loads.
+        """
+        ...
+
+    def assistantGenerate(
+        self, request: AssistantModelRequest, ctx: AssistantModelContext
+    ) -> AsyncIterator[AssistantModelChunk]:
+        """Answer one request. The host reads chunks until ``done``, then either
+        returns the text to the user or runs the tool calls and asks again with
+        their results in ``messages``.
+
+        Args:
+            request: Conversation, tools and schema for this answer.
+            ctx: Who asks, in which language, and how long the host waits.
+
+        Returns:
+            The answer in pieces.
+        """
+        ...

package/dist/server/src/_camera_ui_tools/camera_ui_sdk/plugin/contract.py

@@ -35,6 +35,9 @@ class PluginRole(StrEnum):
     CameraAndSensorProvider = "cameraAndSensorProvider"
     """Manages cameras and exposes sensors, on its own cameras and, with ``consumes`` set, on foreign ones."""
 
+    Service = "service"
+    """Serves camera.ui itself, for example with a language model. Touches no cameras and no sensors, and is never assigned to one."""
+
 
 class PluginInterface(StrEnum):
     """Capability flags a plugin advertises in its contract.
@@ -64,6 +67,9 @@ class PluginInterface(StrEnum):
     ClipDetection = "ClipDetection"
     """Implements ClipDetectionInterface (CLIP image and text embeddings used for semantic search)."""
 
+    FaceEmbedding = "FaceEmbedding"
+    """Implements FaceEmbeddingInterface (turns a face crop into a vector). Matching against enrolled faces happens in the NVR."""
+
     DiscoveryProvider = "DiscoveryProvider"
     """Implements DiscoveryProvider (network scan + adoption). Only valid for camera-controlling roles."""
 
@@ -91,6 +97,9 @@ class PluginInterface(StrEnum):
     AssistantTools = "AssistantTools"
     """Implements AssistantToolProvider, so the assistant can offer the plugin's tools to the model."""
 
+    AssistantModels = "AssistantModels"
+    """Implements AssistantModelProvider, so its models appear in the assistant settings."""
+
 
 class PluginCapability(StrEnum):
     """Permission a plugin requests so it can call a host-provided system

package/dist/server/src/_camera_ui_tools/camera_ui_sdk/plugin/helper.py

@@ -134,6 +134,8 @@ def validate_contract_consistency(contract: PluginContract, plugin_name: str | N
         raise ValueError(f"{prefix}SensorProvider plugins must provide at least one sensor type.")
     if role == PluginRole.CameraAndSensorProvider and len(provides) == 0:
         raise ValueError(f"{prefix}CameraAndSensorProvider plugins must provide at least one sensor type.")
+    if role == PluginRole.Service and (len(provides) > 0 or len(contract["consumes"]) > 0):
+        raise ValueError(f"{prefix}Service plugins cannot provide or consume sensors.")
 
 
 def is_hub(contract: PluginContract) -> bool:
@@ -155,6 +157,25 @@ def is_hub(contract: PluginContract) -> bool:
     return contract["role"] == PluginRole.Hub
 
 
+def is_service(contract: PluginContract) -> bool:
+    """Report whether the plugin only serves camera.ui itself (role Service),
+    which means it never appears in a camera's plugin list.
+
+    Args:
+        contract: Plugin contract to inspect.
+
+    Returns:
+        True if the role is :attr:`PluginRole.Service`.
+
+    Example:
+        ```python
+        if is_service(contract):
+            skip_camera_assignment()
+        ```
+    """
+    return contract["role"] == PluginRole.Service
+
+
 def can_create_cameras(contract: PluginContract) -> bool:
     """Report whether the plugin can create cameras (role is CameraController
     or CameraAndSensorProvider). Used to gate camera-creating operations such

package/dist/server/src/_camera_ui_tools/camera_ui_sdk/plugin/interfaces.py

@@ -24,6 +24,7 @@ from .assistant import AssistantToolProvider
 
 if TYPE_CHECKING:
     from ..camera import CameraConfig, CameraDevice
+    from ..camera.enums import Point
     from ..manager import AdoptedSensor, DiscoveredCamera, DiscoveredSensor
     from ..sensor.base import Sensor, SensorLike
     from ..storage import DeviceStorage, JsonSchemaWithoutCallbacks
@@ -96,10 +97,7 @@ class FaceDetectionPluginResponse(TypedDict):
     """True when the run produced at least one detection."""
 
     detections: list[FaceDetection]
-    """Detected faces, each with its embedding."""
-
-    embeddingModel: NotRequired[str]
-    """Model that produced the embeddings; consumers must not mix models."""
+    """Located faces. Vectors come from a face-embedding plugin, not from here."""
 
 
 class LicensePlateDetectionPluginResponse(TypedDict):
@@ -135,6 +133,22 @@ class ClipDetectionPluginResponse(TypedDict):
     """[floor, ceiling] of raw text-image cosine scores for this model; consumers map scores to a 0..1 relevance scale and treat a missing band as score 0."""
 
 
+class FaceEmbeddingPluginResponse(TypedDict):
+    """Result of a face embedding run on a single image."""
+
+    embedding: list[float]
+    """Embedding vector for the face, empty when no face could be embedded."""
+
+    embeddingModel: str
+    """Model that produced the embedding; consumers must not mix models."""
+
+    landmarks: NotRequired[list[Point]]
+    """The five points the face was aligned on, in 0 - 1 of the input image: right eye, left eye, nose, right and left mouth corner."""
+
+    quality: NotRequired[float]
+    """How sure the model is that those points sit on a face (0 - 1)."""
+
+
 class ClipTextEmbeddingResult(TypedDict):
     """Result of a CLIP text embedding request."""
 
@@ -254,6 +268,38 @@ class BasePlugin(ABC, Generic[StorageT]):
         return None
 
 
+class ServicePlugin(BasePlugin[StorageT]):
+    """Base class for a plugin with the role :attr:`PluginRole.Service`: it
+    serves camera.ui itself and never touches cameras, so the camera lifecycle
+    hooks are already implemented as no-ops.
+
+    Example:
+        ```python
+        class MyModels(ServicePlugin):
+            async def assistantModels(self) -> list[AssistantModelSpec]:
+                return [
+                    {
+                        "id": "local",
+                        "name": "Local model",
+                        "contextTokens": 8192,
+                        "vision": False,
+                        "toolCalling": False,
+                        "structuredOutput": True,
+                    }
+                ]
+        ```
+    """
+
+    async def configureCameras(self, cameras: list[CameraDevice]) -> None:
+        """Does nothing, a service plugin owns no cameras."""
+
+    async def onCameraAdded(self, camera: CameraDevice) -> None:
+        """Does nothing, a service plugin owns no cameras."""
+
+    async def onCameraReleased(self, cameraId: str) -> None:
+        """Does nothing, a service plugin owns no cameras."""
+
+
 @runtime_checkable
 class DiscoveryProvider(Protocol):
     """Implemented by plugins that can scan the network for new cameras and
@@ -499,6 +545,36 @@ class ClassifierDetectionInterface(Protocol):
         ...
 
 
+@runtime_checkable
+class FaceEmbeddingInterface(Protocol):
+    """Implemented by plugins that turn a face crop into an embedding vector.
+
+    Split from face detection so the two can run on different hosts: locating a
+    face is cheap, embedding it is not.
+    """
+
+    async def embedFaceImages(
+        self,
+        images: list[bytes],
+        config: dict[str, Any] | None = None,
+        landmarks: list[list[Point] | None] | None = None,
+    ) -> list[FaceEmbeddingPluginResponse | None]:
+        """Embed a batch of encoded images (JPEG/PNG), each showing one face.
+
+        One result per input in the same order. An empty ``embedding`` means the
+        picture holds no face this model can use, None that the plugin could not run
+        at all: the caller may drop such a picture, so the two must not be mixed up.
+        Meant for re-embedding stored pictures after an embedding-model change.
+        ``landmarks`` holds the points an earlier result returned for the same picture,
+        one entry per image: with them the face is not searched again.
+        """
+        ...
+
+    async def faceEmbeddingSettings(self) -> list[JsonSchema] | None:
+        """Return the JSON schema for the face-embedding settings form in the UI, or None for no schema."""
+        ...
+
+
 @runtime_checkable
 class ClipDetectionInterface(Protocol):
     """Implemented by plugins that generate CLIP image and text embeddings

package/dist/server/src/_camera_ui_tools/camera_ui_sdk/sensor/__init__.py

@@ -65,6 +65,10 @@ from .face import (
     FaceResult,
     FaceSensor,
 )
+from .face_embedder import (
+    FaceEmbedderSensor,
+    FaceEmbeddingResult,
+)
 from .garage import (
     GarageControl,
     GarageState,
@@ -204,6 +208,7 @@ __all__ = [
     "TrackedDetection",
     "ClipEmbedding",
     "ClipResult",
+    "FaceEmbeddingResult",
     # Public sensor classes
     "MotionSensor",
     "MotionDetectorSensor",
@@ -213,6 +218,7 @@ __all__ = [
     "AudioDetectorSensor",
     "FaceSensor",
     "FaceDetectorSensor",
+    "FaceEmbedderSensor",
     "LicensePlateSensor",
     "LicensePlateDetectorSensor",
     "ClassifierSensor",

package/dist/server/src/_camera_ui_tools/camera_ui_sdk/sensor/base.py

@@ -52,6 +52,8 @@ class SensorType(StrEnum):
     """Audio event detection (glass break, scream, etc.)."""
     Face = "face"
     """Face detection and recognition."""
+    FaceEmbedder = "faceEmbedder"
+    """Face embedding generation from a face crop, for recognition against enrolled faces."""
     LicensePlate = "licensePlate"
     """License plate detection and OCR."""
     Classifier = "classifier"

package/dist/server/src/_camera_ui_tools/camera_ui_sdk/sensor/face.py

@@ -19,7 +19,7 @@ class FaceProperty(StrEnum):
     Detected = "detected"
     """Whether any face is currently detected."""
     Detections = "detections"
-    """List of detected faces with optional identity, embedding, and thumbnail."""
+    """List of detected faces, with an identity where one was matched."""
     Identities = "identities"
     """Names of the faces recognized during the active detection phase."""
 
@@ -31,10 +31,6 @@ class FaceDetection(Detection):
     """Sub-detection attribute, fixed to "face"."""
     identity: NotRequired[str]
     """Recognized identity name, if matched against known faces."""
-    embedding: NotRequired[list[float]]
-    """Face embedding vector for recognition/comparison."""
-    thumbnail: NotRequired[bytes]
-    """JPEG thumbnail crop of the detected face."""
 
 
 class FaceSensorProperties(TypedDict):
@@ -123,8 +119,8 @@ class FaceSensor(Sensor[FaceSensorProperties, TStorage, str], Generic[TStorage])
         - ``reportDetections(True)``: face detected without specifics (e.g. a
           bare face-event from a discovery provider). The SDK synthesizes a
           single full-frame face detection without identity.
-        - ``reportDetections(True, [...])``: explicit face detections with
-          identity, embedding, and/or thumbnail.
+        - ``reportDetections(True, [...])``: explicit face detections, with an
+          identity where the plugin recognizes the face itself.
         - ``reportDetections(False)``: clear.
 
         Args:

package/dist/server/src/_camera_ui_tools/camera_ui_sdk/sensor/face_embedder.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+from abc import abstractmethod
+from collections.abc import Mapping
+from typing import Any, Generic, NotRequired
+
+from typing_extensions import TypedDict, TypeVar
+
+from ..camera.enums import Point
+from .base import Sensor, SensorCategory, SensorType
+from .detection import VideoFrameData
+from .spec import ModelSpec
+
+
+class FaceEmbeddingResult(TypedDict):
+    """Return type for FaceEmbedderSensor.embedFaces()."""
+
+    embedding: list[float]
+    """Embedding vector for the face in this crop, empty when no face could be embedded."""
+    embeddingModel: str
+    """Identifier of the embedding model that produced the vector."""
+    landmarks: NotRequired[list[Point]]
+    """The five points the face was aligned on, in 0 - 1 of the input image: right eye, left eye, nose, right and left mouth corner."""
+    quality: NotRequired[float]
+    """How sure the model is that those points sit on a face (0 - 1)."""
+
+
+TStorage = TypeVar("TStorage", bound=Mapping[str, Any], default=dict[str, Any])
+
+
+class FaceEmbedderSensor(Sensor[dict[str, Any], TStorage, str], Generic[TStorage]):
+    """Face embedder that turns a face crop into a vector for recognition.
+
+    Extend this class and implement ``embedFaces``.
+
+    The crop comes from the face detector's box, cut from the source frame, so
+    the sensor runs after face detection rather than on the raw scene. What a
+    vector means depends on the model that produced it: vectors from different
+    models are not comparable, which is why every result carries its
+    ``embeddingModel``.
+    """
+
+    _requires_frames = True
+
+    def __init__(self, name: str = "Face Embedder", *, native_id: str | None = None) -> None:
+        super().__init__(name, native_id=native_id)
+
+    @property
+    def type(self) -> SensorType:
+        return SensorType.FaceEmbedder
+
+    @property
+    def category(self) -> SensorCategory:
+        return SensorCategory.Sensor
+
+    @property
+    @abstractmethod
+    def modelSpec(self) -> ModelSpec: ...
+
+    @abstractmethod
+    async def embedFaces(self, frames: list[VideoFrameData]) -> list[FaceEmbeddingResult]:
+        """Embed faces in batch. Each frame is one face crop, padded around the detected
+        box and pre-scaled to ``modelSpec['input']``. Must return exactly one
+        FaceEmbeddingResult per input frame, in the same order; return an empty
+        ``embedding`` for a crop the model could not use."""
+        ...
+
+    async def updateValue(self, property: str, value: Any) -> None:
+        """Read-only sensor: external writes are ignored."""

package/dist/server/src/api/controllers/assistant.controller.js

@@ -11,7 +11,7 @@ export class AssistantController {
     }
     async getInfo(req, reply) {
         try {
-            return reply.code(200).send(this.service.info(req.locals.user?.role));
+            return reply.code(200).send(await this.service.info(req.locals.user?.role));
         }
         catch (error) {
             return reply.code(500).send({ statusCode: 500, message: error.message });