@camera.ui/server 0.0.49 → 0.0.50-internal.0

package/dist/server/src/_camera_ui_tools/camera_ui_rpc_client/base/connection.py

@@ -0,0 +1,784 @@
+from __future__ import annotations
+
+import asyncio
+import inspect
+import logging
+import os
+import ssl
+import tempfile
+from collections.abc import Awaitable
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Callable, Literal, TypedDict, TypeVar, Union, cast
+
+import nats
+from nats.aio.client import Client as NatsConnection
+from nats.aio.msg import Msg
+from nats.aio.subscription import Subscription
+from typing_extensions import NotRequired
+
+from ..constants import DEFAULTS, MESSAGE
+from ..typings import (
+    BroadcastMessage,
+    ClientConnectionOptions,
+    ClientMetrics,
+    ConnectOptions,
+    Message,
+    ReconnectOptions,
+    RequestMessage,
+    ResponseMessage,
+    RoutingMetadata,
+    RPCHandler,
+    RPCRequest,
+    RPCResponse,
+    SerializedError,
+    TimeoutOptions,
+)
+from ..utils.logger import setup_logger
+from ..utils.metrics import ConnectionMetricsTracker
+from ..utils.packer import pack, unpack
+from ..utils.routing import EndpointType
+from ..utils.subscriptions import SubscriptionWrapper
+from ..utils.tasks import TaskSet
+from ..utils.uid import uid
+from ..utils.validate import ValidationUtils
+from .errors import ERROR_CODES, ErrorFactory, NatsError
+
+logger: logging.Logger = setup_logger("camera:ui:connection")
+
+T = TypeVar("T")
+R = TypeVar("R")
+
+
+class GroupType:
+    subscription: Subscription
+    handlers: dict[str, Handlers[Any, Any]]
+    type_: Literal["message", "request", "rpc", "generator"]
+    ref_count: int
+
+    def __init__(self, subscription: Subscription):
+        self.subscription = subscription
+        self.handlers = {}
+        self.type_ = "message"
+        self.ref_count: int = 0
+
+
+class MessageContext(TypedDict):
+    subject: str
+    timestamp: int
+    id: str
+    raw: NotRequired[Msg | None]
+
+
+MessageHandler = Callable[[Message[T]], None]
+AsyncMessageHandler = Callable[[Message[T]], Awaitable[None]]
+MessageWithResponseHandler = Callable[[Message[T]], Awaitable[R]]
+MessageHandlers = Union[MessageHandler[T], AsyncMessageHandler[T]]
+Handlers = Union[MessageHandlers[T], MessageWithResponseHandler[T, R], RPCHandler]
+
+
+class RPCConnectionManager:
+    options: ClientConnectionOptions
+
+    def __init__(self, type_: EndpointType, options: ClientConnectionOptions):
+        self.options: ClientConnectionOptions = options
+        self.type: EndpointType = type_
+        self._connection: NatsConnection | None = None
+        self._connection_promise: asyncio.Future[None] | None = None
+        self._subscription_groups: dict[str, GroupType] = {}
+        self._task_set: TaskSet = TaskSet("RPCConnectionManager")
+        self._metrics_tracker: ConnectionMetricsTracker = ConnectionMetricsTracker()
+        self._is_disconnected: bool = False
+
+        ValidationUtils.validate_connection_options(options)
+
+    async def connect(self) -> None:
+        if self.is_connected():
+            return
+
+        if self._connection_promise and not self._connection_promise.done():
+            await self._connection_promise
+            return
+
+        client_id = self.options["metadata"]["id"]
+        logger.debug(f"[{client_id}] Establishing connection...")
+
+        self._connection_promise = asyncio.Future()
+
+        try:
+            await self._establish_connection()
+            self._connection_promise.set_result(None)
+            current_promise = self._connection_promise
+            self._connection_promise = None
+            await current_promise
+
+            logger.debug(f"[{client_id}] Connection established successfully.")
+        except Exception as error:
+            client_id = self.options["metadata"]["id"]
+            logger.debug(f"[{client_id}] Connection failed during initial connect: {str(error)}")
+            if self._connection_promise and not self._connection_promise.done():
+                self._connection_promise.set_exception(error)
+            self._connection_promise = None
+            raise error
+
+    async def disconnect(self) -> None:
+        self._is_disconnected = True
+        client_id = self.options["metadata"]["id"]
+        logger.debug(f"[{client_id}] Disconnecting...")
+        await self._drain_connection()
+        self._connection = None
+        self._connection_promise = None
+        logger.debug(f"[{client_id}] Disconnected.")
+
+    def is_connected(self) -> bool:
+        return (
+            self._connection is not None
+            and not self._connection.is_closed
+            and not self._connection.is_draining
+        )
+
+    def is_closed(self) -> bool:
+        return self._connection is None or self._connection.is_closed or self._connection.is_draining
+
+    def get_connection(self) -> NatsConnection:
+        if not self.is_connected():
+            raise ErrorFactory.connection.invalid_state(
+                None,
+                {
+                    "description": "Connection is not established",
+                    "code": ERROR_CODES.CONNECTION.INVALID_STATE,
+                },
+            )
+        return cast(NatsConnection, self._connection)
+
+    async def publish(
+        self,
+        subject: str,
+        data: Any,
+        error: SerializedError | None = None,
+        routing: RoutingMetadata | None = None,
+    ) -> None:
+        client_id = self.options["metadata"]["id"]
+        if self.is_closed():
+            logger.debug(f"[{client_id}] Publish skipped: Connection closed or draining. Subject: {subject}")
+            raise ErrorFactory.connection.not_connected(
+                None,
+                {
+                    "description": "Failed to publish message: Connection closed or draining",
+                    "code": ERROR_CODES.CONNECTION.NOT_CONNECTED,
+                    "subject": subject,
+                },
+            )
+
+        message: BroadcastMessage[Any] = {
+            "id": uid(),
+            "type": MESSAGE.TYPES.BROADCAST,
+            "data": data,
+            "metadata": self.options["metadata"],
+            "timestamp": int(datetime.now().timestamp() * 1000),
+            "routing": routing,
+            "error": error,
+        }
+
+        try:
+            await self.get_connection().publish(subject, self._pack_message(message))
+            self._metrics_tracker.track_publish()
+            logger.debug(f"Published message to {subject} {message['id']}")
+        except Exception as e:
+            self._metrics_tracker.track_message_error()
+            raise ErrorFactory.message.delivery_failed(
+                e,
+                {
+                    "description": "Failed to publish message",
+                    "code": ERROR_CODES.MESSAGE.DELIVERY_FAILED,
+                    "subject": subject,
+                },
+            ) from e
+
+    async def request(
+        self,
+        subject: str,
+        data: Any,
+        timeout: int | None = None,
+        routing: RoutingMetadata | None = None,
+    ) -> ResponseMessage[Any]:
+        client_id = self.options["metadata"]["id"]
+        if self.is_closed():
+            logger.debug(f"[{client_id}] Request skipped: Connection closed or draining. Subject: {subject}")
+            raise ErrorFactory.connection.not_connected(
+                None,
+                {
+                    "description": "Failed to send request: Connection closed or draining",
+                    "code": ERROR_CODES.CONNECTION.NOT_CONNECTED,
+                    "subject": subject,
+                },
+            )
+
+        if timeout is None:
+            timeout = self.get_timeout_options().get("request", DEFAULTS.TIMEOUTS.REQUEST)
+
+        request_id = uid()
+
+        request_msg: RequestMessage[Any] = {
+            "id": request_id,
+            "type": MESSAGE.TYPES.REQUEST,
+            "subject": subject,
+            "data": data,
+            "metadata": self.options["metadata"],
+            "timestamp": int(datetime.now().timestamp() * 1000),
+            "routing": routing,
+        }
+
+        self._metrics_tracker.track_request_start(request_id)
+
+        try:
+            response = await self.get_connection().request(
+                subject, self._pack_message(request_msg), timeout=timeout / 1000
+            )
+            result: ResponseMessage[Any] = self._unpack_message(response.data)
+
+            self._metrics_tracker.track_request_end(request_id, bool(result.get("error")))
+            if "error" in result and result["error"]:
+                raise NatsError.from_json(result["error"])
+
+            return result
+
+        except Exception as error:
+            self._metrics_tracker.track_request_end(request_id, True)
+            if isinstance(error, NatsError):
+                raise error
+            raise ErrorFactory.connection.request_failed(
+                error,
+                {
+                    "description": "Failed to send request or process response",
+                    "code": "CONNECTION_REQUEST_FAILED",
+                    "subject": subject,
+                },
+            ) from error
+
+    async def request_rpc(self, subject: str, data: Any, timeout: int | None = None) -> RPCResponse[Any]:
+        client_id = self.options["metadata"]["id"]
+        if self.is_closed():
+            logger.debug(
+                f"[{client_id}] RPC Request skipped: Connection closed or draining. Subject: {subject}"
+            )
+            raise ErrorFactory.connection.not_connected(
+                None,
+                {
+                    "description": "Failed to send RPC request: Connection closed or draining",
+                    "code": ERROR_CODES.CONNECTION.NOT_CONNECTED,
+                    "subject": subject,
+                },
+            )
+
+        if timeout is None:
+            timeout = self.get_timeout_options().get("rpc", DEFAULTS.TIMEOUTS.RPC)
+
+        request: RPCRequest[Any] = {
+            "id": uid(),
+            "type": MESSAGE.TYPES.RPC_REQUEST,
+            "subject": subject,
+            "data": data,
+            "metadata": self.options["metadata"],
+            "timestamp": int(datetime.now().timestamp() * 1000),
+        }
+
+        self._metrics_tracker.track_rpc_start()
+
+        try:
+            response = await self.get_connection().request(
+                subject, self._pack_message(request), timeout=timeout / 1000
+            )
+            result: RPCResponse[Any] = self._unpack_message(response.data)
+
+            self._metrics_tracker.track_rpc_end(bool(result.get("error")))
+            if "error" in result and result["error"]:
+                raise NatsError.from_json(result["error"])
+
+            return result
+
+        except Exception as error:
+            self._metrics_tracker.track_rpc_end(True)
+            if isinstance(error, NatsError):
+                raise error
+            raise ErrorFactory.rpc.request_failed(
+                error,
+                {
+                    "description": "Failed to send RPC request or process response",
+                    "code": "RPC_REQUEST_FAILED",
+                    "subject": subject,
+                },
+            ) from error
+
+    async def handle_request(self, subject: str, handler: Handlers[Any, Any]) -> SubscriptionWrapper:
+        return await self.subscribe(subject, handler, None, type_="request")
+
+    async def subscribe(
+        self,
+        subject: str,
+        handler: Handlers[Any, Any],
+        handler_id: str | None = None,
+        type_: Literal["message", "request", "rpc", "generator"] = "message",
+    ) -> SubscriptionWrapper:
+        client_id = self.options["metadata"]["id"]
+        if self.is_closed():
+            logger.debug(
+                f"[{client_id}] Subscribe skipped: Connection closed or draining. Subject: {subject}"
+            )
+            raise ErrorFactory.connection.not_connected(
+                None,
+                {
+                    "description": "Failed to subscribe: Connection closed or draining",
+                    "code": ERROR_CODES.CONNECTION.NOT_CONNECTED,
+                    "subject": subject,
+                },
+            )
+
+        if handler_id is None:
+            handler_id = uid()
+
+        group = self._subscription_groups.get(subject)
+
+        if not group:
+            try:
+                subscription = await self.get_connection().subscribe(subject)
+            except Exception as error:
+                raise ErrorFactory.connection.subscribe_failed(
+                    error,
+                    {
+                        "description": f"Failed to create NATS subscription for subject {subject}",
+                        "code": "CONNECTION_SUBSCRIBE_FAILED",
+                        "subject": subject,
+                        "metadata": {"type": type_},
+                    },
+                ) from error
+
+            group = GroupType(subscription)
+            group.type_ = type_
+            self._subscription_groups[subject] = group
+            self._task_set.add(self._handle_subscription(subscription, subject, group))
+            logger.debug(f"[{client_id}] Created subscription group for {subject}")
+        else:
+            if group.type_ != type_:
+                raise ValueError(f"Cannot mix handlers on subject {subject}")
+
+            if (type_ == "request" or type_ == "rpc" or type_ == "generator") and len(group.handlers) > 0:
+                raise ValueError(f"Request/RPC/Generator handler already registered for subject {subject}")
+
+        if handler_id in group.handlers:
+            logger.debug(f"[{client_id}] Handler {handler_id} already registered for subject {subject}")
+
+        group.handlers[handler_id] = handler
+        group.ref_count += 1
+
+        self._metrics_tracker.track_subscription_change(+1, type_)
+
+        logger.debug(
+            f"[{client_id}] Subscribed to {subject} (handler: {handler_id}, type: {type_}, refCount: {group.ref_count})"
+        )
+
+        return SubscriptionWrapper(subject, handler_id, self.unsubscribe)
+
+    async def unsubscribe(self, subject: str, handler_id: str) -> None:
+        group = self._subscription_groups.get(subject)
+        if not group:
+            return
+
+        if handler_id in group.handlers:
+            del group.handlers[handler_id]
+            group.ref_count -= 1
+            self._metrics_tracker.track_subscription_change(-1, group.type_)
+
+            if group.ref_count <= 0:
+                try:
+                    await group.subscription.unsubscribe()
+                    del self._subscription_groups[subject]
+                except Exception:
+                    del self._subscription_groups[subject]
+
+    def get_metrics(self) -> ClientMetrics:
+        return self._metrics_tracker.get_metrics()
+
+    async def _establish_connection(self) -> None:
+        self._is_disconnected = False
+
+        time_options = self.get_timeout_options()
+        connect_options = self.get_connect_options()
+        reconnect_options = self.get_reconnect_options()
+
+        wait_for_server = connect_options.get("wait_for_server", False)
+        max_attempts = reconnect_options.get("max_attempts", DEFAULTS.RETRY.MAX_RECONNECT_ATTEMPTS)
+        delay = reconnect_options.get("delay", DEFAULTS.RETRY.DELAY)
+        reconnect_enabled = reconnect_options.get("enabled", False)
+
+        attempts = 0
+
+        while not self._is_disconnected:
+            attempts += 1
+            try:
+                client_id = self.options["metadata"]["id"]
+                logger.debug(f"[{client_id}] Connection attempt {attempts}...")
+
+                ssl_ctx = None
+                if "tls" in self.options and self.options["tls"]:
+                    ssl_ctx = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH)
+                    if "ca" in self.options["tls"]:
+                        ssl_ctx.load_verify_locations(cadata=self.options["tls"]["ca"])
+
+                    if "cert" in self.options["tls"] and "key" in self.options["tls"]:
+                        unique_id = uid()
+                        cert_path = Path(tempfile.gettempdir()) / f"cert_{os.getpid()}_{unique_id}.pem"
+                        key_path = Path(tempfile.gettempdir()) / f"key_{os.getpid()}_{unique_id}.pem"
+
+                        try:
+                            cert_path.write_text(self.options["tls"]["cert"], encoding="utf-8")
+                            key_path.write_text(self.options["tls"]["key"], encoding="utf-8")
+
+                            cert_path.chmod(0o600)
+                            key_path.chmod(0o600)
+
+                            ssl_ctx.load_cert_chain(certfile=str(cert_path), keyfile=str(key_path))
+                        finally:
+                            try:
+                                if cert_path.exists():
+                                    cert_path.unlink()
+                                if key_path.exists():
+                                    key_path.unlink()
+                            except Exception:
+                                pass  # Ignore
+
+                async def reconnect_callback() -> None:
+                    client_id = self.options["metadata"]["id"]
+                    logger.debug(f"[{client_id}] Connection reconnected.")
+
+                async def closed_callback(error: Exception | None = None) -> None:
+                    client_id = self.options["metadata"]["id"]
+                    logger.debug(f"[{client_id}] Connection closed: {error if error else ''}")
+                    self._subscription_groups.clear()
+                    self._metrics_tracker.reset_metrics()
+                    self._is_disconnected = True
+                    self._connection = None
+
+                async def ignore_cb(e: Any | None = None) -> None:
+                    pass
+
+                connect_timeout = time_options.get("connection", DEFAULTS.TIMEOUTS.CONNECTION)
+
+                connection = await nats.connect(
+                    name=self.options["metadata"]["id"],
+                    servers=self.options["endpoints"],
+                    **self._get_nats_auth_options(),
+                    allow_reconnect=reconnect_enabled,
+                    connect_timeout=connect_timeout,
+                    max_reconnect_attempts=max_attempts if reconnect_enabled else 0,
+                    reconnect_time_wait=delay,
+                    pending_size=6 * 1024 * 1024,
+                    flush_timeout=5.0,
+                    no_echo=True,
+                    tls=ssl_ctx,
+                    error_cb=ignore_cb,
+                    disconnected_cb=ignore_cb,
+                    closed_cb=closed_callback,
+                    reconnected_cb=reconnect_callback,
+                    discovered_server_cb=ignore_cb,
+                )
+
+                client_id = self.options["metadata"]["id"]
+                logger.debug(f"[{client_id}] NATS connect() successful.")
+
+                self._connection = connection
+                return
+
+            except Exception as error:
+                if self._is_disconnected:
+                    return
+
+                if not wait_for_server:
+                    client_id = self.options["metadata"]["id"]
+                    logger.debug(f"[{client_id}] Connection attempt {attempts} failed: {str(error)}")
+                    raise ErrorFactory.connection.connection_failed(
+                        error,
+                        {
+                            "description": "Failed to connect to NATS server",
+                            "code": ERROR_CODES.CONNECTION.FAILED,
+                            "metadata": {"attempts": attempts},
+                        },
+                    ) from error
+
+                client_id = self.options["metadata"]["id"]
+                logger.debug(f"[{client_id}] Connection attempt {attempts} failed: {str(error)}")
+                await asyncio.sleep(delay / 1000)
+
+    async def _drain_connection(self) -> None:
+        if self._connection and not self._connection.is_closed:
+            try:
+                await self._connection.drain()
+            except Exception:
+                logger.error("Error draining connection", exc_info=True)
+
+    async def _handle_subscription(self, subscription: Subscription, subject: str, group: GroupType) -> None:
+        client_id = self.options["metadata"]["id"]
+        logger.debug(f"[{client_id}] Starting message loop for subscription {subject}")
+
+        async for msg in subscription.messages:
+            if subject not in self._subscription_groups:
+                logger.debug(f"[{client_id}] Message loop for {subject} exiting: Group removed.")
+                break
+
+            try:
+                decoded_message = self._unpack_message(msg.data)
+                context: MessageContext = {
+                    "subject": msg.subject,
+                    "id": decoded_message["id"],
+                    "timestamp": int(datetime.now().timestamp() * 1000),
+                    "raw": msg,
+                }
+
+                self._metrics_tracker.track_receive()
+
+                if group.type_ == "request":
+                    self._task_set.add(self._handle_request_message(msg, decoded_message, group))
+                elif group.type_ == "rpc":
+                    self._task_set.add(self._handle_rpc_message(msg, decoded_message, group))
+                elif group.type_ == "generator":
+                    self._task_set.add(self._handle_generator_initiation_message(msg, decoded_message, group))
+                else:
+                    self._task_set.add(self._handle_normal_message(decoded_message, context, group))
+
+            except Exception:
+                self._metrics_tracker.track_message_error()
+                logger.error(f"[{client_id}] Error processing message for {subject}", exc_info=True)
+
+        logger.debug(f"[{client_id}] Message loop for subscription {subject} terminated.")
+        if subject in self._subscription_groups:
+            del self._subscription_groups[subject]
+
+    async def _handle_request_message(self, msg: Msg, message: Message[Any], group: GroupType) -> None:
+        handler = next(iter(group.handlers.values()), None)
+        if not handler:
+            return
+
+        try:
+            if inspect.iscoroutinefunction(handler) or inspect.isawaitable(handler):
+                result = await handler(message)
+            else:
+                result = await asyncio.to_thread(handler, message)
+
+            await self._respond_to(msg, result, message["id"])
+        except Exception as error:
+            self._metrics_tracker.track_message_error()
+            nats_error = ErrorFactory.message.execution_failed(
+                error,
+                {
+                    "description": "Handler execution failed",
+                    "code": ERROR_CODES.MESSAGE.EXECUTION_FAILED,
+                    "subject": msg.subject,
+                },
+            )
+            await self._respond_to(msg, None, message["id"], nats_error.to_json())
+
+    async def _handle_rpc_message(self, msg: Msg, message: Message[Any], group: GroupType) -> None:
+        handler = next(iter(group.handlers.values()), None)
+        if not handler:
+            return
+
+        try:
+            args: list[Any] = (
+                message["data"]
+                if isinstance(message["data"], list)
+                else [message["data"]]
+                if message.get("data") is not None
+                else []
+            )
+
+            if inspect.iscoroutinefunction(handler) or inspect.isawaitable(handler):
+                result = await handler(*args)
+            else:
+                result = await asyncio.to_thread(handler, *args)
+
+            await self._respond_to_rpc(msg, result, message["id"])
+        except Exception as error:
+            self._metrics_tracker.track_message_error()
+            nats_error = ErrorFactory.rpc.execution_failed(
+                error,
+                {
+                    "description": "RPC handler execution failed",
+                    "code": ERROR_CODES.MESSAGE.EXECUTION_FAILED,
+                    "subject": msg.subject,
+                },
+            )
+            await self._respond_to_rpc(msg, None, message["id"], None, True, nats_error.to_json())
+
+    async def _handle_generator_initiation_message(
+        self, msg: Msg, message: Message[Any], group: GroupType
+    ) -> None:
+        handler = next(iter(group.handlers.values()), None)
+        if not handler:
+            return
+
+        try:
+            if inspect.iscoroutinefunction(handler) or inspect.isawaitable(handler):
+                await handler(message)
+            else:
+                await asyncio.to_thread(handler, message)
+        except Exception:
+            self._metrics_tracker.track_message_error()
+            logger.error("Error processing generator request", exc_info=True)
+
+    async def _handle_normal_message(
+        self, message: Message[Any], context: MessageContext, group: GroupType
+    ) -> None:
+        tasks = [self._execute_handler(handler, message, context) for handler in group.handlers.values()]
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+    async def _execute_handler(
+        self, handler: Handlers[Any, Any], message: Message[Any], context: MessageContext
+    ) -> None:
+        try:
+            if inspect.iscoroutinefunction(handler) or inspect.isawaitable(handler):
+                await handler(message)
+            else:
+                await asyncio.to_thread(handler, message)
+        except Exception:
+            self._metrics_tracker.track_message_error()
+            logger.error("Error executing handler", exc_info=True)
+
+    async def _respond_to(
+        self,
+        msg: Msg,
+        data: Any,
+        request_id: str,
+        error: SerializedError | None = None,
+    ) -> None:
+        try:
+            if self.is_closed():
+                client_id = self.options["metadata"]["id"]
+                logger.debug(
+                    f"[{client_id}] Respond skipped: Connection closed or draining. Subject: {msg.subject}"
+                )
+                raise ErrorFactory.connection.not_connected(
+                    None,
+                    {
+                        "description": "Failed to respond: Connection closed or draining",
+                        "code": ERROR_CODES.CONNECTION.NOT_CONNECTED,
+                        "subject": msg.subject,
+                    },
+                )
+
+            response: ResponseMessage[Any] = {
+                "id": uid(),
+                "type": MESSAGE.TYPES.RESPONSE,
+                "requestId": request_id,
+                "data": data,
+                "metadata": self.options["metadata"],
+                "error": error,
+                "timestamp": int(datetime.now().timestamp() * 1000),
+            }
+
+            if msg.reply:
+                await msg.respond(self._pack_message(response))
+        except Exception:
+            self._metrics_tracker.track_message_error()
+            logger.error("Error sending request response", exc_info=True)
+
+    async def _respond_to_rpc(
+        self,
+        msg: Msg,
+        data: Any,
+        request_id: str,
+        stream_id: str | None = None,
+        done: bool | None = None,
+        error: SerializedError | None = None,
+    ) -> None:
+        try:
+            if self.is_closed():
+                client_id = self.options["metadata"]["id"]
+                logger.debug(
+                    f"[{client_id}] RPC Respond skipped: Connection closed or draining. Subject: {msg.subject}"
+                )
+                raise ErrorFactory.connection.not_connected(
+                    None,
+                    {
+                        "description": "Failed to respond: Connection closed or draining",
+                        "code": ERROR_CODES.CONNECTION.NOT_CONNECTED,
+                        "subject": msg.subject,
+                    },
+                )
+
+            response: RPCResponse[Any] = {
+                "id": uid(),
+                "type": MESSAGE.TYPES.RPC_RESPONSE,
+                "requestId": request_id,
+                "data": data,
+                "metadata": self.options["metadata"],
+                "streamId": stream_id,
+                "done": done,
+                "error": error,
+                "timestamp": int(datetime.now().timestamp() * 1000),
+            }
+
+            await msg.respond(self._pack_message(response))
+        except Exception:
+            self._metrics_tracker.track_message_error()
+            logger.error("Error sending RPC response", exc_info=True)
+
+    def _pack_message(self, msg: Any) -> bytes:
+        try:
+            return pack(msg)
+        except Exception as error:
+            self._metrics_tracker.track_message_error()
+            raise ErrorFactory.message.invalid_format(
+                error,
+                {
+                    "description": "Failed to pack message",
+                    "code": ERROR_CODES.MESSAGE.INVALID_FORMAT,
+                    "metadata": {"raw": msg},
+                },
+            ) from error
+
+    def _unpack_message(self, data: bytes) -> Any:
+        try:
+            return unpack(data)
+        except Exception as error:
+            raise ErrorFactory.message.invalid_format(
+                error,
+                {
+                    "description": "Failed to parse message",
+                    "code": ERROR_CODES.MESSAGE.INVALID_FORMAT,
+                    "metadata": {"raw": data},
+                },
+            ) from error
+
+    def _get_nats_auth_options(self) -> dict[str, str]:
+        if "server" in self.options["auth"]:
+            return {
+                "user": self.options["auth"]["server"]["user"],
+                "password": self.options["auth"]["server"]["password"],
+            }
+        return {
+            "user": self.options["auth"]["user"],
+            "password": self.options["auth"]["password"],
+        }
+
+    def get_connect_options(self) -> ConnectOptions:
+        default_options: ConnectOptions = {
+            "wait_for_server": False,
+        }
+        return {**default_options, **(self.options.get("connect", {}) or {})}
+
+    def get_reconnect_options(self) -> ReconnectOptions:
+        default_options: ReconnectOptions = {
+            "enabled": False,
+            "max_attempts": DEFAULTS.RETRY.MAX_RECONNECT_ATTEMPTS,
+            "delay": DEFAULTS.RETRY.DELAY,
+        }
+        return {**default_options, **(self.options.get("reconnect", {}) or {})}
+
+    def get_timeout_options(self) -> TimeoutOptions:
+        default_options: TimeoutOptions = {
+            "rpc": DEFAULTS.TIMEOUTS.RPC,
+            "request": DEFAULTS.TIMEOUTS.REQUEST,
+            "connection": DEFAULTS.TIMEOUTS.CONNECTION,
+        }
+
+        return {**default_options, **(self.options.get("timeout", {}) or {})}