@scrypted/server 0.6.24 → 0.7.2

package/python/rpc.py

@@ -4,7 +4,6 @@ import traceback
 import inspect
 from typing_extensions import TypedDict
 import weakref
-import sys
 
 jsonSerializable = set()
 jsonSerializable.add(float)
@@ -16,14 +15,16 @@ jsonSerializable.add(list)
 
 
 async def maybe_await(value):
-    if (inspect.iscoroutinefunction(value) or inspect.iscoroutine(value)):
+    if (inspect.isawaitable(value)):
         return await value
     return value
 
 
-class RpcResultException(Exception):
-    name = None
-    stack = None
+class RPCResultError(Exception):
+    name: str
+    stack: str
+    message: str
+    caught: Exception
 
     def __init__(self, caught, message):
         self.caught = caught
@@ -58,16 +59,31 @@ class RpcProxy(object):
         self.__dict__['__proxy_entry'] = entry
         self.__dict__['__proxy_constructor'] = proxyConstructorName
         self.__dict__['__proxy_peer'] = peer
-        self.__dict__['__proxy_props'] = proxyProps
+        self.__dict__[RpcPeer.PROPERTY_PROXY_PROPERTIES] = proxyProps
         self.__dict__['__proxy_oneway_methods'] = proxyOneWayMethods
 
+    def __aiter__(self):
+        if self.__dict__[RpcPeer.PROPERTY_PROXY_PROPERTIES] and 'Symbol(Symbol.asyncIterator)' in self.__dict__[RpcPeer.PROPERTY_PROXY_PROPERTIES]:
+            return self
+        raise Exception('RpcProxy is not an async iterable')
+
+    async def __anext__(self):
+        if self.__dict__[RpcPeer.PROPERTY_PROXY_PROPERTIES] and 'Symbol(Symbol.asyncIterator)' in self.__dict__[RpcPeer.PROPERTY_PROXY_PROPERTIES]:
+            try:
+                return await RpcProxyMethod(self, self.__dict__[RpcPeer.PROPERTY_PROXY_PROPERTIES]['Symbol(Symbol.asyncIterator)']['next'])()
+            except RPCResultError as e:
+                if e.name == 'StopAsyncIteration':
+                    raise StopAsyncIteration()
+                raise
+        raise Exception('RpcProxy is not an async iterable')
+
     def __getattr__(self, name):
         if name == '__proxy_finalizer_id':
             return self.dict['__proxy_entry']['finalizerId']
         if name in self.__dict__:
             return self.__dict__[name]
-        if self.__dict__['__proxy_props'] and name in self.__dict__['__proxy_props']:
-            return self.__dict__['__proxy_props'][name]
+        if self.__dict__[RpcPeer.PROPERTY_PROXY_PROPERTIES] and name in self.__dict__[RpcPeer.PROPERTY_PROXY_PROPERTIES]:
+            return self.__dict__[RpcPeer.PROPERTY_PROXY_PROPERTIES][name]
         return RpcProxyMethod(self, name)
 
     def __setattr__(self, name: str, value: Any) -> None:
@@ -85,6 +101,10 @@ class RpcProxy(object):
 
 
 class RpcPeer:
+    RPC_RESULT_ERROR_NAME = 'RPCResultError'
+    PROPERTY_PROXY_PROPERTIES = '__proxy_props'
+    PROPERTY_JSON_COPY_SERIALIZE_CHILDREN = '__json_copy_serialize_children'
+
     def __init__(self, send: Callable[[object, Callable[[Exception], None], Dict], None]) -> None:
         self.send = send
         self.idCounter = 1
@@ -97,12 +117,14 @@ class RpcPeer:
         self.pendingResults: Mapping[str, Future] = {}
         self.remoteWeakProxies: Mapping[str, any] = {}
         self.nameDeserializerMap: Mapping[str, RpcSerializer] = {}
+        self.onProxySerialization: Callable[[Any, str], Any] = None
+        self.killed = False
 
     def __apply__(self, proxyId: str, oneWayMethods: List[str], method: str, args: list):
         serializationContext: Dict = {}
         serializedArgs = []
         for arg in args:
-            serializedArgs.append(self.serialize(arg, False, serializationContext))
+            serializedArgs.append(self.serialize(arg, serializationContext))
 
         rpcApply = {
             'type': 'apply',
@@ -124,21 +146,112 @@ class RpcPeer:
             self.send(rpcApply, reject, serializationContext)
         return self.createPendingResult(send)
 
-    def kill(self):
+    def kill(self, message: str = None):
+        # not thread safe..
+        if self.killed:
+            return
         self.killed = True
 
-    def createErrorResult(self, result: Any, name: str, message: str, tb: str):
-        result['stack'] = tb if tb else 'no stack'
-        result['result'] = name if name else 'no name'
-        result['message'] = message if message else 'no message'
+        error = RPCResultError(None, message or 'peer was killed')
+        # this.killedDeferred.reject(error);
+        for str, future in self.pendingResults.items():
+            future.set_exception(error)
+
+        self.pendingResults = None
+        self.params = None
+        self.remoteWeakProxies = None
+        self.localProxyMap = None
+        self.localProxied = None
+
+    def createErrorResult(self, result: Any, e: Exception):
+        s = self.serializeError(e)
+        result['result'] = s
+        result['throw'] = True
+
+        # TODO 3/2/2023 deprecate these properties
+        tb = traceback.format_exc()
+        message = str(e)
+        result['stack'] = tb or '[no stack]',
+        result['message'] = message or '[no message]',
+        # END TODO
+
+
+    def deserializeError(e: Dict) -> RPCResultError:
+        error = RPCResultError(None, e.get('message'))
+        error.stack = e.get('stack')
+        error.name = e.get('name')
+        return error
+
+    def serializeError(self, e: Exception):
+        tb = traceback.format_exc()
+        name = type(e).__name__
+        message = str(e)
+
+        serialized = {
+            'stack': tb or '[no stack]',
+            'name': name or '[no name]',
+            'message': message or '[no message]',
+        }
+
+        return {
+            '__remote_constructor_name': RpcPeer.RPC_RESULT_ERROR_NAME,
+            '__serialized_value': serialized,
+        }
+    
+    # def getProxyProperties(value):
+    #     return getattr(value, RpcPeer.PROPERTY_PROXY_PROPERTIES, None)
+
+    # def setProxyProperties(value, properties):
+    #     setattr(value, RpcPeer.PROPERTY_PROXY_PROPERTIES, properties)
+
+    def prepareProxyProperties(value):
+        if not hasattr(value, '__aiter__') or not hasattr(value, '__anext__'):
+            return getattr(value, RpcPeer.PROPERTY_PROXY_PROPERTIES, None)
+
+        props = getattr(value, RpcPeer.PROPERTY_PROXY_PROPERTIES, None) or {}
+        if not props.get('Symbol(Symbol.asyncIterator)'):
+            props['Symbol(Symbol.asyncIterator)'] = {
+                'next': '__anext__',
+                'throw': 'athrow',
+                'return': 'aclose',
+            }
+        return props
+
+    def isTransportSafe(value: any):
+        return not value or (type(value) in jsonSerializable)
+
+    def serialize(self, value, serializationContext: Dict):
+        if type(value) == dict and value.get(RpcPeer.PROPERTY_JSON_COPY_SERIALIZE_CHILDREN, None):
+            ret = {}
+            for (key, val) in value.items():
+                ret[key] = self.serialize(val, serializationContext)
+            return ret
 
-    def serialize(self, value, requireProxy, serializationContext: Dict):
-        if (not value or (not requireProxy and type(value) in jsonSerializable)):
+        if (RpcPeer.isTransportSafe(value)):
             return value
 
         __remote_constructor_name = 'Function' if callable(value) else value.__proxy_constructor if hasattr(
             value, '__proxy_constructor') else type(value).__name__
 
+        if isinstance(value, Exception):
+            return self.serializeError(value)
+
+        serializerMapName = self.constructorSerializerMap.get(
+            type(value), None)
+        if serializerMapName:
+            __remote_constructor_name = serializerMapName
+            serializer = self.nameDeserializerMap.get(serializerMapName, None)
+            serialized = serializer.serialize(value, serializationContext)
+            ret = {
+                '__remote_proxy_id': None,
+                '__remote_proxy_finalizer_id': None,
+                '__remote_constructor_name': __remote_constructor_name,
+                '__remote_proxy_props': RpcPeer.prepareProxyProperties(value),
+                '__remote_proxy_oneway_methods': getattr(value, '__proxy_oneway_methods', None),
+                '__serialized_value': serialized,
+            }
+            return ret
+
         proxiedEntry = self.localProxied.get(value, None)
         if proxiedEntry:
             proxiedEntry['finalizerId'] = str(self.proxyCounter)
@@ -147,7 +260,7 @@ class RpcPeer:
                 '__remote_proxy_id': proxiedEntry['id'],
                 '__remote_proxy_finalizer_id': proxiedEntry['finalizerId'],
                 '__remote_constructor_name': __remote_constructor_name,
-                '__remote_proxy_props': getattr(value, '__proxy_props', None),
+                '__remote_proxy_props': RpcPeer.prepareProxyProperties(value),
                 '__remote_proxy_oneway_methods': getattr(value, '__proxy_oneway_methods', None),
             }
             return ret
@@ -160,22 +273,6 @@ class RpcPeer:
             }
             return ret
 
-        serializerMapName = self.constructorSerializerMap.get(
-            type(value), None)
-        if serializerMapName:
-            __remote_constructor_name = serializerMapName
-            serializer = self.nameDeserializerMap.get(serializerMapName, None)
-            serialized = serializer.serialize(value, serializationContext)
-            ret = {
-                '__remote_proxy_id': None,
-                '__remote_proxy_finalizer_id': None,
-                '__remote_constructor_name': __remote_constructor_name,
-                '__remote_proxy_props': getattr(value, '__proxy_props', None),
-                '__remote_proxy_oneway_methods': getattr(value, '__proxy_oneway_methods', None),
-                '__serialized_value': serialized,
-            }
-            return ret
-
         proxyId = str(self.proxyCounter)
         self.proxyCounter = self.proxyCounter + 1
         proxiedEntry = {
@@ -185,17 +282,24 @@ class RpcPeer:
         self.localProxied[value] = proxiedEntry
         self.localProxyMap[proxyId] = value
 
+        if self.onProxySerialization:
+            __remote_proxy_props = self.onProxySerialization(value, proxyId)
+        else:
+            __remote_proxy_props = RpcPeer.prepareProxyProperties(value)
+
         ret = {
             '__remote_proxy_id': proxyId,
             '__remote_proxy_finalizer_id': proxyId,
             '__remote_constructor_name': __remote_constructor_name,
-            '__remote_proxy_props': getattr(value, '__proxy_props', None),
+            '__remote_proxy_props': __remote_proxy_props,
             '__remote_proxy_oneway_methods': getattr(value, '__proxy_oneway_methods', None),
         }
 
         return ret
 
     def finalize(self, localProxiedEntry: LocalProxiedEntry):
+        if self.killed:
+            return
         id = localProxiedEntry['id']
         self.remoteWeakProxies.pop(id, None)
         rpcFinalize = {
@@ -224,6 +328,13 @@ class RpcPeer:
         if type(value) != dict:
             return value
 
+        copySerializeChildren = value.get(RpcPeer.PROPERTY_JSON_COPY_SERIALIZE_CHILDREN, None)
+        if copySerializeChildren:
+            ret = {}
+            for (key, val) in value.items():
+                ret[key] = self.deserialize(val, deserializationContext)
+            return ret
+
         __remote_proxy_id = value.get('__remote_proxy_id', None)
         __remote_proxy_finalizer_id = value.get(
             '__remote_proxy_finalizer_id', None)
@@ -235,6 +346,9 @@ class RpcPeer:
         __remote_proxy_oneway_methods = value.get(
             '__remote_proxy_oneway_methods', None)
 
+        if __remote_constructor_name == RpcPeer.RPC_RESULT_ERROR_NAME:
+            return RpcPeer.deserializeError(__serialized_value)
+
         if __remote_proxy_id:
             weakref = self.remoteWeakProxies.get('__remote_proxy_id', None)
             proxy = weakref() if weakref else None
@@ -247,7 +361,7 @@ class RpcPeer:
         if __local_proxy_id:
             ret = self.localProxyMap.get(__local_proxy_id, None)
             if not ret:
-                raise RpcResultException(
+                raise RPCResultError(
                     None, 'invalid local proxy id %s' % __local_proxy_id)
             return ret
 
@@ -258,7 +372,7 @@ class RpcPeer:
 
         return value
 
-    async def handleMessage(self, message: Any, deserializationContext: Dict):
+    async def handleMessage(self, message: Dict, deserializationContext: Dict):
         try:
             messageType = message['type']
             if messageType == 'param':
@@ -271,8 +385,7 @@ class RpcPeer:
                 try:
                     value = self.params.get(message['param'], None)
                     value = await maybe_await(value)
-                    result['result'] = self.serialize(
-                        value, message.get('requireProxy', None), serializationContext)
+                    result['result'] = self.serialize(value, serializationContext)
                 except Exception as e:
                     tb = traceback.format_exc()
                     self.createErrorResult(
@@ -299,6 +412,9 @@ class RpcPeer:
                     for arg in (message['args'] or []):
                         args.append(self.deserialize(arg, deserializationContext))
 
+                    # if method == 'asend' and hasattr(target, '__aiter__') and hasattr(target, '__anext__') and not len(args):
+                    #     args.append(None)
+
                     value = None
                     if method:
                         if not hasattr(target, method):
@@ -309,12 +425,11 @@ class RpcPeer:
                     else:
                         value = await maybe_await(target(*args))
 
-                    result['result'] = self.serialize(value, False, serializationContext)
+                    result['result'] = self.serialize(value, serializationContext)
+                except StopAsyncIteration as e:
+                    self.createErrorResult(result, e)
                 except Exception as e:
-                    tb = traceback.format_exc()
-                    # print('failure', method, e, tb)
-                    self.createErrorResult(
-                        result, type(e).__name__, str(e), tb)
+                    self.createErrorResult(result, e)
 
                 if not message.get('oneway', False):
                     self.send(result, None, serializationContext)
@@ -323,18 +438,21 @@ class RpcPeer:
                 id = message['id']
                 future = self.pendingResults.get(id, None)
                 if not future:
-                    raise RpcResultException(
+                    raise RPCResultError(
                         None, 'unknown result %s' % id)
                 del self.pendingResults[id]
-                if hasattr(message, 'message') or hasattr(message, 'stack'):
-                    e = RpcResultException(
+                if (hasattr(message, 'message') or hasattr(message, 'stack')) and not hasattr(message, 'throw'):
+                    e = RPCResultError(
                         None, message.get('message', None))
                     e.stack = message.get('stack', None)
                     e.name = message.get('name', None)
                     future.set_exception(e)
                     return
-                future.set_result(self.deserialize(
-                    message.get('result', None), deserializationContext))
+                deserialized = self.deserialize(message.get('result', None), deserializationContext)
+                if message.get('throw'):
+                    future.set_exception(deserialized)
+                else:
+                    future.set_result(deserialized)
             elif messageType == 'finalize':
                 finalizerId = message.get('__local_proxy_finalizer_id', None)
                 proxyId = message['__local_proxy_id']
@@ -347,7 +465,7 @@ class RpcPeer:
                     self.localProxied.pop(local, None)
                     local = self.localProxyMap.pop(proxyId, None)
             else:
-                raise RpcResultException(
+                raise RPCResultError(
                     None, 'unknown rpc message type %s' % type)
         except Exception as e:
             print("unhandled rpc error", self.peerName, e)
@@ -361,7 +479,7 @@ class RpcPeer:
         self.idCounter = self.idCounter + 1
         future = Future()
         self.pendingResults[id] = future
-        await cb(id, lambda e: future.set_exception(RpcResultException(e, None)))
+        await cb(id, lambda e: future.set_exception(RPCResultError(e, None)))
         return await future
 
     async def getParam(self, param):

package/python/rpc_reader.py

@@ -2,33 +2,16 @@ from __future__ import annotations
 
 import asyncio
 import base64
-import gc
 import json
-import sys
 import os
-import platform
-import shutil
-import subprocess
+import sys
 import threading
-import time
-import traceback
-import zipfile
 from asyncio.events import AbstractEventLoop
-from asyncio.futures import Future
-from asyncio.streams import StreamReader, StreamWriter
-from collections.abc import Mapping
-from io import StringIO
 from os import sys
-from typing import Any, List, Optional, Set, Tuple
+from typing import List
 
 import aiofiles
-import scrypted_python.scrypted_sdk.types
-from scrypted_python.scrypted_sdk import ScryptedStatic, PluginFork
-from scrypted_python.scrypted_sdk.types import Device, DeviceManifest, EventDetails, ScryptedInterfaceProperty, Storage
-from typing_extensions import TypedDict
 import rpc
-import multiprocessing
-import multiprocessing.connection
 
 
 class BufferSerializer(rpc.RpcSerializer):
@@ -53,39 +36,61 @@ class SidebandBufferSerializer(rpc.RpcSerializer):
         buffer = buffers.pop()
         return buffer
 
-async def readLoop(loop, peer: rpc.RpcPeer, reader):
+async def readLoop(loop, peer: rpc.RpcPeer, reader: asyncio.StreamReader):
     deserializationContext = {
         'buffers': []
     }
 
+    if isinstance(reader, asyncio.StreamReader):
+        async def read(n):
+            return await reader.readexactly(n)
+    else:
+        async def read(n):
+            return await reader.read(n)
+
+
     while True:
-        try:
-            lengthBytes = await reader.read(4)
-            typeBytes = await reader.read(1)
-            type = typeBytes[0]
-            length = int.from_bytes(lengthBytes, 'big')
-            data = await reader.read(length - 1)
-
-            if type == 1:
-                deserializationContext['buffers'].append(data)
-                continue
-
-            message = json.loads(data)
-            asyncio.run_coroutine_threadsafe(
-                peer.handleMessage(message, deserializationContext), loop)
-
-            deserializationContext = {
-                'buffers': []
-            }
-        except Exception as e:
-            print('read loop error: ' + peer.peerName, e)
-            sys.exit()
-
-async def prepare_peer_readloop(loop: AbstractEventLoop, readFd: int, writeFd: int):
-    reader = await aiofiles.open(readFd, mode='rb')
+        lengthBytes = await read(4)
+        typeBytes = await read(1)
+        type = typeBytes[0]
+        length = int.from_bytes(lengthBytes, 'big')
+        data = await read(length - 1)
+
+        if type == 1:
+            deserializationContext['buffers'].append(data)
+            continue
+
+        message = json.loads(data)
+        asyncio.run_coroutine_threadsafe(
+            peer.handleMessage(message, deserializationContext), loop)
+
+        deserializationContext = {
+            'buffers': []
+        }
+
+async def prepare_peer_readloop(loop: AbstractEventLoop, readFd: int = None, writeFd: int = None, reader: asyncio.StreamReader = None, writer: asyncio.StreamWriter = None):
+    reader = reader or await aiofiles.open(readFd, mode='rb')
 
     mutex = threading.Lock()
 
+    if writer:
+        def write(buffers, reject):
+            try:
+                for b in buffers:
+                    writer.write(b)
+            except Exception as e:
+                if reject:
+                    reject(e)
+            return None
+    else:
+        def write(buffers, reject):
+            try:
+                for b in buffers:
+                    os.write(writeFd, b)
+            except Exception as e:
+                if reject:
+                    reject(e)
+
     def send(message, reject=None, serializationContext=None):
         with mutex:
             if serializationContext:
@@ -95,34 +100,26 @@ async def prepare_peer_readloop(loop: AbstractEventLoop, readFd: int, writeFd: i
                         length = len(buffer) + 1
                         lb = length.to_bytes(4, 'big')
                         type = 1
-                        try:
-                            os.write(writeFd, lb)
-                            os.write(writeFd, bytes([type]))
-                            os.write(writeFd, buffer)
-                        except Exception as e:
-                            if reject:
-                                reject(e)
-                            return
+                        write([lb, bytes([type]), buffer], reject)
 
             jsonString = json.dumps(message)
             b = bytes(jsonString, 'utf8')
             length = len(b) + 1
             lb = length.to_bytes(4, 'big')
             type = 0
-            try:
-                os.write(writeFd, lb)
-                os.write(writeFd, bytes([type]))
-                os.write(writeFd, b)
-            except Exception as e:
-                if reject:
-                    reject(e)
+            write([lb, bytes([type]), b], reject)
 
     peer = rpc.RpcPeer(send)
     peer.nameDeserializerMap['Buffer'] = SidebandBufferSerializer()
     peer.constructorSerializerMap[bytes] = 'Buffer'
     peer.constructorSerializerMap[bytearray] = 'Buffer'
+    peer.constructorSerializerMap[memoryview] = 'Buffer'
 
     async def peerReadLoop():
-        await readLoop(loop, peer, reader)
+        try:
+            await readLoop(loop, peer, reader)
+        except:
+            peer.kill()
+            raise
 
     return peer, peerReadLoop

package/src/http-interfaces.ts

@@ -61,7 +61,11 @@ export function createResponseInterface(res: Response, unzippedDir: string, file
                     }
                 }
             }
-            res.sendFile(filePath);
+
+            // prefer etag
+            res.sendFile(filePath, {
+                cacheControl: false,
+            });
         }
 
         sendSocket(socket: net.Socket, options: HttpResponseOptions) {

package/src/listen-zero.ts

@@ -26,10 +26,14 @@ export async function listenZeroSingleClient(hostname?: string) {
 
     clientPromise.catch(() => { });
 
+    let host = hostname;
+    if (!host || host === '0.0.0.0')
+        host = '127.0.0.1';
+
     return {
         server,
-        url: `tcp://127.0.0.1:${port}`,
-        host: '127.0.0.1',
+        url: `tcp://${host}:${port}`,
+        host,
         port,
         clientPromise,
     }