@scrypted/server 0.0.82 → 0.0.86

package/python/plugin-remote.py

@@ -1,10 +1,13 @@
 from __future__ import annotations
+from asyncio.futures import Future
+from asyncio.streams import StreamReader, StreamWriter
 import os
 from os import sys
+from sys import stderr, stdout
 more = os.path.join(os.getcwd(), 'node_modules/@scrypted/sdk')
 sys.path.insert(0, more)
 import scrypted_python.scrypted_sdk
-from scrypted_python.scrypted_sdk.types import MediaManager, MediaObject, ScryptedInterfaceProperty
+from scrypted_python.scrypted_sdk.types import DeviceManifest, MediaManager, MediaObject, ScryptedInterfaceProperty
 
 from collections.abc import Mapping
 from genericpath import exists
@@ -13,7 +16,7 @@ import asyncio
 from asyncio.events import AbstractEventLoop
 import json
 import aiofiles
-from typing import TypedDict
+from typing import Tuple, TypedDict
 import base64
 import time
 import zipfile
@@ -50,9 +53,9 @@ class DeviceState(scrypted_python.scrypted_sdk.DeviceState):
 
     def setScryptedProperty(self, property: str, value: Any):
         if property == ScryptedInterfaceProperty.id.value:
-            raise Exception("id is read only");
+            raise Exception("id is read only")
         if property == ScryptedInterfaceProperty.mixins.value:
-            raise Exception("mixins is read only");
+            raise Exception("mixins is read only")
         if property == ScryptedInterfaceProperty.interfaces.value:
             raise Exception("interfaces is a read only post-mixin computed property, use providedInterfaces");
 
@@ -66,6 +69,11 @@ class DeviceState(scrypted_python.scrypted_sdk.DeviceState):
         self.systemManager.api.setState(self.nativeId, property, value)
 
 
+class DeviceStorage:
+    id: str
+    nativeId: str
+    storage: Mapping[str, str] = {}
+
 class DeviceManager(scrypted_python.scrypted_sdk.DeviceManager):
     def __init__(self, nativeIds: Mapping[str, DeviceStorage], systemManager: SystemManager) -> None:
         super().__init__()
@@ -80,6 +88,9 @@ class DeviceManager(scrypted_python.scrypted_sdk.DeviceManager):
     async def onDeviceEvent(self, nativeId: str, eventInterface: str, eventData: Any = None) -> None:
         await self.systemManager.api.onDeviceEvent(nativeId, eventInterface, eventData)
 
+    async def onDevicesChanged(self, devices: DeviceManifest) -> None:
+        return await self.systemManager.api.onDevicesChanged(devices)
+
 
 class BufferSerializer(rpc.RpcSerializer):
     def serialize(self, value):
@@ -88,20 +99,13 @@ class BufferSerializer(rpc.RpcSerializer):
     def deserialize(self, value):
         return base64.b64decode(value)
 
-
-class DeviceStorage:
-    id: str
-    nativeId: str
-    storage: Mapping[str, str] = {}
-
-
 class PluginRemote:
     systemState: Mapping[str, Mapping[str, SystemDeviceState]] = {}
     nativeIds: Mapping[str, DeviceStorage] = {}
     pluginId: str
     mediaManager: MediaManager
     loop: AbstractEventLoop
-    consoles: Mapping[str, StringIO] = {}
+    consoles: Mapping[str, Future[Tuple[StreamReader, StreamWriter]]] = {}
 
     def __init__(self, api, pluginId, loop: AbstractEventLoop):
         self.api = api
@@ -115,14 +119,36 @@ class PluginRemote:
             'setNativeId',
         ]
 
-    def print(self, nativeId: str, *values: object, sep: Optional[str] = ...,
-            end: Optional[str] = ...,
-            flush: bool = ...,):
-        console = self.consoles.get(nativeId)
-        if not console:
-            console = StringIO()
-            self.consoles[nativeId] = console
-        print(*values, sep = sep, end = end, file = console, flush = flush)
+    async def print_async(self, nativeId: str, *values: object, sep: Optional[str] = ' ',
+            end: Optional[str] = '\n',
+            flush: bool = False,):
+        consoleFuture = self.consoles.get(nativeId)
+        if not consoleFuture:
+            consoleFuture = Future()
+            self.consoles[nativeId] = consoleFuture
+            plugins = await self.api.getComponent('plugins')
+            port = await plugins.getRemoteServicePort(self.pluginId, 'console-writer')
+            connection = await asyncio.open_connection(port = port)
+            _, writer = connection
+            if not nativeId:
+                nid = 'undefined'
+            else:
+                nid = nativeId
+            nid += '\n'
+            writer.write(nid.encode('utf8'))
+            consoleFuture.set_result(connection)
+        _, writer = await consoleFuture
+        strio = StringIO()
+        print(*values, sep=sep, end=end, flush=flush, file=strio)
+        strio.seek(0)
+        b = strio.read().encode('utf8')
+        writer.write(b)
+
+
+    def print(self, nativeId: str, *values: object, sep: Optional[str] = ' ',
+            end: Optional[str] = '\n',
+            flush: bool = False,):
+        asyncio.run_coroutine_threadsafe(self.print_async(nativeId, *values, sep=sep, end=end, flush=flush), self.loop)
 
     async def loadZip(self, packageJson, zipData, options=None):
         zipPath = options['filename']
@@ -139,13 +165,14 @@ class PluginRemote:
 
         if 'requirements.txt' in zip.namelist():
             requirements = zip.open('requirements.txt').read()
-            str_requirements = requirements.decode('utf8');
+            str_requirements = requirements.decode('utf8')
 
             requirementstxt = os.path.join(python_modules, 'requirements.txt')
+            installed_requirementstxt = os.path.join(python_modules, 'installed-requirements.txt')
 
             need_pip = True
             try:
-                existing = open(requirementstxt).read()
+                existing = open(installed_requirementstxt).read()
                 need_pip = existing != str_requirements
             except:
                 pass
@@ -159,8 +186,21 @@ class PluginRemote:
                 f.close()
 
                 # os.system('pip install -r %s --target %s' % (requirementstxt, python_modules))
-                result = subprocess.check_output(['pip', 'install', '-r', requirementstxt, '--target', python_modules], stderr=subprocess.STDOUT, text=True)
-                print(result)
+                p = subprocess.Popen(['pip', 'install', '-r', requirementstxt, '--target', python_modules], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+                while True:
+                    line = p.stdout.readline()
+                    if not line:
+                        break
+                    line = line.decode('utf8').rstrip('\r\n')
+                    print(line)
+                result = p.wait()
+                print('pip install result %s' % result)
+                if result:
+                    raise Exception('non-zero result from pip %s' % result)
+
+                f = open(installed_requirementstxt, 'wb')
+                f.write(requirements)
+                f.close()
             else:
                 print('requirements.txt (up to date)')
                 print(str_requirements)

package/python/rpc.py

@@ -276,7 +276,7 @@ class RpcPeer:
                     print('failure', method, e)
                     tb = traceback.format_exc()
                     self.createErrorResult(
-                        result, type(e).__name, str(e), tb)
+                        result, type(e).__name__, str(e), tb)
 
                 if not message.get('oneway', False):
                     self.send(result)

package/src/plugin/plugin-console.ts

@@ -0,0 +1,115 @@
+import { ScryptedNativeId } from '@scrypted/sdk/types'
+import { EventEmitter } from 'ws';
+import { listenZero } from './listen-zero';
+import { Server } from 'net';
+import { once } from 'events';
+import net from 'net'
+import { Readable } from 'stream';
+
+export interface ConsoleServer {
+    readPort: number,
+    writePort: number,
+    readServer: net.Server,
+    writeServer: net.Server,
+    sockets: Set<net.Socket>;
+}
+export async function createConsoleServer(stdout: Readable, stderr: Readable) {
+    const outputs = new Map<string, Buffer[]>();
+    const appendOutput = (data: Buffer, nativeId: ScryptedNativeId) => {
+        if (!nativeId)
+            nativeId = undefined;
+        let buffers = outputs.get(nativeId);
+        if (!buffers) {
+            buffers = [];
+            outputs.set(nativeId, buffers);
+        }
+        buffers.push(data);
+        // when we're over 4000 lines or whatever these buffer are,
+        // truncate down to 2000.
+        if (buffers.length > 4000)
+            outputs.set(nativeId, buffers.slice(buffers.length - 2000))
+    };
+
+    const sockets = new Set<net.Socket>();
+
+    const events = new EventEmitter();
+    events.on('stdout', appendOutput);
+    events.on('stderr', appendOutput);
+
+    stdout.on('data', data => events.emit('stdout', data));
+    stderr.on('data', data => events.emit('stderr', data));
+
+    const readServer = new Server(async (socket) => {
+        sockets.add(socket);
+
+        let [filter] = await once(socket, 'data');
+        filter = filter.toString().trim();
+        if (filter === 'undefined')
+            filter = undefined;
+
+        const buffers = outputs.get(filter);
+        if (buffers) {
+            const concat = Buffer.concat(buffers);
+            outputs.set(filter, [concat]);
+            socket.write(concat);
+        }
+
+        const cb = (data: Buffer, nativeId: ScryptedNativeId) => {
+            if (nativeId !== filter)
+                return;
+            socket.write(data);
+        };
+        events.on('stdout', cb)
+        events.on('stderr', cb)
+
+        const cleanup = () => {
+            events.removeListener('stdout', cb);
+            events.removeListener('stderr', cb);
+            socket.destroy();
+            socket.removeAllListeners();
+            sockets.delete(socket);
+        };
+
+        socket.on('close', cleanup);
+        socket.on('error', cleanup);
+        socket.on('end', cleanup);
+    });
+
+    const writeServer = new Server(async (socket) => {
+        sockets.add(socket);
+        const [data] = await once(socket, 'data');
+        let filter: string = data.toString();
+        const newline = filter.indexOf('\n');
+        if (newline !== -1) {
+            socket.unshift(Buffer.from(filter.substring(newline + 1)));
+        }
+        filter = filter.substring(0, newline);
+
+        if (filter === 'undefined')
+            filter = undefined;
+
+        const cb = (data: Buffer) => events.emit('stdout', data, filter);
+
+        socket.on('data', cb);
+
+        const cleanup = () => {
+            socket.destroy();
+            socket.removeAllListeners();
+            sockets.delete(socket);
+        };
+
+        socket.once('close', cleanup);
+        socket.once('error', cleanup);
+        socket.once('end', cleanup);
+    });
+    const readPort = await listenZero(readServer);
+    const writePort = await listenZero(writeServer);
+
+    return {
+        readPort,
+        writePort,
+        readServer,
+        writeServer,
+        sockets,
+    };
+}

package/src/plugin/plugin-device.ts

@@ -16,7 +16,6 @@ interface MixinTable {
 export class PluginDeviceProxyHandler implements ProxyHandler<any>, ScryptedDevice {
     scrypted: ScryptedRuntime;
     id: string;
-    // proxy: Promise<any>;
     mixinTable: Promise<MixinTable[]>;
 
     constructor(scrypted: ScryptedRuntime, id: string) {
@@ -24,6 +23,7 @@ export class PluginDeviceProxyHandler implements ProxyHandler<any>, ScryptedDevi
         this.id = id;
     }
 
+    // should this be async?
     invalidate() {
         const mixinTable = this.mixinTable;
         this.mixinTable = undefined;