@scrypted/server 0.0.104 → 0.0.108

package/src/plugin/plugin-remote.ts

@@ -6,8 +6,7 @@ import { PluginAPI, PluginLogger, PluginRemote, PluginRemoteLoadZipOptions } fro
 import { SystemManagerImpl } from './system';
 import { RpcPeer, RPCResultError, PROPERTY_PROXY_ONEWAY_METHODS, PROPERTY_JSON_DISABLE_SERIALIZATION } from '../rpc';
 import { BufferSerializer } from './buffer-serializer';
-import { EventEmitter } from 'events';
-import { createWebSocketClass } from './plugin-remote-websocket';
+import { createWebSocketClass, WebSocketConnectCallbacks, WebSocketMethods } from './plugin-remote-websocket';
 
 class DeviceLogger implements Logger {
     nativeId: ScryptedNativeId;
@@ -74,8 +73,9 @@ class EndpointManagerImpl implements EndpointManager {
     }
 
     async getUrlSafeIp() {
+        // ipv6 addresses have colons and need to be bracketed for url safety
         const ip: string = await this.api.getComponent('SCRYPTED_IP_ADDRESS')
-        return ip.includes(':') ? `[${ip}]` : ip;
+        return ip?.includes(':') ? `[${ip}]` : ip;
     }
 
     async getAuthenticatedPath(nativeId?: ScryptedNativeId): Promise<string> {
@@ -261,14 +261,7 @@ class StorageImpl implements Storage {
     }
 }
 
-interface WebSocketCallbacks {
-    end: any;
-    error: any;
-    data: any;
-}
-
-
-export async function setupPluginRemote(peer: RpcPeer, api: PluginAPI, pluginId: string): Promise<PluginRemote> {
+export async function setupPluginRemote(peer: RpcPeer, api: PluginAPI, pluginId: string, getSystemState: () => { [id: string]: { [property: string]: SystemDeviceState } }): Promise<PluginRemote> {
     try {
         // the host/remote connection can be from server to plugin (node to node),
         // core plugin to web (node to browser).
@@ -276,16 +269,46 @@ export async function setupPluginRemote(peer: RpcPeer, api: PluginAPI, pluginId:
         // but in plugin-host, mark Buffer as transport safe.
         peer.addSerializer(Buffer, 'Buffer', new BufferSerializer());
         const getRemote = await peer.getParam('getRemote');
-        return await getRemote(api, pluginId);
+        const remote = await getRemote(api, pluginId);
+
+        await remote.setSystemState(getSystemState());
+        api.listen((id, eventDetails, eventData) => {
+            // ScryptedDevice events will be handled specially and repropagated by the remote.
+            if (eventDetails.eventInterface === ScryptedInterface.ScryptedDevice) {
+                if (eventDetails.property === ScryptedInterfaceProperty.id) {
+                    // a change on the id property means device was deleted
+                    remote.updateDeviceState(eventData, undefined);
+                }
+                else {
+                    // a change on anything else is a descriptor update
+                    remote.updateDeviceState(id, getSystemState()[id]);
+                }
+                return;
+            }
+
+            if (eventDetails.property) {
+                remote.notify(id, eventDetails.eventTime, eventDetails.eventInterface, eventDetails.property, getSystemState()[id]?.[eventDetails.property], eventDetails.changed);
+            }
+            else {
+                remote.notify(id, eventDetails.eventTime, eventDetails.eventInterface, eventDetails.property, eventData, eventDetails.changed);
+            }
+        });
+
+        return remote;
     }
     catch (e) {
         throw new RPCResultError(peer, 'error while retrieving PluginRemote', e);
     }
 }
 
+export interface WebSocketCustomHandler {
+    id: string,
+    methods: WebSocketMethods;
+}
+
 export interface PluginRemoteAttachOptions {
     createMediaManager?: (systemManager: SystemManager) => Promise<MediaManager>;
-    getServicePort?: (name: string) => Promise<number>;
+    getServicePort?: (name: string, ...args: any[]) => Promise<number>;
     getDeviceConsole?: (nativeId?: ScryptedNativeId) => Console;
     getPluginConsole?: () => Console;
     getMixinConsole?: (id: string, nativeId?: ScryptedNativeId) => Console;
@@ -308,8 +331,9 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
         const systemManager = new SystemManagerImpl();
         const deviceManager = new DeviceManagerImpl(systemManager, getDeviceConsole, getMixinConsole);
         const endpointManager = new EndpointManagerImpl();
-        const ioSockets: { [id: string]: WebSocketCallbacks } = {};
         const mediaManager = await api.getMediaManager() || await createMediaManager(systemManager);
+        peer.params['mediaManager'] = mediaManager;
+        const ioSockets: { [id: string]: WebSocketConnectCallbacks } = {};
 
         systemManager.api = api;
         deviceManager.api = api;
@@ -382,11 +406,11 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
             async updateDeviceState(id: string, state: { [property: string]: SystemDeviceState }) {
                 if (!state) {
                     delete systemManager.state[id];
-                    systemManager.events.notify(id, Date.now(), ScryptedInterface.ScryptedDevice, ScryptedInterfaceProperty.id, id, true);
+                    systemManager.events.notify(undefined, undefined, ScryptedInterface.ScryptedDevice, ScryptedInterfaceProperty.id, id, true);
                 }
                 else {
                     systemManager.state[id] = state;
-                    systemManager.events.notify(id, Date.now(), ScryptedInterface.ScryptedDevice, undefined, undefined, true);
+                    systemManager.events.notify(id, undefined, ScryptedInterface.ScryptedDevice, undefined, state, true);
                 }
             },
 
@@ -431,32 +455,16 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
                     volume.writeFileSync(name, entry.getData());
                 }
 
-                function websocketConnect(url: string, protocols: any, connect: any, end: any, error: any, data: any) {
-                    if (url.startsWith('io://')) {
-                        const id = url.substring('io://'.length);
-
-                        ioSockets[id] = {
-                            data,
-                            error,
-                            end
-                        };
-
-                        connect(undefined, {
-                            close: () => api.ioClose(id),
-                        }, (message: string) => api.ioSend(id, message));
-                    }
-                    else if (url.startsWith('ws://')) {
-                        const id = url.substring('ws://'.length);
+                function websocketConnect(url: string, protocols: any, callbacks: WebSocketConnectCallbacks) {
+                    if (url.startsWith('io://') || url.startsWith('ws://')) {
+                        const id = url.substring('xx://'.length);
 
-                        ioSockets[id] = {
-                            data,
-                            error,
-                            end
-                        };
+                        ioSockets[id] = callbacks;
 
-                        connect(undefined, {
+                        callbacks.connect(undefined, {
                             close: () => api.ioClose(id),
-                        }, (message: string) => api.ioSend(id, message));
+                            send: (message: string) => api.ioSend(id, message),
+                        });
                     }
                     else {
                         throw new Error('unsupported websocket');

package/src/runtime.ts

@@ -4,9 +4,8 @@ import { ScryptedNativeId, Device, EngineIOHandler, HttpRequest, HttpRequestHand
 import { PluginDeviceProxyHandler } from './plugin/plugin-device';
 import { Plugin, PluginDevice, ScryptedAlert } from './db-types';
 import { getState, ScryptedStateManager, setState } from './state';
-import { Request, Response, Router } from 'express';
+import { Request, Response } from 'express';
 import { createResponseInterface } from './http-interfaces';
-import bodyParser from 'body-parser';
 import http, { ServerResponse } from 'http';
 import https from 'https';
 import express from 'express';
@@ -15,7 +14,7 @@ import { getDisplayName, getDisplayRoom, getDisplayType, getProvidedNameOrDefaul
 import { URL } from "url";
 import qs from "query-string";
 import { PluginComponent } from './services/plugin';
-import { Server as WebSocketServer } from "ws";
+import WebSocket, { Server as WebSocketServer } from "ws";
 import axios from 'axios';
 import tar from 'tar';
 import { once } from 'events';
@@ -30,6 +29,7 @@ import io from 'engine.io';
 import { spawn as ptySpawn } from 'node-pty';
 import rimraf from 'rimraf';
 import { getPluginVolume } from './plugin/plugin-volume';
+import { PluginHttp } from './plugin/plugin-http';
 
 interface DeviceProxyPair {
     handler: PluginDeviceProxyHandler;
@@ -39,13 +39,17 @@ interface DeviceProxyPair {
 const MIN_SCRYPTED_CORE_VERSION = 'v0.0.146';
 const PLUGIN_DEVICE_STATE_VERSION = 2;
 
-export class ScryptedRuntime {
+interface HttpPluginData {
+    pluginHost: PluginHost;
+    pluginDevice: PluginDevice
+}
+
+export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
     datastore: Level;
     plugins: { [id: string]: PluginHost } = {};
     pluginDevices: { [id: string]: PluginDevice } = {};
     devices: { [id: string]: DeviceProxyPair } = {};
     stateManager = new ScryptedStateManager(this);
-    app: Router;
     logger = new Logger(this, '', 'Scrypted');
     devicesLogger = this.logger.getLogger('device', 'Devices');
     wss = new WebSocketServer({ noServer: true });
@@ -55,30 +59,12 @@ export class ScryptedRuntime {
     });
 
     constructor(datastore: Level, insecure: http.Server, secure: https.Server, app: express.Application) {
+        super(app);
         this.datastore = datastore;
         this.app = app;
 
         app.disable('x-powered-by');
 
-        app.all(['/endpoint/@:owner/:pkg/public/engine.io/*', '/endpoint/:pkg/public/engine.io/*'], (req, res) => {
-            this.endpointHandler(req, res, true, true, this.handleEngineIOEndpoint.bind(this))
-        });
-
-        app.all(['/endpoint/@:owner/:pkg/engine.io/*', '/endpoint/@:owner/:pkg/engine.io/*'], (req, res) => {
-            this.endpointHandler(req, res, false, true, this.handleEngineIOEndpoint.bind(this))
-        });
-
-        // stringify all http endpoints
-        app.all(['/endpoint/@:owner/:pkg/public', '/endpoint/@:owner/:pkg/public/*', '/endpoint/:pkg', '/endpoint/:pkg/*'], bodyParser.text() as any);
-
-        app.all(['/endpoint/@:owner/:pkg/public', '/endpoint/@:owner/:pkg/public/*', '/endpoint/:pkg/public', '/endpoint/:pkg/public/*'], (req, res) => {
-            this.endpointHandler(req, res, true, false, this.handleRequestEndpoint.bind(this))
-        });
-
-        app.all(['/endpoint/@:owner/:pkg', '/endpoint/@:owner/:pkg/*', '/endpoint/:pkg', '/endpoint/:pkg/*'], (req, res) => {
-            this.endpointHandler(req, res, false, false, this.handleRequestEndpoint.bind(this))
-        });
-
         app.get('/web/oauth/callback', (req, res) => {
             this.oauthCallback(req, res);
         });
@@ -190,7 +176,7 @@ export class ScryptedRuntime {
         }
     }
 
-    async getPluginForEndpoint(endpoint: string) {
+    async getPluginForEndpoint(endpoint: string): Promise<HttpPluginData> {
         let pluginHost = this.plugins[endpoint] ?? this.getPluginHostForDeviceId(endpoint);
         if (endpoint === '@scrypted/core') {
             // enforce a minimum version on @scrypted/core
@@ -254,110 +240,55 @@ export class ScryptedRuntime {
             this.shellio.handleRequest(req, res);
     }
 
-    async endpointHandler(req: Request, res: Response, isPublicEndpoint: boolean, isEngineIOEndpoint: boolean,
-        handler: (req: Request, res: Response, endpointRequest: HttpRequest, pluginHost: PluginHost, pluginDevice: PluginDevice) => void) {
-
-        const isUpgrade = req.headers.connection?.toLowerCase() === 'upgrade';
-
-        const end = (code: number, message: string) => {
-            if (isUpgrade) {
-                const socket = res.socket;
-                socket.write(`HTTP/1.1 ${code} ${message}\r\n` +
-                    '\r\n');
-                socket.destroy();
-            }
-            else {
-                res.status(code);
-                res.send(message);
-            }
-        };
-
-        if (!isPublicEndpoint && !res.locals.username) {
-            end(401, 'Not Authorized');
-            return;
-        }
-
-        const { owner, pkg } = req.params;
-        let endpoint = pkg;
-        if (owner)
-            endpoint = `@${owner}/${endpoint}`;
-
-        const { pluginHost, pluginDevice } = await this.getPluginForEndpoint(endpoint);
+    async getEndpointPluginData(endpoint: string, isUpgrade: boolean, isEngineIOEndpoint: boolean): Promise<HttpPluginData> {
+        const ret = await this.getPluginForEndpoint(endpoint);
+        const { pluginDevice } = ret;
 
         // check if upgrade requests can be handled. must be websocket.
         if (isUpgrade) {
-            if (req.headers.upgrade?.toLowerCase() !== 'websocket' || !pluginDevice?.state.interfaces.value.includes(ScryptedInterface.EngineIOHandler)) {
-                end(404, 'Not Found');
+            if (!pluginDevice?.state.interfaces.value.includes(ScryptedInterface.EngineIOHandler)) {
                 return;
             }
         }
         else {
             if (!isEngineIOEndpoint && !pluginDevice?.state.interfaces.value.includes(ScryptedInterface.HttpRequestHandler)) {
-                end(404, 'Not Found');
                 return;
             }
         }
 
-        let rootPath = `/endpoint/${endpoint}`;
-        if (isPublicEndpoint)
-            rootPath += '/public'
-
-        const body = req.body && typeof req.body !== 'string' ? JSON.stringify(req.body) : req.body;
+        return ret;
+    }
 
-        const httpRequest: HttpRequest = {
-            body,
-            headers: req.headers,
-            method: req.method,
-            rootPath,
-            url: req.url,
-            isPublicEndpoint,
-            username: res.locals.username,
-        };
+    async handleWebSocket(endpoint: string, httpRequest: HttpRequest, ws: WebSocket, pluginData: HttpPluginData): Promise<void> {
+        const { pluginDevice } = pluginData;
 
-        if (isEngineIOEndpoint && !isUpgrade && isPublicEndpoint) {
-            res.header("Access-Control-Allow-Origin", '*');
+        const handler = this.getDevice<EngineIOHandler>(pluginDevice._id);
+        const id = 'ws-' + this.wsAtomic++;
+        const pluginHost = this.plugins[endpoint] ?? this.getPluginHostForDeviceId(endpoint);
+        if (!pluginHost) {
+            ws.close();
+            return;
         }
+        pluginHost.ws[id] = ws;
 
-        if (!isEngineIOEndpoint && isUpgrade) {
-            this.wss.handleUpgrade(req, req.socket, (req as any).upgradeHead, async (ws) => {
-                try {
-                    const handler = this.getDevice<EngineIOHandler>(pluginDevice._id);
-                    const id = 'ws-' + this.wsAtomic++;
-                    const pluginHost = this.plugins[endpoint] ?? this.getPluginHostForDeviceId(endpoint);
-                    if (!pluginHost) {
-                        ws.close();
-                        return;
-                    }
-                    pluginHost.ws[id] = ws;
-
-                    ws.on('message', async (message) => {
-                        try {
-                            pluginHost.remote.ioEvent(id, 'message', message)
-                        }
-                        catch (e) {
-                            ws.close();
-                        }
-                    });
-                    ws.on('close', async (reason) => {
-                        try {
-                            pluginHost.remote.ioEvent(id, 'close');
-                        }
-                        catch (e) {
-                        }
-                        delete pluginHost.ws[id];
-                    });
-
-                    await handler.onConnection(httpRequest, `ws://${id}`);
-                }
-                catch (e) {
-                    console.error('websocket plugin error', e);
-                    ws.close();
-                }
-            });
-        }
-        else {
-            handler(req, res, httpRequest, pluginHost, pluginDevice);
-        }
+        ws.on('message', async (message) => {
+            try {
+                pluginHost.remote.ioEvent(id, 'message', message)
+            }
+            catch (e) {
+                ws.close();
+            }
+        });
+        ws.on('close', async (reason) => {
+            try {
+                pluginHost.remote.ioEvent(id, 'close');
+            }
+            catch (e) {
+            }
+            delete pluginHost.ws[id];
+        });
+
+        await handler.onConnection(httpRequest, `ws://${id}`);
     }
 
     async getComponent(componentId: string): Promise<any> {
@@ -381,7 +312,9 @@ export class ScryptedRuntime {
         }
     }
 
-    async handleEngineIOEndpoint(req: Request, res: ServerResponse, endpointRequest: HttpRequest, pluginHost: PluginHost, pluginDevice: PluginDevice) {
+    async handleEngineIOEndpoint(req: Request, res: ServerResponse, endpointRequest: HttpRequest, pluginData: HttpPluginData) {
+        const { pluginHost, pluginDevice } = pluginData;
+
         (req as any).scrypted = {
             endpointRequest,
             pluginDevice,
@@ -392,21 +325,15 @@ export class ScryptedRuntime {
             pluginHost.io.handleRequest(req, res);
     }
 
-    async handleRequestEndpoint(req: Request, res: Response, endpointRequest: HttpRequest, pluginHost: PluginHost, pluginDevice: PluginDevice) {
-        try {
-            const handler = this.getDevice<HttpRequestHandler>(pluginDevice._id);
-            if (handler.interfaces.includes(ScryptedInterface.EngineIOHandler) && req.headers.connection === 'upgrade' && req.headers.upgrade?.toLowerCase() === 'websocket') {
-                this.wss.handleUpgrade(req, req.socket, null, ws => {
-                    console.log(ws);
-                });
-            }
-            handler.onRequest(endpointRequest, createResponseInterface(res, pluginHost));
-        }
-        catch (e) {
-            res.status(500);
-            res.send(e.toString());
-            console.error(e);
+    async handleRequestEndpoint(req: Request, res: Response, endpointRequest: HttpRequest, pluginData: HttpPluginData) {
+        const { pluginHost, pluginDevice } = pluginData;
+        const handler = this.getDevice<HttpRequestHandler>(pluginDevice._id);
+        if (handler.interfaces.includes(ScryptedInterface.EngineIOHandler) && req.headers.connection === 'upgrade' && req.headers.upgrade?.toLowerCase() === 'websocket') {
+            this.wss.handleUpgrade(req, req.socket, null, ws => {
+                console.log(ws);
+            });
         }
+        handler.onRequest(endpointRequest, createResponseInterface(res, pluginHost.zip));
     }
 
     killPlugin(plugin: Plugin) {

package/src/scrypted-main.ts

@@ -116,6 +116,10 @@ else {
         const db = level(dbPath);
         await db.open();
 
+        if (process.env.SCRYPTED_RESET_ALL_USERS === 'true') {
+            await db.removeAll(ScryptedUser);
+        }
+
         let certSetting = await db.tryGet(Settings, 'certificate') as Settings;
 
         if (certSetting?.value?.version !== CURRENT_SELF_SIGNED_CERTIFICATE_VERSION) {

package/src/services/plugin.ts

@@ -120,7 +120,7 @@ export class PluginComponent {
         console.log('done updating plugins');
     }
 
-    async getRemoteServicePort(pluginId: string, name: string): Promise<number> {
+    async getRemoteServicePort(pluginId: string, name: string, ...args: any[]): Promise<number> {
         if (name === 'console') {
             const consoleServer = await this.scrypted.plugins[pluginId].consoleServer;
             return consoleServer.readPort;
@@ -129,6 +129,6 @@ export class PluginComponent {
             const consoleServer = await this.scrypted.plugins[pluginId].consoleServer;
             return consoleServer.writePort;
         }
-        return this.scrypted.plugins[pluginId].remote.getServicePort(name);
+        return this.scrypted.plugins[pluginId].remote.getServicePort(name, ...args);
     }
 }

package/src/state.ts

@@ -72,23 +72,15 @@ export class ScryptedStateManager extends EventRegistry {
     }
 
     updateDescriptor(device: PluginDevice) {
-        for (const plugin of Object.values(this.scrypted.plugins)) {
-            plugin.remote?.updateDeviceState(device._id, device.state);
-        }
+        this.notify(device._id, undefined, ScryptedInterface.ScryptedDevice, undefined, device.state, true);
     }
 
     removeDevice(id: string) {
-        for (const plugin of Object.values(this.scrypted.plugins)) {
-            plugin.remote?.updateDeviceState(id, undefined);
-        }
-
         this.notify(undefined, undefined, ScryptedInterface.ScryptedDevice, ScryptedInterfaceProperty.id, id, true);
     }
 
     notifyInterfaceEvent(device: PluginDevice, eventInterface: ScryptedInterface | string, value: any) {
-        const eventTime = Date.now();
-
-        this.notify(device?._id, eventTime, eventInterface, undefined, value, true);
+        this.notify(device?._id, Date.now(), eventInterface, undefined, value, true);
     }
 
     setState(id: string, property: string, value: any) {