@scrypted/server 0.0.105 → 0.0.109

package/src/rpc.ts

@@ -1,5 +1,7 @@
 import vm from 'vm';
 
+const finalizerIdSymbol = Symbol('rpcFinalizerId');
+
 function getDefaultTransportSafeArgumentTypes() {
     const jsonSerializable = new Set<string>();
     jsonSerializable.add(Number.name);
@@ -40,6 +42,7 @@ interface RpcOob extends RpcMessage {
 
 interface RpcRemoteProxyValue {
     __remote_proxy_id: string;
+    __remote_proxy_finalizer_id: string;
     __remote_constructor_name: string;
     __remote_proxy_props: any;
     __remote_proxy_oneway_methods: string[];
@@ -52,6 +55,7 @@ interface RpcLocalProxyValue {
 
 interface RpcFinalize extends RpcMessage {
     __local_proxy_id: string;
+    __local_proxy_finalizer_id: string;
 }
 
 interface Deferred {
@@ -78,20 +82,17 @@ export const PROPERTY_PROXY_PROPERTIES = '__proxy_props';
 export const PROPERTY_JSON_COPY_SERIALIZE_CHILDREN = '__json_copy_serialize_children';
 
 class RpcProxy implements ProxyHandler<any> {
+
     constructor(public peer: RpcPeer,
-        public id: string,
+        public entry: LocalProxiedEntry,
         public constructorName: string,
         public proxyProps: any,
         public proxyOneWayMethods: string[]) {
-        this.peer = peer;
-        this.id = id;
-        this.constructorName = constructorName;
-        this.proxyProps = proxyProps;
     }
 
     get(target: any, p: PropertyKey, receiver: any): any {
         if (p === '__proxy_id')
-            return this.id;
+            return this.entry.id;
         if (p === '__proxy_constructor')
             return this.constructorName;
         if (p === '__proxy_peer')
@@ -114,6 +115,12 @@ class RpcProxy implements ProxyHandler<any> {
         return new Proxy(() => p, this);
     }
 
+    set(target: any, p: string | symbol, value: any, receiver: any): boolean {
+        if (p === finalizerIdSymbol)
+            this.entry.finalizerId = value;
+        return true;
+    }
+
     apply(target: any, thisArg: any, argArray?: any): any {
         const method = target();
         const args: any[] = [];
@@ -124,7 +131,7 @@ class RpcProxy implements ProxyHandler<any> {
         const rpcApply: RpcApply = {
             type: "apply",
             id: undefined,
-            proxyId: this.id,
+            proxyId: this.entry.id,
             args,
             method,
         };
@@ -187,16 +194,21 @@ export interface RpcSerializer {
     deserialize(serialized: any): any;
 }
 
+interface LocalProxiedEntry {
+    id: string;
+    finalizerId: string;
+}
+
 export class RpcPeer {
     idCounter = 1;
     onOob: (oob: any) => void;
     params: { [name: string]: any } = {};
     pendingResults: { [id: string]: Deferred } = {};
     proxyCounter = 1;
-    localProxied = new Map<any, string>();
+    localProxied = new Map<any, LocalProxiedEntry>();
     localProxyMap: { [id: string]: any } = {};
     remoteWeakProxies: { [id: string]: WeakRef<any> } = {};
-    finalizers = new FinalizationRegistry(id => this.finalize(id as string));
+    finalizers = new FinalizationRegistry(entry => this.finalize(entry as LocalProxiedEntry));
     nameDeserializerMap = new Map<string, RpcSerializer>();
     constructorSerializerMap = new Map<string, string>();
     transportSafeArgumentTypes = getDefaultTransportSafeArgumentTypes();
@@ -238,10 +250,11 @@ export class RpcPeer {
         this.constructorSerializerMap.set(ctr, name);
     }
 
-    finalize(id: string) {
-        delete this.remoteWeakProxies[id];
+    finalize(entry: LocalProxiedEntry) {
+        delete this.remoteWeakProxies[entry.id];
         const rpcFinalize: RpcFinalize = {
-            __local_proxy_id: id,
+            __local_proxy_id: entry.id,
+            __local_proxy_finalizer_id: entry.finalizerId,
             type: 'finalize',
         }
         this.send(rpcFinalize);
@@ -294,9 +307,12 @@ export class RpcPeer {
             return ret;
         }
 
-        const { __remote_proxy_id, __local_proxy_id, __remote_constructor_name, __serialized_value, __remote_proxy_props, __remote_proxy_oneway_methods } = value;
+        const { __remote_proxy_id, __remote_proxy_finalizer_id, __local_proxy_id, __remote_constructor_name, __serialized_value, __remote_proxy_props, __remote_proxy_oneway_methods } = value;
         if (__remote_proxy_id) {
-            const proxy = this.remoteWeakProxies[__remote_proxy_id]?.deref() || this.newProxy(__remote_proxy_id, __remote_constructor_name, __remote_proxy_props, __remote_proxy_oneway_methods);
+            let proxy = this.remoteWeakProxies[__remote_proxy_id]?.deref();
+            if (!proxy)
+                proxy = this.newProxy(__remote_proxy_id, __remote_constructor_name, __remote_proxy_props, __remote_proxy_oneway_methods);
+            proxy[finalizerIdSymbol] = __remote_proxy_finalizer_id;
             return proxy;
         }
 
@@ -329,10 +345,13 @@ export class RpcPeer {
 
         let __remote_constructor_name = value.__proxy_constructor || value.constructor?.name?.toString();
 
-        let proxyId = this.localProxied.get(value);
-        if (proxyId) {
+        let proxiedEntry = this.localProxied.get(value);
+        if (proxiedEntry) {
+            const __remote_proxy_finalizer_id = (this.proxyCounter++).toString();
+            proxiedEntry.finalizerId = __remote_proxy_finalizer_id;
             const ret: RpcRemoteProxyValue = {
-                __remote_proxy_id: proxyId,
+                __remote_proxy_id: proxiedEntry.id,
+                __remote_proxy_finalizer_id,
                 __remote_constructor_name,
                 __remote_proxy_props: value?.[PROPERTY_PROXY_PROPERTIES],
                 __remote_proxy_oneway_methods: value?.[PROPERTY_PROXY_ONEWAY_METHODS],
@@ -355,6 +374,7 @@ export class RpcPeer {
             const serialized = serializer.serialize(value);
             const ret: RpcRemoteProxyValue = {
                 __remote_proxy_id: undefined,
+                __remote_proxy_finalizer_id: undefined,
                 __remote_constructor_name,
                 __remote_proxy_props: value?.[PROPERTY_PROXY_PROPERTIES],
                 __remote_proxy_oneway_methods: value?.[PROPERTY_PROXY_ONEWAY_METHODS],
@@ -363,12 +383,17 @@ export class RpcPeer {
             return ret;
         }
 
-        proxyId = (this.proxyCounter++).toString();
-        this.localProxied.set(value, proxyId);
-        this.localProxyMap[proxyId] = value;
+        const __remote_proxy_id = (this.proxyCounter++).toString();
+        proxiedEntry = {
+            id: __remote_proxy_id,
+            finalizerId: __remote_proxy_id,
+        };
+        this.localProxied.set(value, proxiedEntry);
+        this.localProxyMap[__remote_proxy_id] = value;
 
         const ret: RpcRemoteProxyValue = {
-            __remote_proxy_id: proxyId,
+            __remote_proxy_id,
+            __remote_proxy_finalizer_id: __remote_proxy_id,
             __remote_constructor_name,
             __remote_proxy_props: value?.[PROPERTY_PROXY_PROPERTIES],
             __remote_proxy_oneway_methods: value?.[PROPERTY_PROXY_ONEWAY_METHODS],
@@ -378,12 +403,16 @@ export class RpcPeer {
     }
 
     newProxy(proxyId: string, proxyConstructorName: string, proxyProps: any, proxyOneWayMethods: string[]) {
-        const rpc = new RpcProxy(this, proxyId, proxyConstructorName, proxyProps, proxyOneWayMethods);
+        const localProxiedEntry: LocalProxiedEntry = {
+            id: proxyId,
+            finalizerId: undefined,
+        }
+        const rpc = new RpcProxy(this, localProxiedEntry, proxyConstructorName, proxyProps, proxyOneWayMethods);
         const target = proxyConstructorName === 'Function' || proxyConstructorName === 'AsyncFunction' ? function () { } : rpc;
         const proxy = new Proxy(target, rpc);
         const weakref = new WeakRef(proxy);
         this.remoteWeakProxies[proxyId] = weakref;
-        this.finalizers.register(rpc, proxyId);
+        this.finalizers.register(rpc, localProxiedEntry);
         global.gc?.();
         return proxy;
     }
@@ -460,8 +489,16 @@ export class RpcPeer {
                 case 'finalize': {
                     const rpcFinalize = message as RpcFinalize;
                     const local = this.localProxyMap[rpcFinalize.__local_proxy_id];
-                    delete this.localProxyMap[rpcFinalize.__local_proxy_id];
-                    this.localProxied.delete(local);
+                    if (local) {
+                        const localProxiedEntry = this.localProxied.get(local);
+                        // if a finalizer id is specified, it must match.
+                        if (rpcFinalize.__local_proxy_finalizer_id && rpcFinalize.__local_proxy_finalizer_id !== localProxiedEntry?.finalizerId) {
+                            console.error(this.selfName, this.peerName, 'finalizer mismatch')
+                            break;
+                        }
+                        delete this.localProxyMap[rpcFinalize.__local_proxy_id];
+                        this.localProxied.delete(local);
+                    }
                     break;
                 }
                 case 'oob': {

package/src/runtime.ts

@@ -4,9 +4,8 @@ import { ScryptedNativeId, Device, EngineIOHandler, HttpRequest, HttpRequestHand
 import { PluginDeviceProxyHandler } from './plugin/plugin-device';
 import { Plugin, PluginDevice, ScryptedAlert } from './db-types';
 import { getState, ScryptedStateManager, setState } from './state';
-import { Request, Response, Router } from 'express';
+import { Request, Response } from 'express';
 import { createResponseInterface } from './http-interfaces';
-import bodyParser from 'body-parser';
 import http, { ServerResponse } from 'http';
 import https from 'https';
 import express from 'express';
@@ -15,7 +14,7 @@ import { getDisplayName, getDisplayRoom, getDisplayType, getProvidedNameOrDefaul
 import { URL } from "url";
 import qs from "query-string";
 import { PluginComponent } from './services/plugin';
-import { Server as WebSocketServer } from "ws";
+import WebSocket, { Server as WebSocketServer } from "ws";
 import axios from 'axios';
 import tar from 'tar';
 import { once } from 'events';
@@ -30,6 +29,7 @@ import io from 'engine.io';
 import { spawn as ptySpawn } from 'node-pty';
 import rimraf from 'rimraf';
 import { getPluginVolume } from './plugin/plugin-volume';
+import { PluginHttp } from './plugin/plugin-http';
 
 interface DeviceProxyPair {
     handler: PluginDeviceProxyHandler;
@@ -39,13 +39,17 @@ interface DeviceProxyPair {
 const MIN_SCRYPTED_CORE_VERSION = 'v0.0.146';
 const PLUGIN_DEVICE_STATE_VERSION = 2;
 
-export class ScryptedRuntime {
+interface HttpPluginData {
+    pluginHost: PluginHost;
+    pluginDevice: PluginDevice
+}
+
+export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
     datastore: Level;
     plugins: { [id: string]: PluginHost } = {};
     pluginDevices: { [id: string]: PluginDevice } = {};
     devices: { [id: string]: DeviceProxyPair } = {};
     stateManager = new ScryptedStateManager(this);
-    app: Router;
     logger = new Logger(this, '', 'Scrypted');
     devicesLogger = this.logger.getLogger('device', 'Devices');
     wss = new WebSocketServer({ noServer: true });
@@ -55,30 +59,12 @@ export class ScryptedRuntime {
     });
 
     constructor(datastore: Level, insecure: http.Server, secure: https.Server, app: express.Application) {
+        super(app);
         this.datastore = datastore;
         this.app = app;
 
         app.disable('x-powered-by');
 
-        app.all(['/endpoint/@:owner/:pkg/public/engine.io/*', '/endpoint/:pkg/public/engine.io/*'], (req, res) => {
-            this.endpointHandler(req, res, true, true, this.handleEngineIOEndpoint.bind(this))
-        });
-
-        app.all(['/endpoint/@:owner/:pkg/engine.io/*', '/endpoint/@:owner/:pkg/engine.io/*'], (req, res) => {
-            this.endpointHandler(req, res, false, true, this.handleEngineIOEndpoint.bind(this))
-        });
-
-        // stringify all http endpoints
-        app.all(['/endpoint/@:owner/:pkg/public', '/endpoint/@:owner/:pkg/public/*', '/endpoint/:pkg', '/endpoint/:pkg/*'], bodyParser.text() as any);
-
-        app.all(['/endpoint/@:owner/:pkg/public', '/endpoint/@:owner/:pkg/public/*', '/endpoint/:pkg/public', '/endpoint/:pkg/public/*'], (req, res) => {
-            this.endpointHandler(req, res, true, false, this.handleRequestEndpoint.bind(this))
-        });
-
-        app.all(['/endpoint/@:owner/:pkg', '/endpoint/@:owner/:pkg/*', '/endpoint/:pkg', '/endpoint/:pkg/*'], (req, res) => {
-            this.endpointHandler(req, res, false, false, this.handleRequestEndpoint.bind(this))
-        });
-
         app.get('/web/oauth/callback', (req, res) => {
             this.oauthCallback(req, res);
         });
@@ -190,7 +176,7 @@ export class ScryptedRuntime {
         }
     }
 
-    async getPluginForEndpoint(endpoint: string) {
+    async getPluginForEndpoint(endpoint: string): Promise<HttpPluginData> {
         let pluginHost = this.plugins[endpoint] ?? this.getPluginHostForDeviceId(endpoint);
         if (endpoint === '@scrypted/core') {
             // enforce a minimum version on @scrypted/core
@@ -254,110 +240,55 @@ export class ScryptedRuntime {
             this.shellio.handleRequest(req, res);
     }
 
-    async endpointHandler(req: Request, res: Response, isPublicEndpoint: boolean, isEngineIOEndpoint: boolean,
-        handler: (req: Request, res: Response, endpointRequest: HttpRequest, pluginHost: PluginHost, pluginDevice: PluginDevice) => void) {
-
-        const isUpgrade = req.headers.connection?.toLowerCase() === 'upgrade';
-
-        const end = (code: number, message: string) => {
-            if (isUpgrade) {
-                const socket = res.socket;
-                socket.write(`HTTP/1.1 ${code} ${message}\r\n` +
-                    '\r\n');
-                socket.destroy();
-            }
-            else {
-                res.status(code);
-                res.send(message);
-            }
-        };
-
-        if (!isPublicEndpoint && !res.locals.username) {
-            end(401, 'Not Authorized');
-            return;
-        }
-
-        const { owner, pkg } = req.params;
-        let endpoint = pkg;
-        if (owner)
-            endpoint = `@${owner}/${endpoint}`;
-
-        const { pluginHost, pluginDevice } = await this.getPluginForEndpoint(endpoint);
+    async getEndpointPluginData(endpoint: string, isUpgrade: boolean, isEngineIOEndpoint: boolean): Promise<HttpPluginData> {
+        const ret = await this.getPluginForEndpoint(endpoint);
+        const { pluginDevice } = ret;
 
         // check if upgrade requests can be handled. must be websocket.
         if (isUpgrade) {
-            if (req.headers.upgrade?.toLowerCase() !== 'websocket' || !pluginDevice?.state.interfaces.value.includes(ScryptedInterface.EngineIOHandler)) {
-                end(404, 'Not Found');
+            if (!pluginDevice?.state.interfaces.value.includes(ScryptedInterface.EngineIOHandler)) {
                 return;
             }
         }
         else {
             if (!isEngineIOEndpoint && !pluginDevice?.state.interfaces.value.includes(ScryptedInterface.HttpRequestHandler)) {
-                end(404, 'Not Found');
                 return;
             }
         }
 
-        let rootPath = `/endpoint/${endpoint}`;
-        if (isPublicEndpoint)
-            rootPath += '/public'
-
-        const body = req.body && typeof req.body !== 'string' ? JSON.stringify(req.body) : req.body;
+        return ret;
+    }
 
-        const httpRequest: HttpRequest = {
-            body,
-            headers: req.headers,
-            method: req.method,
-            rootPath,
-            url: req.url,
-            isPublicEndpoint,
-            username: res.locals.username,
-        };
+    async handleWebSocket(endpoint: string, httpRequest: HttpRequest, ws: WebSocket, pluginData: HttpPluginData): Promise<void> {
+        const { pluginDevice } = pluginData;
 
-        if (isEngineIOEndpoint && !isUpgrade && isPublicEndpoint) {
-            res.header("Access-Control-Allow-Origin", '*');
+        const handler = this.getDevice<EngineIOHandler>(pluginDevice._id);
+        const id = 'ws-' + this.wsAtomic++;
+        const pluginHost = this.plugins[endpoint] ?? this.getPluginHostForDeviceId(endpoint);
+        if (!pluginHost) {
+            ws.close();
+            return;
         }
+        pluginHost.ws[id] = ws;
 
-        if (!isEngineIOEndpoint && isUpgrade) {
-            this.wss.handleUpgrade(req, req.socket, (req as any).upgradeHead, async (ws) => {
-                try {
-                    const handler = this.getDevice<EngineIOHandler>(pluginDevice._id);
-                    const id = 'ws-' + this.wsAtomic++;
-                    const pluginHost = this.plugins[endpoint] ?? this.getPluginHostForDeviceId(endpoint);
-                    if (!pluginHost) {
-                        ws.close();
-                        return;
-                    }
-                    pluginHost.ws[id] = ws;
-
-                    ws.on('message', async (message) => {
-                        try {
-                            pluginHost.remote.ioEvent(id, 'message', message)
-                        }
-                        catch (e) {
-                            ws.close();
-                        }
-                    });
-                    ws.on('close', async (reason) => {
-                        try {
-                            pluginHost.remote.ioEvent(id, 'close');
-                        }
-                        catch (e) {
-                        }
-                        delete pluginHost.ws[id];
-                    });
-
-                    await handler.onConnection(httpRequest, `ws://${id}`);
-                }
-                catch (e) {
-                    console.error('websocket plugin error', e);
-                    ws.close();
-                }
-            });
-        }
-        else {
-            handler(req, res, httpRequest, pluginHost, pluginDevice);
-        }
+        ws.on('message', async (message) => {
+            try {
+                pluginHost.remote.ioEvent(id, 'message', message)
+            }
+            catch (e) {
+                ws.close();
+            }
+        });
+        ws.on('close', async (reason) => {
+            try {
+                pluginHost.remote.ioEvent(id, 'close');
+            }
+            catch (e) {
+            }
+            delete pluginHost.ws[id];
+        });
+
+        await handler.onConnection(httpRequest, `ws://${id}`);
     }
 
     async getComponent(componentId: string): Promise<any> {
@@ -381,7 +312,9 @@ export class ScryptedRuntime {
         }
     }
 
-    async handleEngineIOEndpoint(req: Request, res: ServerResponse, endpointRequest: HttpRequest, pluginHost: PluginHost, pluginDevice: PluginDevice) {
+    async handleEngineIOEndpoint(req: Request, res: ServerResponse, endpointRequest: HttpRequest, pluginData: HttpPluginData) {
+        const { pluginHost, pluginDevice } = pluginData;
+
         (req as any).scrypted = {
             endpointRequest,
             pluginDevice,
@@ -392,21 +325,15 @@ export class ScryptedRuntime {
             pluginHost.io.handleRequest(req, res);
     }
 
-    async handleRequestEndpoint(req: Request, res: Response, endpointRequest: HttpRequest, pluginHost: PluginHost, pluginDevice: PluginDevice) {
-        try {
-            const handler = this.getDevice<HttpRequestHandler>(pluginDevice._id);
-            if (handler.interfaces.includes(ScryptedInterface.EngineIOHandler) && req.headers.connection === 'upgrade' && req.headers.upgrade?.toLowerCase() === 'websocket') {
-                this.wss.handleUpgrade(req, req.socket, null, ws => {
-                    console.log(ws);
-                });
-            }
-            handler.onRequest(endpointRequest, createResponseInterface(res, pluginHost));
-        }
-        catch (e) {
-            res.status(500);
-            res.send(e.toString());
-            console.error(e);
+    async handleRequestEndpoint(req: Request, res: Response, endpointRequest: HttpRequest, pluginData: HttpPluginData) {
+        const { pluginHost, pluginDevice } = pluginData;
+        const handler = this.getDevice<HttpRequestHandler>(pluginDevice._id);
+        if (handler.interfaces.includes(ScryptedInterface.EngineIOHandler) && req.headers.connection === 'upgrade' && req.headers.upgrade?.toLowerCase() === 'websocket') {
+            this.wss.handleUpgrade(req, req.socket, null, ws => {
+                console.log(ws);
+            });
         }
+        handler.onRequest(endpointRequest, createResponseInterface(res, pluginHost.zip));
     }
 
     killPlugin(plugin: Plugin) {

package/src/scrypted-main.ts

@@ -116,6 +116,10 @@ else {
         const db = level(dbPath);
         await db.open();
 
+        if (process.env.SCRYPTED_RESET_ALL_USERS === 'true') {
+            await db.removeAll(ScryptedUser);
+        }
+
         let certSetting = await db.tryGet(Settings, 'certificate') as Settings;
 
         if (certSetting?.value?.version !== CURRENT_SELF_SIGNED_CERTIFICATE_VERSION) {

package/src/services/plugin.ts

@@ -120,7 +120,7 @@ export class PluginComponent {
         console.log('done updating plugins');
     }
 
-    async getRemoteServicePort(pluginId: string, name: string): Promise<number> {
+    async getRemoteServicePort(pluginId: string, name: string, ...args: any[]): Promise<number> {
         if (name === 'console') {
             const consoleServer = await this.scrypted.plugins[pluginId].consoleServer;
             return consoleServer.readPort;
@@ -129,6 +129,6 @@ export class PluginComponent {
             const consoleServer = await this.scrypted.plugins[pluginId].consoleServer;
             return consoleServer.writePort;
         }
-        return this.scrypted.plugins[pluginId].remote.getServicePort(name);
+        return this.scrypted.plugins[pluginId].remote.getServicePort(name, ...args);
     }
 }

package/src/state.ts

@@ -72,23 +72,15 @@ export class ScryptedStateManager extends EventRegistry {
     }
 
     updateDescriptor(device: PluginDevice) {
-        for (const plugin of Object.values(this.scrypted.plugins)) {
-            plugin.remote?.updateDeviceState(device._id, device.state);
-        }
+        this.notify(device._id, undefined, ScryptedInterface.ScryptedDevice, undefined, device.state, true);
     }
 
     removeDevice(id: string) {
-        for (const plugin of Object.values(this.scrypted.plugins)) {
-            plugin.remote?.updateDeviceState(id, undefined);
-        }
-
         this.notify(undefined, undefined, ScryptedInterface.ScryptedDevice, ScryptedInterfaceProperty.id, id, true);
     }
 
     notifyInterfaceEvent(device: PluginDevice, eventInterface: ScryptedInterface | string, value: any) {
-        const eventTime = Date.now();
-
-        this.notify(device?._id, eventTime, eventInterface, undefined, value, true);
+        this.notify(device?._id, Date.now(), eventInterface, undefined, value, true);
     }
 
     setState(id: string, property: string, value: any) {