@scrypted/server 0.0.107 → 0.0.111

package/src/plugin/plugin-host.ts

@@ -1,15 +1,14 @@
 import { RpcMessage, RpcPeer } from '../rpc';
 import AdmZip from 'adm-zip';
-import { SystemManager, DeviceManager, ScryptedNativeId, Device, EventListenerRegister, EngineIOHandler, ScryptedInterfaceProperty, SystemDeviceState } from '@scrypted/sdk/types'
+import { SystemManager, DeviceManager, ScryptedNativeId, Device, EventListenerRegister, EngineIOHandler, ScryptedInterface, ScryptedInterfaceProperty } from '@scrypted/sdk/types'
 import { ScryptedRuntime } from '../runtime';
 import { Plugin } from '../db-types';
-import io from 'engine.io';
+import io, { Socket } from 'engine.io';
 import { attachPluginRemote, setupPluginRemote } from './plugin-remote';
-import { PluginAPI, PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
+import { PluginAPI, PluginAPIProxy, PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
 import { Logger } from '../logger';
 import { MediaManagerHostImpl, MediaManagerImpl } from './media';
-import { getState } from '../state';
-import WebSocket, { EventEmitter } from 'ws';
+import WebSocket from 'ws';
 import { PassThrough } from 'stream';
 import { Console } from 'console'
 import { sleep } from '../sleep';
@@ -25,6 +24,7 @@ import { ensurePluginVolume } from './plugin-volume';
 import { installOptionalDependencies } from './plugin-npm-dependencies';
 import { ConsoleServer, createConsoleServer } from './plugin-console';
 import { createREPLServer } from './plugin-repl';
+import { LazyRemote } from './plugin-lazy-remote';
 
 export class PluginHost {
     worker: child_process.ChildProcess;
@@ -41,7 +41,6 @@ export class PluginHost {
     api: PluginHostAPI;
     pluginName: string;
     packageJson: any;
-    listener: EventListenerRegister;
     stats: {
         cpuUsage: NodeJS.CpuUsage,
         memoryUsage: NodeJS.MemoryUsage,
@@ -51,7 +50,6 @@ export class PluginHost {
 
     kill() {
         this.killed = true;
-        this.listener.removeListener();
         this.api.removeListeners();
         this.worker.kill();
         this.io.close();
@@ -77,7 +75,6 @@ export class PluginHost {
         return this.pluginName || 'no plugin name';
     }
 
-
     async upsertDevice(upsert: Device) {
         const pi = await this.scrypted.upsertDevice(this.pluginId, upsert, true);
         await this.remote.setNativeId(pi.nativeId, pi._id, pi.storage || {});
@@ -101,6 +98,17 @@ export class PluginHost {
 
         this.io.on('connection', async (socket) => {
             try {
+                try {
+                    if (socket.request.url.indexOf('/api') !== -1) {
+                        await this.createRpcIoPeer(socket);
+                        return;
+                    }
+                }
+                catch (e) {
+                    socket.close();
+                    return;
+                }
+
                 const {
                     endpointRequest,
                     pluginDevice,
@@ -138,8 +146,7 @@ export class PluginHost {
         logger.log('i', `loading ${this.pluginName}`);
         logger.log('i', 'pid ' + this.worker?.pid);
 
-
-        const remotePromise = setupPluginRemote(this.peer, this.api, self.pluginId);
+        const remotePromise = setupPluginRemote(this.peer, this.api, self.pluginId, () => this.scrypted.stateManager.getSystemState());
         const init = (async () => {
             const remote = await remotePromise;
 
@@ -190,19 +197,9 @@ export class PluginHost {
         this.module = init.then(({ module }) => module);
         this.remote = new LazyRemote(remotePromise, init.then(({ remote }) => remote));
 
-        this.listener = scrypted.stateManager.listen((id, eventDetails, eventData) => {
-            if (eventDetails.property) {
-                const device = scrypted.findPluginDeviceById(id);
-                this.remote.notify(id, eventDetails.eventTime, eventDetails.eventInterface, eventDetails.property, device?.state[eventDetails.property], eventDetails.changed);
-            }
-            else {
-                this.remote.notify(id, eventDetails.eventTime, eventDetails.eventInterface, eventDetails.property, eventData, eventDetails.changed);
-            }
-        });
-
         init.catch(e => {
             console.error('plugin failed to load', e);
-            this.listener.removeListener();
+            this.api.removeListeners();
         });
     }
 
@@ -314,6 +311,27 @@ export class PluginHost {
             }
         };
     }
+
+    async createRpcIoPeer(socket: Socket) {
+        let connected = true;
+        const rpcPeer = new RpcPeer(`api/${this.pluginId}`, 'web', (message, reject) => {
+            if (!connected)
+                reject?.(new Error('peer disconnected'));
+            else
+                socket.send(JSON.stringify(message))
+        });
+        socket.on('message', data => rpcPeer.handleMessage(JSON.parse(data as string)));
+        // wrap the host api with a connection specific api that can be torn down on disconnect
+        const api = new PluginAPIProxy(this.api, await this.peer.getParam('mediaManager'));
+        const kill = () => {
+            connected = false;
+            rpcPeer.kill('engine.io connection closed.')
+            api.removeListeners();
+        }
+        socket.on('close', kill);
+        socket.on('error', kill);
+        return setupPluginRemote(rpcPeer, api, null, () => this.scrypted.stateManager.getSystemState());
+    }
 }
 
 export function startPluginRemote() {
@@ -331,13 +349,6 @@ export function startPluginRemote() {
     let api: PluginAPI;
     let pluginId: string;
 
-    function idForNativeId(nativeId: ScryptedNativeId) {
-        if (!deviceManager)
-            return;
-        const ds = deviceManager.getDeviceState(nativeId);
-        return ds?.id;
-    }
-
     const getConsole = (hook: (stdout: PassThrough, stderr: PassThrough) => Promise<void>,
         also?: Console, alsoPrefix?: string) => {
 
@@ -489,7 +500,7 @@ export function startPluginRemote() {
         getPluginConsole,
         getDeviceConsole,
         getMixinConsole,
-        async getServicePort(name) {
+        async getServicePort(name, ...args: any[]) {
             if (name === 'repl') {
                 if (!replPort)
                     throw new Error('REPL unavailable: Plugin not loaded.')
@@ -530,69 +541,3 @@ export function startPluginRemote() {
         });
     })
 }
-
-/**
- * Warning: do not await in any of these methods unless necessary, otherwise
- * execution order of state reporting may fail.
- */
-class LazyRemote implements PluginRemote {
-    remote: PluginRemote;
-
-    constructor(public remotePromise: Promise<PluginRemote>, public remoteReadyPromise: Promise<PluginRemote>) {
-        this.remoteReadyPromise = (async () => {
-            this.remote = await remoteReadyPromise;
-            return this.remote;
-        })();
-    }
-
-    async loadZip(packageJson: any, zipData: Buffer, options?: PluginRemoteLoadZipOptions): Promise<any> {
-        if (!this.remote)
-            await this.remoteReadyPromise;
-        return this.remote.loadZip(packageJson, zipData, options);
-    }
-    async setSystemState(state: { [id: string]: { [property: string]: SystemDeviceState; }; }): Promise<void> {
-        if (!this.remote)
-            await this.remoteReadyPromise;
-        return this.remote.setSystemState(state);
-    }
-    async setNativeId(nativeId: ScryptedNativeId, id: string, storage: { [key: string]: any; }): Promise<void> {
-        if (!this.remote)
-            await this.remoteReadyPromise;
-        return this.remote.setNativeId(nativeId, id, storage);
-    }
-    async updateDeviceState(id: string, state: { [property: string]: SystemDeviceState; }): Promise<void> {
-        try {
-            if (!this.remote)
-                await this.remoteReadyPromise;
-        }
-        catch (e) {
-            return;
-        }
-        return this.remote.updateDeviceState(id, state);
-    }
-    async notify(id: string, eventTime: number, eventInterface: string, property: string, propertyState: SystemDeviceState, changed?: boolean): Promise<void> {
-        try {
-            if (!this.remote)
-                await this.remoteReadyPromise;
-        }
-        catch (e) {
-            return;
-        }
-        return this.remote.notify(id, eventTime, eventInterface, property, propertyState, changed);
-    }
-    async ioEvent(id: string, event: string, message?: any): Promise<void> {
-        if (!this.remote)
-            await this.remoteReadyPromise;
-        return this.remote.ioEvent(id, event, message);
-    }
-    async createDeviceState(id: string, setState: (property: string, value: any) => Promise<void>): Promise<any> {
-        if (!this.remote)
-            await this.remoteReadyPromise;
-        return this.remote.createDeviceState(id, setState);
-    }
-
-    async getServicePort(name: string): Promise<number> {
-        const remote = await this.remotePromise;
-        return remote.getServicePort(name);
-    }
-}

package/src/plugin/plugin-http.ts

@@ -0,0 +1,117 @@
+import { Request, Response, Router } from 'express';
+import bodyParser from 'body-parser';
+import { HttpRequest } from '@scrypted/sdk/types';
+import WebSocket, { Server as WebSocketServer } from "ws";
+import { ServerResponse } from 'http';
+
+export abstract class PluginHttp<T> {
+    wss = new WebSocketServer({ noServer: true });
+
+    constructor(public app: Router) {
+        app.all(['/endpoint/@:owner/:pkg/public/engine.io/*', '/endpoint/:pkg/public/engine.io/*'], (req, res) => {
+            this.endpointHandler(req, res, true, true, this.handleEngineIOEndpoint.bind(this))
+        });
+
+        app.all(['/endpoint/@:owner/:pkg/engine.io/*', '/endpoint/@:owner/:pkg/engine.io/*'], (req, res) => {
+            this.endpointHandler(req, res, false, true, this.handleEngineIOEndpoint.bind(this))
+        });
+
+        // stringify all http endpoints
+        app.all(['/endpoint/@:owner/:pkg/public', '/endpoint/@:owner/:pkg/public/*', '/endpoint/:pkg', '/endpoint/:pkg/*'], bodyParser.text() as any);
+
+        app.all(['/endpoint/@:owner/:pkg/public', '/endpoint/@:owner/:pkg/public/*', '/endpoint/:pkg/public', '/endpoint/:pkg/public/*'], (req, res) => {
+            this.endpointHandler(req, res, true, false, this.handleRequestEndpoint.bind(this))
+        });
+
+        app.all(['/endpoint/@:owner/:pkg', '/endpoint/@:owner/:pkg/*', '/endpoint/:pkg', '/endpoint/:pkg/*'], (req, res) => {
+            this.endpointHandler(req, res, false, false, this.handleRequestEndpoint.bind(this))
+        });
+    }
+    
+    abstract handleEngineIOEndpoint(req: Request, res: ServerResponse, endpointRequest: HttpRequest, pluginData: T): Promise<void>;
+    abstract handleRequestEndpoint(req: Request, res: Response, endpointRequest: HttpRequest, pluginData: T): Promise<void>;
+    abstract getEndpointPluginData(endpoint: string, isUpgrade: boolean, isEngineIOEndpoint: boolean): Promise<T>;
+    abstract handleWebSocket(endpoint: string, httpRequest: HttpRequest, ws: WebSocket, pluginData: T): Promise<void>;
+
+    async endpointHandler(req: Request, res: Response, isPublicEndpoint: boolean, isEngineIOEndpoint: boolean,
+        handler: (req: Request, res: Response, endpointRequest: HttpRequest, pluginData: T) => void) {
+
+        const isUpgrade = req.headers.connection?.toLowerCase() === 'upgrade';
+
+        const end = (code: number, message: string) => {
+            if (isUpgrade) {
+                const socket = res.socket;
+                socket.write(`HTTP/1.1 ${code} ${message}\r\n` +
+                    '\r\n');
+                socket.destroy();
+            }
+            else {
+                res.status(code);
+                res.send(message);
+            }
+        };
+
+        if (!isPublicEndpoint && !res.locals.username) {
+            end(401, 'Not Authorized');
+            return;
+        }
+
+        const { owner, pkg } = req.params;
+        let endpoint = pkg;
+        if (owner)
+            endpoint = `@${owner}/${endpoint}`;
+
+        if (isUpgrade && req.headers.upgrade?.toLowerCase() !== 'websocket') {
+            end(404, 'Not Found');
+            return;
+        }
+
+        const pluginData = await this.getEndpointPluginData(endpoint, isUpgrade, isEngineIOEndpoint);
+        if (!pluginData) {
+            end(404, 'Not Found');
+            return;
+        }
+
+        let rootPath = `/endpoint/${endpoint}`;
+        if (isPublicEndpoint)
+            rootPath += '/public'
+
+        const body = req.body && typeof req.body !== 'string' ? JSON.stringify(req.body) : req.body;
+
+        const httpRequest: HttpRequest = {
+            body,
+            headers: req.headers,
+            method: req.method,
+            rootPath,
+            url: req.url,
+            isPublicEndpoint,
+            username: res.locals.username,
+        };
+
+        if (isEngineIOEndpoint && !isUpgrade && isPublicEndpoint) {
+            res.header("Access-Control-Allow-Origin", '*');
+        }
+
+        if (!isEngineIOEndpoint && isUpgrade) {
+            this.wss.handleUpgrade(req, req.socket, (req as any).upgradeHead, async (ws) => {
+                try {
+                    await this.handleWebSocket(endpoint, httpRequest, ws, pluginData);
+                }
+                catch (e) {
+                    console.error('websocket plugin error', e);
+                    ws.close();
+                }
+            });
+        }
+        else {
+            try {
+                handler(req, res, httpRequest, pluginData);
+            }
+            catch (e) {
+                res.status(500);
+                res.send(e.toString());
+                console.error(e);
+            }
+        }
+    }
+}

package/src/plugin/plugin-lazy-remote.ts

@@ -0,0 +1,70 @@
+import { ScryptedNativeId, SystemDeviceState } from '@scrypted/sdk/types'
+import { PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
+
+/**
+ * This remote is necessary as the host needs to create a remote synchronously
+ * in the constructor and immediately begin queueing commands.
+ * Warning: do not await in any of these methods unless necessary, otherwise
+ * execution order of state reporting may fail.
+ */
+ export class LazyRemote implements PluginRemote {
+    remote: PluginRemote;
+
+    constructor(public remotePromise: Promise<PluginRemote>, public remoteReadyPromise: Promise<PluginRemote>) {
+        this.remoteReadyPromise = (async () => {
+            this.remote = await remoteReadyPromise;
+            return this.remote;
+        })();
+    }
+
+    async loadZip(packageJson: any, zipData: Buffer, options?: PluginRemoteLoadZipOptions): Promise<any> {
+        if (!this.remote)
+            await this.remoteReadyPromise;
+        return this.remote.loadZip(packageJson, zipData, options);
+    }
+    async setSystemState(state: { [id: string]: { [property: string]: SystemDeviceState; }; }): Promise<void> {
+        if (!this.remote)
+            await this.remoteReadyPromise;
+        return this.remote.setSystemState(state);
+    }
+    async setNativeId(nativeId: ScryptedNativeId, id: string, storage: { [key: string]: any; }): Promise<void> {
+        if (!this.remote)
+            await this.remoteReadyPromise;
+        return this.remote.setNativeId(nativeId, id, storage);
+    }
+    async updateDeviceState(id: string, state: { [property: string]: SystemDeviceState; }): Promise<void> {
+        try {
+            if (!this.remote)
+                await this.remoteReadyPromise;
+        }
+        catch (e) {
+            return;
+        }
+        return this.remote.updateDeviceState(id, state);
+    }
+    async notify(id: string, eventTime: number, eventInterface: string, property: string, propertyState: SystemDeviceState, changed?: boolean): Promise<void> {
+        try {
+            if (!this.remote)
+                await this.remoteReadyPromise;
+        }
+        catch (e) {
+            return;
+        }
+        return this.remote.notify(id, eventTime, eventInterface, property, propertyState, changed);
+    }
+    async ioEvent(id: string, event: string, message?: any): Promise<void> {
+        if (!this.remote)
+            await this.remoteReadyPromise;
+        return this.remote.ioEvent(id, event, message);
+    }
+    async createDeviceState(id: string, setState: (property: string, value: any) => Promise<void>): Promise<any> {
+        if (!this.remote)
+            await this.remoteReadyPromise;
+        return this.remote.createDeviceState(id, setState);
+    }
+
+    async getServicePort(name: string, ...args: any[]): Promise<number> {
+        const remote = await this.remotePromise;
+        return remote.getServicePort(name, ...args);
+    }
+}

package/src/plugin/plugin-npm-dependencies.ts

@@ -3,14 +3,18 @@ import fs from 'fs';
 import child_process from 'child_process';
 import path from 'path';
 import { once } from 'events';
+import process from 'process';
+import mkdirp from "mkdirp";
 
 export async function installOptionalDependencies(console: Console, packageJson: any) {
     const pluginVolume = ensurePluginVolume(packageJson.name);
-    const optPj = path.join(pluginVolume, 'package.json');
+    const nodePrefix = path.join(pluginVolume, `${process.platform}-${process.arch}`);
+    const packageJsonPath = path.join(nodePrefix, 'package.json');
+    const currentInstalledPackageJsonPath = path.join(nodePrefix, 'package.installed.json');
 
     let currentPackageJson: any;
     try {
-        currentPackageJson = JSON.parse(fs.readFileSync(optPj).toString());
+        currentPackageJson = JSON.parse(fs.readFileSync(currentInstalledPackageJsonPath).toString());
     }
     catch (e) {
     }
@@ -34,21 +38,18 @@ export async function installOptionalDependencies(console: Console, packageJson:
     delete reduced.optionalDependencies;
     delete reduced.devDependencies;
 
-    try {
-        fs.writeFileSync(optPj, JSON.stringify(reduced));
-
-        const cp = child_process.spawn('npm', ['--prefix', pluginVolume, 'install'], {
-            cwd: pluginVolume,
-            stdio: 'inherit',
-        });
-    
-        await once(cp, 'exit');
-        if (cp.exitCode !== 0)
-            throw new Error('npm installation failed with exit code ' + cp.exitCode);
-    }
-    catch (e) {
-        fs.rmSync(optPj);
-        throw e;
-    }
+    mkdirp.sync(nodePrefix);
+    fs.writeFileSync(packageJsonPath, JSON.stringify(reduced));
+
+    const cp = child_process.spawn('npm', ['--prefix', nodePrefix, 'install'], {
+        cwd: nodePrefix,
+        stdio: 'inherit',
+    });
+
+    await once(cp, 'exit');
+    if (cp.exitCode !== 0)
+        throw new Error('npm installation failed with exit code ' + cp.exitCode);
+
+    fs.writeFileSync(currentInstalledPackageJsonPath, JSON.stringify(reduced));
     console.log('native dependencies installed.');
 }

package/src/plugin/plugin-remote-websocket.ts

@@ -2,7 +2,7 @@ interface WebSocketEvent {
     type: string;
     reason?: string;
     message?: string;
-    data?: string|ArrayBufferLike;
+    data?: string | ArrayBufferLike;
     source?: any;
 }
 
@@ -12,7 +12,7 @@ interface WebSocketEventListener {
 
 // @ts-ignore
 class WebSocketEventTarget {
-    events: { [type: string]: WebSocketEventListener[]} = {};
+    events: { [type: string]: WebSocketEventListener[] } = {};
 
     dispatchEvent(event: WebSocketEvent) {
         const list = this.events[event.type];
@@ -53,32 +53,20 @@ function defineEventAttribute(p: any, type: string) {
     });
 }
 
-interface WebSocketEndCallback {
-    (): void;
+export interface WebSocketConnectCallbacks {
+    connect(e: Error, ws: WebSocketMethods): void;
+    end(): void;
+    error(e: Error): void;
+    data(data: string | ArrayBufferLike): void;
 }
 
-interface WebSocketErrorCallback {
-    (e: Error): void;
+export interface WebSocketConnect {
+    (url: string, protocols: string[], callbacks: WebSocketConnectCallbacks): void;
 }
 
-interface WebSocketDataCallback {
-    (data: string | ArrayBufferLike): void;
-}
-
-interface WebSocketSend {
-    (message: string|ArrayBufferLike): void;
-}
-
-interface WebSocketConnectCallback {
-    (e: Error, ws: any, send: WebSocketSend): void;
-}
-
-interface WebSocketConnect {
-    (url: string, protocols: string[],
-        connect: WebSocketConnectCallback,
-        end: WebSocketEndCallback,
-        error: WebSocketErrorCallback,
-        data: WebSocketDataCallback): void;
+export interface WebSocketMethods {
+    send(message: string | ArrayBufferLike): void;
+    close(message: string): void;
 }
 
 export function createWebSocketClass(__websocketConnect: WebSocketConnect) {
@@ -88,8 +76,7 @@ export function createWebSocketClass(__websocketConnect: WebSocketConnect) {
         _url: string;
         _protocols: string[];
         readyState: number;
-        send: (message: string|ArrayBufferLike) => void;
-        _ws: any;
+        _ws: WebSocketMethods;
 
         constructor(url: string, protocols?: string[]) {
             super();
@@ -97,44 +84,52 @@ export function createWebSocketClass(__websocketConnect: WebSocketConnect) {
             this._protocols = protocols;
             this.readyState = 0;
 
-            __websocketConnect(url, protocols, (e, ws, send) => {
-                // connect
-                if (e != null) {
+            __websocketConnect(url, protocols, {
+                connect: (e, ws) => {
+                    // connect
+                    if (e != null) {
+                        this.dispatchEvent({
+                            type: 'error',
+                            message: e.toString(),
+                        });
+                        return;
+                    }
+
+                    this._ws = ws;
+                    this.readyState = 1;
+                    this.dispatchEvent({
+                        type: 'open',
+                    });
+                },
+                end: () => {
+                    // end
+                    this.readyState = 3;
+                    this.dispatchEvent({
+                        type: 'close',
+                        reason: 'closed',
+                    });
+                },
+                error: (e: Error) => {
+                    // error
+                    this.readyState = 3;
                     this.dispatchEvent({
                         type: 'error',
                         message: e.toString(),
                     });
-                    return;
+                },
+                data: (data: string | ArrayBufferLike) => {
+                    // data
+                    this.dispatchEvent({
+                        type: 'message',
+                        data: data,
+                        source: this,
+                    });
                 }
+            })
+        }
 
-                this._ws = ws;
-                this.send = send;
-                this.readyState = 1;
-                this.dispatchEvent({
-                    type: 'open',
-                });
-            }, () => {
-                // end
-                this.readyState = 3;
-                this.dispatchEvent({
-                    type: 'close',
-                    reason: 'closed',
-                });
-            }, (e: Error) => {
-                // error
-                this.readyState = 3;
-                this.dispatchEvent({
-                    type: 'error',
-                    message: e.toString(),
-                });
-            }, (data: string | ArrayBufferLike) => {
-                // data
-                this.dispatchEvent({
-                    type: 'message',
-                    data: data,
-                    source: this,
-                });
-            });
+        send(message: string | ArrayBufferLike) {
+            this._ws.send(message);
         }
 
         get url() {
@@ -145,8 +140,8 @@ export function createWebSocketClass(__websocketConnect: WebSocketConnect) {
             return "";
         }
 
-        close() {
-            this._ws.close();
+        close(reason: string) {
+            this._ws.close(reason);
         }
     }