@scrypted/server 0.0.181 → 0.1.1

package/src/plugin/plugin-host.ts

@@ -1,17 +1,19 @@
 import { Device, EngineIOHandler } from '@scrypted/types';
 import AdmZip from 'adm-zip';
 import crypto from 'crypto';
-import io, { Socket } from 'engine.io';
+import * as io from 'engine.io';
 import fs from 'fs';
 import mkdirp from 'mkdirp';
 import path from 'path';
 import rimraf from 'rimraf';
-import WebSocket from 'ws';
+import WebSocket, { once } from 'ws';
 import { Plugin } from '../db-types';
+import { IOServer, IOServerSocket } from '../io';
 import { Logger } from '../logger';
 import { RpcPeer } from '../rpc';
 import { ScryptedRuntime } from '../runtime';
 import { sleep } from '../sleep';
+import { SidebandBufferSerializer } from './buffer-serializer';
 import { MediaManagerHostImpl } from './media';
 import { PluginAPIProxy, PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
 import { ConsoleServer, createConsoleServer } from './plugin-console';
@@ -34,8 +36,16 @@ export class PluginHost {
     module: Promise<any>;
     scrypted: ScryptedRuntime;
     remote: PluginRemote;
-    io = io(undefined, {
+    io: IOServer = new io.Server({
         pingTimeout: 120000,
+        perMessageDeflate: true,
+        cors: (req, callback) => {
+            const header = this.scrypted.getAccessControlAllowOrigin(req.headers);
+            callback(undefined, {
+                origin: header,
+                credentials: true,
+            })
+        },
     });
     ws: { [id: string]: WebSocket } = {};
     api: PluginHostAPI;
@@ -123,6 +133,26 @@ export class PluginHost {
             try {
                 try {
                     if (socket.request.url.indexOf('/api') !== -1) {
+                        if (socket.request.url.indexOf('/public') !== -1) {
+                            socket.send(JSON.stringify({
+                                // @ts-expect-error
+                                id: socket.id,
+                            }));
+                            const timeout = new Promise((_, rj) => setTimeout(() => rj(new Error('timeout')), 10000));
+                            try {
+                                await Promise.race([
+                                    once(socket, '/api/activate'),
+                                    timeout,
+                                ]);
+                                // client will send a start request when it's ready to process events.
+                                await once(socket, 'message');
+                            }
+                            catch (e) {
+                                socket.close();
+                                return;
+                            }
+                        }
+
                         await this.createRpcIoPeer(socket);
                         return;
                     }
@@ -139,14 +169,17 @@ export class PluginHost {
 
                 const handler = this.scrypted.getDevice<EngineIOHandler>(pluginDevice._id);
 
+                // @ts-expect-error
+                const id = socket.id;
+
                 socket.on('message', message => {
-                    this.remote.ioEvent(socket.id, 'message', message)
+                    this.remote.ioEvent(id, 'message', message)
                 });
                 socket.on('close', reason => {
-                    this.remote.ioEvent(socket.id, 'close');
+                    this.remote.ioEvent(id, 'close');
                 });
 
-                await handler.onConnection(endpointRequest, `io://${socket.id}`);
+                await handler.onConnection(endpointRequest, `io://${id}`);
             }
             catch (e) {
                 console.error('engine.io plugin error', e);
@@ -311,22 +344,40 @@ export class PluginHost {
             disconnect();
         });
 
-        this.peer.onOob = (oob: any) => {
-            if (oob.type === 'stats') {
-                this.stats = oob;
-            }
-        };
+        this.peer.params.updateStats = (stats: any) => {
+            this.stats = stats;
+        }
     }
 
-    async createRpcIoPeer(socket: Socket) {
+    async createRpcIoPeer(socket: IOServerSocket) {
         let connected = true;
-        const rpcPeer = new RpcPeer(`api/${this.pluginId}`, 'web', (message, reject) => {
-            if (!connected)
+        const rpcPeer = new RpcPeer(`api/${this.pluginId}`, 'web', (message, reject, serializationContext) => {
+            if (!connected) {
                 reject?.(new Error('peer disconnected'));
-            else
-                socket.send(JSON.stringify(message))
+                return;
+            }
+            const buffers = serializationContext?.buffers;
+            if (buffers) {
+                for (const buffer of buffers) {
+                    socket.send(buffer);
+                }
+            }
+            socket.send(JSON.stringify(message))
+        });
+        let pendingSerializationContext: any = {};
+        socket.on('message', data => {
+            if (data.constructor === Buffer || data.constructor === ArrayBuffer) {
+                pendingSerializationContext = pendingSerializationContext || {
+                    buffers: [],
+                };
+                const buffers: Buffer[] = pendingSerializationContext.buffers;
+                buffers.push(Buffer.from(data));
+                return;
+            }
+            const messageSerializationContext = pendingSerializationContext;
+            pendingSerializationContext = undefined;
+            rpcPeer.handleMessage(JSON.parse(data as string), messageSerializationContext);
         });
-        socket.on('message', data => rpcPeer.handleMessage(JSON.parse(data as string)));
         // wrap the host api with a connection specific api that can be torn down on disconnect
         const api = new PluginAPIProxy(this.api, await this.peer.getParam('mediaManager'));
         const kill = () => {
@@ -336,6 +387,8 @@ export class PluginHost {
         }
         socket.on('close', kill);
         socket.on('error', kill);
+
+        rpcPeer.addSerializer(Buffer, 'Buffer', new SidebandBufferSerializer());
         return setupPluginRemote(rpcPeer, api, null, () => this.scrypted.stateManager.getSystemState());
     }
 }

package/src/plugin/plugin-http.ts

@@ -1,29 +1,32 @@
-import { Request, Response, Router } from 'express';
-import bodyParser from 'body-parser';
 import { HttpRequest } from '@scrypted/types';
-import WebSocket, { Server as WebSocketServer } from "ws";
+import bodyParser from 'body-parser';
+import { Request, Response, Router } from 'express';
 import { ServerResponse } from 'http';
+import WebSocket, { Server as WebSocketServer } from "ws";
 
 export abstract class PluginHttp<T> {
     wss = new WebSocketServer({ noServer: true });
 
     constructor(public app: Router) {
-        app.all(['/endpoint/@:owner/:pkg/public/engine.io/*', '/endpoint/:pkg/public/engine.io/*'], (req, res) => {
+    }
+
+    addMiddleware() {
+        this.app.all(['/endpoint/@:owner/:pkg/public/engine.io/*', '/endpoint/:pkg/public/engine.io/*'], (req, res) => {
             this.endpointHandler(req, res, true, true, this.handleEngineIOEndpoint.bind(this))
         });
 
-        app.all(['/endpoint/@:owner/:pkg/engine.io/*', '/endpoint/@:owner/:pkg/engine.io/*'], (req, res) => {
+        this.app.all(['/endpoint/@:owner/:pkg/engine.io/*', '/endpoint/@:owner/:pkg/engine.io/*'], (req, res) => {
             this.endpointHandler(req, res, false, true, this.handleEngineIOEndpoint.bind(this))
         });
 
         // stringify all http endpoints
-        app.all(['/endpoint/@:owner/:pkg/public', '/endpoint/@:owner/:pkg/public/*', '/endpoint/:pkg', '/endpoint/:pkg/*'], bodyParser.text() as any);
+        this.app.all(['/endpoint/@:owner/:pkg/public', '/endpoint/@:owner/:pkg/public/*', '/endpoint/:pkg', '/endpoint/:pkg/*'], bodyParser.text() as any);
 
-        app.all(['/endpoint/@:owner/:pkg/public', '/endpoint/@:owner/:pkg/public/*', '/endpoint/:pkg/public', '/endpoint/:pkg/public/*'], (req, res) => {
+        this.app.all(['/endpoint/@:owner/:pkg/public', '/endpoint/@:owner/:pkg/public/*', '/endpoint/:pkg/public', '/endpoint/:pkg/public/*'], (req, res) => {
             this.endpointHandler(req, res, true, false, this.handleRequestEndpoint.bind(this))
         });
 
-        app.all(['/endpoint/@:owner/:pkg', '/endpoint/@:owner/:pkg/*', '/endpoint/:pkg', '/endpoint/:pkg/*'], (req, res) => {
+        this.app.all(['/endpoint/@:owner/:pkg', '/endpoint/@:owner/:pkg/*', '/endpoint/:pkg', '/endpoint/:pkg/*'], (req, res) => {
             this.endpointHandler(req, res, false, false, this.handleRequestEndpoint.bind(this))
         });
     }

package/src/plugin/plugin-remote-websocket.ts

@@ -10,7 +10,6 @@ interface WebSocketEventListener {
     (evt: WebSocketEvent): void;
 }
 
-// @ts-ignore
 class WebSocketEventTarget {
     events: { [type: string]: WebSocketEventListener[] } = {};
 
@@ -71,7 +70,6 @@ export interface WebSocketMethods {
 
 export function createWebSocketClass(__websocketConnect: WebSocketConnect): any {
 
-    // @ts-ignore
     class WebSocket extends WebSocketEventTarget {
         _url: string;
         _protocols: string[];

package/src/plugin/plugin-remote-worker.ts

@@ -149,16 +149,18 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
         }, getDeviceConsole(nativeId), `[${systemManager.getDeviceById(mixinId)?.name}]`);
     }
 
-    let lastCpuUsage: NodeJS.CpuUsage;
-    setInterval(() => {
-        const cpuUsage = process.cpuUsage(lastCpuUsage);
-        lastCpuUsage = cpuUsage;
-        peer.sendOob({
-            type: 'stats',
-            cpu: cpuUsage,
-            memoryUsage: process.memoryUsage(),
-        });
-    }, 10000);
+    peer.getParam('updateStats').then((updateStats: (stats: any) => void) => {
+        let lastCpuUsage: NodeJS.CpuUsage;
+        setInterval(() => {
+            const cpuUsage = process.cpuUsage(lastCpuUsage);
+            lastCpuUsage = cpuUsage;
+            updateStats({
+                type: 'stats',
+                cpu: cpuUsage,
+                memoryUsage: process.memoryUsage(),
+            });
+        }, 10000);
+    });
 
     let replPort: Promise<number>;
 

package/src/plugin/plugin-remote.ts

@@ -300,7 +300,8 @@ export async function setupPluginRemote(peer: RpcPeer, api: PluginAPI, pluginId:
         // core plugin to web (node to browser).
         // always add the BufferSerializer, so serialization is gauranteed to work.
         // but in plugin-host, mark Buffer as transport safe.
-        peer.addSerializer(Buffer, 'Buffer', new BufferSerializer());
+        if (!peer.constructorSerializerMap.get(Buffer))
+            peer.addSerializer(Buffer, 'Buffer', new BufferSerializer());
         const getRemote = await peer.getParam('getRemote');
         const remote = await getRemote(api, pluginId);
 
@@ -355,7 +356,8 @@ export interface PluginRemoteAttachOptions {
 export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOptions): Promise<ScryptedStatic> {
     const { createMediaManager, getServicePort, getDeviceConsole, getMixinConsole, getPluginConsole } = options || {};
 
-    peer.addSerializer(Buffer, 'Buffer', new BufferSerializer());
+    if (!peer.constructorSerializerMap.get(Buffer))
+        peer.addSerializer(Buffer, 'Buffer', new BufferSerializer());
 
     let done: (scrypted: ScryptedStatic) => void;
     const retPromise = new Promise<ScryptedStatic>(resolve => done = resolve);

package/src/rpc.ts

@@ -43,10 +43,6 @@ interface RpcResult extends RpcMessage {
     result?: any;
 }
 
-interface RpcOob extends RpcMessage {
-    oob: any;
-}
-
 interface RpcRemoteProxyValue {
     __remote_proxy_id: string | undefined;
     __remote_proxy_finalizer_id: string | undefined;
@@ -127,8 +123,9 @@ class RpcProxy implements PrimitiveProxyHandler<any> {
         // undefined is not JSON serializable.
         const method = target() || null;
         const args: any[] = [];
+        const serializationContext: any = {};
         for (const arg of (argArray || [])) {
-            args.push(this.peer.serialize(arg));
+            args.push(this.peer.serialize(arg, serializationContext));
         }
 
         const rpcApply: RpcApply = {
@@ -147,7 +144,7 @@ class RpcProxy implements PrimitiveProxyHandler<any> {
 
         return this.peer.createPendingResult((id, reject) => {
             rpcApply.id = id;
-            this.peer.send(rpcApply, reject);
+            this.peer.send(rpcApply, reject, serializationContext);
         })
     }
 }
@@ -193,8 +190,8 @@ catch (e) {
 }
 
 export interface RpcSerializer {
-    serialize(value: any): any;
-    deserialize(serialized: any): any;
+    serialize(value: any, serializationContext?: any): any;
+    deserialize(serialized: any, serializationContext?: any): any;
 }
 
 interface LocalProxiedEntry {
@@ -204,7 +201,6 @@ interface LocalProxiedEntry {
 
 export class RpcPeer {
     idCounter = 1;
-    onOob?: (oob: any) => void;
     params: { [name: string]: any } = {};
     pendingResults: { [id: string]: Deferred } = {};
     proxyCounter = 1;
@@ -213,7 +209,7 @@ export class RpcPeer {
     remoteWeakProxies: { [id: string]: WeakRef<any> } = {};
     finalizers = new FinalizationRegistry(entry => this.finalize(entry as LocalProxiedEntry));
     nameDeserializerMap = new Map<string, RpcSerializer>();
-    constructorSerializerMap = new Map<string, string>();
+    constructorSerializerMap = new Map<any, string>();
     transportSafeArgumentTypes = RpcPeer.getDefaultTransportSafeArgumentTypes();
 
     static readonly finalizerIdSymbol = Symbol('rpcFinalizerId');
@@ -251,7 +247,7 @@ export class RpcPeer {
     static readonly PROPERTY_PROXY_PROPERTIES = '__proxy_props';
     static readonly PROPERTY_JSON_COPY_SERIALIZE_CHILDREN = '__json_copy_serialize_children';
 
-    constructor(public selfName: string, public peerName: string, public send: (message: RpcMessage, reject?: (e: Error) => void) => void) {
+    constructor(public selfName: string, public peerName: string, public send: (message: RpcMessage, reject?: (e: Error) => void, serializationContext?: any) => void) {
     }
 
     createPendingResult(cb: (id: string, reject: (e: Error) => void) => void): Promise<any> {
@@ -310,13 +306,6 @@ export class RpcPeer {
         });
     }
 
-    sendOob(oob: any) {
-        this.send({
-            type: 'oob',
-            oob,
-        } as RpcOob)
-    }
-
     evalLocal<T>(script: string, filename?: string, coercedParams?: { [name: string]: any }): T {
         const params = Object.assign({}, this.params, coercedParams);
         let compile: CompileFunction;
@@ -339,7 +328,7 @@ export class RpcPeer {
         result.message = (e as Error).message || 'no message';
     }
 
-    deserialize(value: any): any {
+    deserialize(value: any, deserializationContext: any): any {
         if (!value)
             return value;
 
@@ -347,7 +336,7 @@ export class RpcPeer {
         if (copySerializeChildren) {
             const ret: any = {};
             for (const [key, val] of Object.entries(value)) {
-                ret[key] = this.deserialize(val);
+                ret[key] = this.deserialize(val, deserializationContext);
             }
             return ret;
         }
@@ -370,17 +359,17 @@ export class RpcPeer {
 
         const deserializer = this.nameDeserializerMap.get(__remote_constructor_name);
         if (deserializer) {
-            return deserializer.deserialize(__serialized_value);
+            return deserializer.deserialize(__serialized_value, deserializationContext);
         }
 
         return value;
     }
 
-    serialize(value: any): any {
+    serialize(value: any, serializationContext: any): any {
         if (value?.[RpcPeer.PROPERTY_JSON_COPY_SERIALIZE_CHILDREN] === true) {
             const ret: any = {};
             for (const [key, val] of Object.entries(value)) {
-                ret[key] = this.serialize(val);
+                ret[key] = this.serialize(val, serializationContext);
             }
             return ret;
         }
@@ -419,7 +408,7 @@ export class RpcPeer {
             const serializer = this.nameDeserializerMap.get(serializerMapName);
             if (!serializer)
                 throw new Error('serializer not found for ' + serializerMapName);
-            const serialized = serializer.serialize(value);
+            const serialized = serializer.serialize(value, serializationContext);
             const ret: RpcRemoteProxyValue = {
                 __remote_proxy_id: undefined,
                 __remote_proxy_finalizer_id: undefined,
@@ -464,17 +453,18 @@ export class RpcPeer {
         return proxy;
     }
 
-    async handleMessage(message: RpcMessage) {
+    async handleMessage(message: RpcMessage, deserializationContext?: any) {
         try {
             switch (message.type) {
                 case 'param': {
                     const rpcParam = message as RpcParam;
+                    const serializationContext: any = {};
                     const result: RpcResult = {
                         type: 'result',
                         id: rpcParam.id,
-                        result: this.serialize(this.params[rpcParam.param])
+                        result: this.serialize(this.params[rpcParam.param], serializationContext)
                     };
-                    this.send(result);
+                    this.send(result, undefined, serializationContext);
                     break;
                 }
                 case 'apply': {
@@ -483,6 +473,7 @@ export class RpcPeer {
                         type: 'result',
                         id: rpcApply.id || '',
                     };
+                    const serializationContext: any = {};
 
                     try {
                         const target = this.localProxyMap[rpcApply.proxyId];
@@ -491,7 +482,7 @@ export class RpcPeer {
 
                         const args = [];
                         for (const arg of (rpcApply.args || [])) {
-                            args.push(this.deserialize(arg));
+                            args.push(this.deserialize(arg, deserializationContext));
                         }
 
                         let value: any;
@@ -505,7 +496,7 @@ export class RpcPeer {
                             value = await target(...args);
                         }
 
-                        result.result = this.serialize(value);
+                        result.result = this.serialize(value, serializationContext);
                     }
                     catch (e) {
                         // console.error('failure', rpcApply.method, e);
@@ -513,7 +504,7 @@ export class RpcPeer {
                     }
 
                     if (!rpcApply.oneway)
-                        this.send(result);
+                        this.send(result, undefined, serializationContext);
                     break;
                 }
                 case 'result': {
@@ -530,7 +521,7 @@ export class RpcPeer {
                         deferred.reject(e);
                         return;
                     }
-                    deferred.resolve(this.deserialize(rpcResult.result));
+                    deferred.resolve(this.deserialize(rpcResult.result, deserializationContext));
                     break;
                 }
                 case 'finalize': {
@@ -547,11 +538,6 @@ export class RpcPeer {
                     }
                     break;
                 }
-                case 'oob': {
-                    const rpcOob = message as RpcOob;
-                    this.onOob?.(rpcOob.oob);
-                    break;
-                }
                 default:
                     throw new Error(`unknown rpc message type ${message.type}`);
             }

package/src/runtime.ts

@@ -6,7 +6,7 @@ import { Plugin, PluginDevice, ScryptedAlert } from './db-types';
 import { getState, ScryptedStateManager, setState } from './state';
 import { Request, Response } from 'express';
 import { createResponseInterface } from './http-interfaces';
-import http, { ServerResponse } from 'http';
+import http, { ServerResponse, IncomingHttpHeaders } from 'http';
 import https from 'https';
 import express from 'express';
 import { LogEntry, Logger, makeAlertId } from './logger';
@@ -25,13 +25,15 @@ import semver from 'semver';
 import { ServiceControl } from './services/service-control';
 import { Alerts } from './services/alerts';
 import { Info } from './services/info';
-import io from 'engine.io';
+import * as io from 'engine.io';
 import { spawn as ptySpawn } from 'node-pty';
 import rimraf from 'rimraf';
 import { getPluginVolume } from './plugin/plugin-volume';
 import { PluginHttp } from './plugin/plugin-http';
 import AdmZip from 'adm-zip';
 import path from 'path';
+import { CORSControl, CORSServer } from './services/cors';
+import { IOServer, IOServerSocket } from './io';
 
 interface DeviceProxyPair {
     handler: PluginDeviceProxyHandler;
@@ -56,9 +58,18 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
     devicesLogger = this.logger.getLogger('device', 'Devices');
     wss = new WebSocketServer({ noServer: true });
     wsAtomic = 0;
-    shellio = io(undefined, {
+    shellio: IOServer = new io.Server({
         pingTimeout: 120000,
+        perMessageDeflate: true,
+        cors: (req, callback) => {
+            const header = this.getAccessControlAllowOrigin(req.headers);
+            callback(undefined, {
+                origin: header,
+                credentials: true,
+            })
+        },
     });
+    cors: CORSServer[] = [];
 
     constructor(datastore: Level, insecure: http.Server, secure: https.Server, app: express.Application) {
         super(app);
@@ -67,6 +78,48 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
 
         app.disable('x-powered-by');
 
+        this.app.options(['/endpoint/@:owner/:pkg/engine.io/api/activate', '/endpoint/@:owner/:pkg/engine.io/api/activate'], (req, res) => {
+            this.addAccessControlHeaders(req, res);
+            res.setHeader('Access-Control-Allow-Methods', 'POST, OPTIONS');
+            res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, Content-Length, X-Requested-With');
+            res.send(200);
+        });
+
+        this.app.post(['/endpoint/@:owner/:pkg/engine.io/api/activate', '/endpoint/@:owner/:pkg/engine.io/api/activate'], (req, res) => {
+            const { username } = (req as any);
+            if (!username) {
+                res.status(401);
+                res.send('Not Authorized');
+                return;
+            }
+
+            const { owner, pkg } = req.params;
+            let endpoint = pkg;
+            if (owner)
+                endpoint = `@${owner}/${endpoint}`;
+
+            const { id } = req.body;
+            try {
+                const host = this.plugins?.[endpoint];
+                if (!host)
+                    throw new Error('invalid plugin');
+                // @ts-expect-error
+                const socket: IOServerSocket = host.io.clients[id];
+                if (!socket)
+                    throw new Error('invalid socket');
+                socket.emit('/api/activate');
+                res.send({
+                    id,
+                })
+            }
+            catch (e) {
+                res.status(500);
+                res.end();
+            }
+        });
+
+        this.addMiddleware();
+
         app.get('/web/oauth/callback', (req, res) => {
             this.oauthCallback(req, res);
         });
@@ -122,6 +175,34 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
         }, 60 * 60 * 1000);
     }
 
+    addAccessControlHeaders(req: http.IncomingMessage, res: http.ServerResponse) {
+        res.setHeader('Vary', 'Origin,Referer');
+        const header = this.getAccessControlAllowOrigin(req.headers);
+        if (header)
+            res.setHeader('Access-Control-Allow-Origin', header);
+    }
+
+    getAccessControlAllowOrigin(headers: http.IncomingHttpHeaders) {
+        let { origin, referer } = headers;
+        if (!origin && referer) {
+            try {
+                const u = new URL(headers.referer)
+                origin = u.origin;
+            }
+            catch (e) {
+                return;
+            }
+        }
+        if (!origin)
+            return;
+        const servers: string[] = process.env.SCRYPTED_ACCESS_CONTROL_ALLOW_ORIGINS?.split(',') || [];
+        servers.push(...Object.values(this.cors).map(entry => entry.server));
+        if (!servers.includes(origin))
+            return;
+
+        return origin;
+    }
+
     getDeviceLogger(device: PluginDevice): Logger {
         return this.devicesLogger.getLogger(device._id, getState(device, ScryptedInterfaceProperty.name));
     }
@@ -311,6 +392,8 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
                 return this.logger;
             case 'alerts':
                 return new Alerts(this);
+            case 'cors':
+                return new CORSControl(this);
         }
     }