@scrypted/server 0.0.135 → 0.0.140

package/src/plugin/plugin-console.ts

@@ -10,9 +10,7 @@ export interface ConsoleServer {
     pluginConsole: Console;
     readPort: number,
     writePort: number,
-    readServer: net.Server,
-    writeServer: net.Server,
-    sockets: Set<net.Socket>;
+    destroy(): void;
 }
 
 export interface StdPassThroughs {
@@ -129,11 +127,22 @@ export async function createConsoleServer(remoteStdout: Readable, remoteStderr:
     const writePort = await listenZero(writeServer);
 
     return {
+        destroy() {
+            for (const socket of sockets) {
+                socket.destroy();
+            }
+            sockets.clear();
+            outputs.clear();
+
+            try {
+                readServer.close();
+                writeServer.close();
+            }
+            catch (e) {
+            }
+        },
         pluginConsole,
         readPort,
         writePort,
-        readServer,
-        writeServer,
-        sockets,
     };
 }

package/src/plugin/plugin-device.ts

@@ -2,7 +2,7 @@ import { DeviceProvider, EventDetails, EventListenerOptions, EventListenerRegist
 import { ScryptedRuntime } from "../runtime";
 import { PluginDevice } from "../db-types";
 import { MixinProvider } from "@scrypted/types";
-import { handleFunctionInvocations, PrimitiveProxyHandler } from "../rpc";
+import { RpcPeer, PrimitiveProxyHandler } from "../rpc";
 import { getState } from "../state";
 import { getDisplayType } from "../infer-defaults";
 import { allInterfaceProperties, isValidInterfaceMethod, methodInterfaces } from "./descriptor";
@@ -285,7 +285,7 @@ export class PluginDeviceProxyHandler implements PrimitiveProxyHandler<any>, Scr
     get(target: any, p: PropertyKey, receiver: any): any {
         if (p === 'constructor')
             return;
-        const handled = handleFunctionInvocations(this, target, p, receiver);
+        const handled = RpcPeer.handleFunctionInvocations(this, target, p, receiver);
         if (handled)
             return handled;
         const pluginDevice = this.scrypted.findPluginDeviceById(this.id);

package/src/plugin/plugin-host-api.ts

@@ -6,12 +6,12 @@ import { Logger } from '../logger';
 import { getState } from '../state';
 import { PluginHost } from './plugin-host';
 import debounce from 'lodash/debounce';
-import { PROPERTY_PROXY_ONEWAY_METHODS } from '../rpc';
+import { RpcPeer } from '../rpc';
 
 export class PluginHostAPI extends PluginAPIManagedListeners implements PluginAPI {
     pluginId: string;
 
-    [PROPERTY_PROXY_ONEWAY_METHODS] = [
+    [RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS] = [
         'onMixinEvent',
         'onDeviceEvent',
         'setStorage',

package/src/plugin/plugin-host.ts

@@ -12,24 +12,21 @@ import WebSocket from 'ws';
 import { sleep } from '../sleep';
 import { PluginHostAPI } from './plugin-host-api';
 import path from 'path';
-import child_process from 'child_process';
 import { PluginDebug } from './plugin-debug';
-import readline from 'readline';
-import { Readable, Writable } from 'stream';
 import { ensurePluginVolume, getScryptedVolume } from './plugin-volume';
-import { getPluginNodePath } from './plugin-npm-dependencies';
 import { ConsoleServer, createConsoleServer } from './plugin-console';
 import { LazyRemote } from './plugin-lazy-remote';
 import crypto from 'crypto';
 import fs from 'fs';
 import mkdirp from 'mkdirp';
 import rimraf from 'rimraf';
+import { RuntimeWorker } from './runtime/runtime-worker';
+import { PythonRuntimeWorker } from './runtime/python-worker';
+import { NodeForkWorker } from './runtime/node-fork-worker';
+import { NodeThreadWorker } from './runtime/node-thread-worker';
 
 export class PluginHost {
-    static sharedWorker: child_process.ChildProcess;
-    static sharedWorkerImmediateRestart = false;
-
-    worker: child_process.ChildProcess;
+    worker: RuntimeWorker;
     peer: RpcPeer;
     pluginId: string;
     module: Promise<any>;
@@ -53,10 +50,8 @@ export class PluginHost {
     kill() {
         this.killed = true;
         this.api.removeListeners();
-        // things might get a bit race prone, so clear out the shared worker before killing.
-        if (this.worker === PluginHost.sharedWorker)
-            PluginHost.sharedWorker = undefined;
-        this.worker.kill('SIGKILL');
+        this.peer.kill('plugin killed');
+        this.worker.kill();
         this.io.close();
         for (const s of Object.values(this.ws)) {
             s.close();
@@ -66,14 +61,7 @@ export class PluginHost {
         const deviceIds = new Set<string>(Object.values(this.scrypted.pluginDevices).filter(d => d.pluginId === this.pluginId).map(d => d._id));
         this.scrypted.invalidateMixins(deviceIds);
 
-        this.consoleServer?.then(server => {
-            server.readServer.close();
-            server.writeServer.close();
-            for (const s of server.sockets) {
-                s.destroy();
-            }
-        });
-        setTimeout(() => this.peer.kill('plugin killed'), 500);
+        this.consoleServer.then(server => server.destroy());
     }
 
     toString() {
@@ -86,7 +74,7 @@ export class PluginHost {
         return pi._id;
     }
 
-    constructor(scrypted: ScryptedRuntime, plugin: Plugin, public pluginDebug?: PluginDebug) {
+    constructor(scrypted: ScryptedRuntime, plugin: Plugin, pluginDebug?: PluginDebug) {
         this.scrypted = scrypted;
         this.pluginId = plugin._id;
         this.pluginName = plugin.packageJson?.name;
@@ -101,9 +89,8 @@ export class PluginHost {
         const pluginVolume = ensurePluginVolume(this.pluginId);
 
         this.startPluginHost(logger, {
-            NODE_PATH: path.join(getPluginNodePath(this.pluginId), 'node_modules'),
             SCRYPTED_PLUGIN_VOLUME: pluginVolume,
-        }, this.packageJson.scrypted.runtime);
+        }, pluginDebug);
 
         this.io.on('connection', async (socket) => {
             try {
@@ -235,137 +222,41 @@ export class PluginHost {
         });
     }
 
-    startPluginHost(logger: Logger, env?: any, runtime?: string) {
+    startPluginHost(logger: Logger, env: any, pluginDebug: PluginDebug) {
         let connected = true;
 
-        if (runtime === 'python') {
-            const args: string[] = [
-                '-u',
-            ];
-            if (this.pluginDebug) {
-                args.push(
-                    '-m',
-                    'debugpy',
-                    '--listen',
-                    `0.0.0.0:${this.pluginDebug.inspectPort}`,
-                    '--wait-for-client',
-                )
-            }
-            args.push(
-                path.join(__dirname, '../../python', 'plugin-remote.py'),
-            )
-
-            this.worker = child_process.spawn('python3', args, {
-                // stdin, stdout, stderr, peer in, peer out
-                stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
-                env: Object.assign({
-                    PYTHONPATH: path.join(process.cwd(), 'node_modules/@scrypted/types'),
-                }, process.env, env),
-            });
-
-            const peerin = this.worker.stdio[3] as Writable;
-            const peerout = this.worker.stdio[4] as Readable;
-
-            peerin.on('error', e => connected = false);
-            peerout.on('error', e => connected = false);
-
-            this.peer = new RpcPeer('host', this.pluginId, (message, reject) => {
-                if (connected) {
-                    peerin.write(JSON.stringify(message) + '\n', e => e && reject?.(e));
-                }
-                else if (reject) {
-                    reject(new Error('peer disconnected'));
-                }
-            });
-
-            const readInterface = readline.createInterface({
-                input: peerout,
-                terminal: false,
-            });
-            readInterface.on('line', line => {
-                this.peer.handleMessage(JSON.parse(line));
+        if (this.packageJson.scrypted.runtime === 'python') {
+            this.worker =  new PythonRuntimeWorker(this.pluginId, {
+                env,
+                pluginDebug,
             });
         }
         else {
-            const execArgv: string[] = process.execArgv.slice();
-            if (this.pluginDebug) {
-                execArgv.push(`--inspect=0.0.0.0:${this.pluginDebug.inspectPort}`);
-            }
-
-            const useSharedWorker = process.env.SCRYPTED_SHARED_WORKER &&
-                this.packageJson.scrypted.sharedWorker !== false &&
-                this.packageJson.scrypted.realfs !== true &&
-                Object.keys(this.packageJson.optionalDependencies || {}).length === 0;
-            if (useSharedWorker) {
-                if (!PluginHost.sharedWorker) {
-                    const worker = child_process.fork(require.main.filename, ['child', '@scrypted/shared'], {
-                        stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
-                        env: Object.assign({}, process.env, env),
-                        serialization: 'advanced',
-                        execArgv,
-                    });
-                    PluginHost.sharedWorker = worker;
-                    PluginHost.sharedWorker.setMaxListeners(100);
-                    const clearSharedWorker = () => {
-                        if (worker === PluginHost.sharedWorker)
-                            PluginHost.sharedWorker = undefined;
-                    };
-                    PluginHost.sharedWorker.on('close', () => clearSharedWorker);
-                    PluginHost.sharedWorker.on('error', () => clearSharedWorker);
-                    PluginHost.sharedWorker.on('exit', () => clearSharedWorker);
-                    PluginHost.sharedWorker.on('disconnect', () => clearSharedWorker);
-                }
-                PluginHost.sharedWorker.send({
-                    type: 'start',
-                    pluginId: this.pluginId,
-                });
-                this.worker = PluginHost.sharedWorker;
-                this.worker.on('message', (message: any) => {
-                    if (message.pluginId === this.pluginId)
-                        this.peer.handleMessage(message.message)
-                });
-    
-                this.peer = new RpcPeer('host', this.pluginId, (message, reject) => {
-                    if (connected) {
-                        this.worker.send({
-                            type: 'message',
-                            pluginId: this.pluginId,
-                            message: message,
-                        }, undefined, e => {
-                            if (e && reject)
-                                reject(e);
-                        });
-                    }
-                    else if (reject) {
-                        reject(new Error('peer disconnected'));
-                    }
+            if (!process.env.SCRYPTED_SHARED_WORKER || (this.packageJson.optionalDependencies && Object.keys(this.packageJson.optionalDependencies).length)) {
+                this.worker = new NodeForkWorker(this.pluginId, {
+                    env,
+                    pluginDebug,
                 });
             }
             else {
-                this.worker = child_process.fork(require.main.filename, ['child', this.pluginId], {
-                    stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
-                    env: Object.assign({}, process.env, env),
-                    serialization: 'advanced',
-                    execArgv,
-                });
-                this.worker.on('message', message => this.peer.handleMessage(message as any));
-    
-                this.peer = new RpcPeer('host', this.pluginId, (message, reject) => {
-                    if (connected) {
-                        this.worker.send(message, undefined, e => {
-                            if (e && reject)
-                                reject(e);
-                        });
-                    }
-                    else if (reject) {
-                        reject(new Error('peer disconnected'));
-                    }
+                this.worker = new NodeThreadWorker(this.pluginId, {
+                    env,
+                    pluginDebug,
                 });
             }
-
-            this.peer.transportSafeArgumentTypes.add(Buffer.name);
         }
 
+        this.peer = new RpcPeer('host', this.pluginId, (message, reject) => {
+            if (connected) {
+                this.worker.send(message, reject);
+            }
+            else if (reject) {
+                reject(new Error('peer disconnected'));
+            }
+        });
+
+        this.worker.setupRpcPeer(this.peer);
+
         this.worker.stdout.on('data', data => console.log(data.toString()));
         this.worker.stderr.on('data', data => console.error(data.toString()));
         this.consoleServer = createConsoleServer(this.worker.stdout, this.worker.stderr);
@@ -375,21 +266,26 @@ export class PluginHost {
             pluginConsole.log('starting plugin', this.pluginId, this.packageJson.version);
         });
 
-        this.worker.on('close', () => {
+        const disconnect = () => {
             connected = false;
+            this.peer.kill('plugin disconnected');
+        };
+
+        this.worker.on('close', () => {
             logger.log('e', `${this.pluginName} close`);
+            disconnect();
         });
         this.worker.on('disconnect', () => {
-            connected = false;
             logger.log('e', `${this.pluginName} disconnected`);
+            disconnect();
         });
         this.worker.on('exit', async (code, signal) => {
-            connected = false;
             logger.log('e', `${this.pluginName} exited ${code} ${signal}`);
+            disconnect();
         });
         this.worker.on('error', e => {
-            connected = false;
             logger.log('e', `${this.pluginName} error ${e}`);
+            disconnect();
         });
 
         this.peer.onOob = (oob: any) => {

package/src/plugin/plugin-remote-websocket.ts

@@ -69,7 +69,7 @@ export interface WebSocketMethods {
     close(message: string): void;
 }
 
-export function createWebSocketClass(__websocketConnect: WebSocketConnect) {
+export function createWebSocketClass(__websocketConnect: WebSocketConnect): any {
 
     // @ts-ignore
     class WebSocket extends WebSocketEventTarget {

package/src/plugin/plugin-remote-worker.ts

@@ -10,50 +10,8 @@ import net from 'net'
 import { installOptionalDependencies } from './plugin-npm-dependencies';
 import { createREPLServer } from './plugin-repl';
 
-export function startSharedPluginRemote() {
-    process.setMaxListeners(100);
-    process.on('message', (message: any) => {
-        if (message.type === 'start')
-            startPluginRemote(message.pluginId)
-    });
-}
-
-export function startPluginRemote(pluginId: string) {
-    let peerSend: (message: RpcMessage, reject?: (e: Error) => void) => void;
-    let peerListener: NodeJS.MessageListener;
-
-    if (process.argv[3] === '@scrypted/shared') {
-        peerSend = (message, reject) => process.send({
-            pluginId,
-            message,
-        }, undefined, {
-            swallowErrors: !reject,
-        }, e => {
-            if (e)
-                reject?.(e);
-        });
-        peerListener = (message: any) => {
-            if (message.type === 'message' && message.pluginId === pluginId)
-                peer.handleMessage(message.message);
-        }
-    }
-    else {
-        peerSend = (message, reject) => process.send(message, undefined, {
-            swallowErrors: !reject,
-        }, e => {
-            if (e)
-                reject?.(e);
-        });
-        peerListener = message => peer.handleMessage(message as RpcMessage);
-    }
-
+export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessage, reject?: (e: Error) => void) => void) {
     const peer = new RpcPeer('unknown', 'host', peerSend);
-    peer.transportSafeArgumentTypes.add(Buffer.name);
-    process.on('message', peerListener);
-    process.on('disconnect', () => {
-        console.error('peer host disconnected, exiting.');
-        process.exit(1);
-    });
 
     let systemManager: SystemManager;
     let deviceManager: DeviceManager;
@@ -259,6 +217,9 @@ export function startPluginRemote(pluginId: string) {
         systemManager = scrypted.systemManager;
         deviceManager = scrypted.deviceManager;
 
+        process.removeAllListeners('uncaughtException');
+        process.removeAllListeners('unhandledRejection');
+
         process.on('uncaughtException', e => {
             getPluginConsole().error('uncaughtException', e);
             scrypted.log.e('uncaughtException ' + e?.toString());
@@ -267,5 +228,7 @@ export function startPluginRemote(pluginId: string) {
             getPluginConsole().error('unhandledRejection', e);
             scrypted.log.e('unhandledRejection ' + e?.toString());
         });
-    })
+    });
+
+    return peer;
 }

package/src/plugin/plugin-remote.ts

@@ -4,7 +4,7 @@ import path from 'path';
 import { ScryptedNativeId, DeviceManager, Logger, Device, DeviceManifest, DeviceState, EndpointManager, SystemDeviceState, ScryptedStatic, SystemManager, MediaManager, ScryptedMimeTypes, ScryptedInterface, ScryptedInterfaceProperty, HttpRequest } from '@scrypted/types'
 import { PluginAPI, PluginLogger, PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
 import { SystemManagerImpl } from './system';
-import { RpcPeer, RPCResultError, PROPERTY_PROXY_ONEWAY_METHODS, PROPERTY_JSON_DISABLE_SERIALIZATION } from '../rpc';
+import { RpcPeer, RPCResultError } from '../rpc';
 import { BufferSerializer } from './buffer-serializer';
 import { createWebSocketClass, WebSocketConnectCallbacks, WebSocketMethods } from './plugin-remote-websocket';
 import fs from 'fs';
@@ -88,7 +88,7 @@ class EndpointManagerImpl implements EndpointManager {
     }
     async getPublicCloudEndpoint(nativeId?: ScryptedNativeId): Promise<string> {
         const local = await this.getPublicLocalEndpoint(nativeId);
-        const mo = this.mediaManager.createMediaObject(local, ScryptedMimeTypes.LocalUrl);
+        const mo = this.mediaManager.createMediaObject(Buffer.from(local), ScryptedMimeTypes.LocalUrl);
         return this.mediaManager.convertMediaObjectToUrl(mo, ScryptedMimeTypes.LocalUrl);
     }
     async getPublicLocalEndpoint(nativeId?: ScryptedNativeId): Promise<string> {
@@ -361,9 +361,9 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
 
         const localStorage = new StorageImpl(deviceManager, undefined);
 
-        const remote: PluginRemote & { [PROPERTY_JSON_DISABLE_SERIALIZATION]: boolean, [PROPERTY_PROXY_ONEWAY_METHODS]: string[] } = {
-            [PROPERTY_JSON_DISABLE_SERIALIZATION]: true,
-            [PROPERTY_PROXY_ONEWAY_METHODS]: [
+        const remote: PluginRemote & { [RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION]: boolean, [RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS]: string[] } = {
+            [RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION]: true,
+            [RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS]: [
                 'notify',
                 'updateDeviceState',
                 'setSystemState',

package/src/plugin/runtime/child-process-worker.ts

@@ -0,0 +1,49 @@
+import { EventEmitter } from "ws";
+import { RuntimeWorker, RuntimeWorkerOptions } from "./runtime-worker";
+import child_process from 'child_process';
+import { RpcMessage, RpcPeer } from "../../rpc";
+
+export abstract class ChildProcessWorker extends EventEmitter implements RuntimeWorker {
+    worker: child_process.ChildProcess;
+
+    constructor(public pluginId: string, options: RuntimeWorkerOptions) {
+        super();
+    }
+
+    setupWorker() {
+        this.worker.on('close', () => this.emit('close'));
+        this.worker.on('disconnect', () => this.emit('disconnect'));
+        this.worker.on('exit', (code, signal) => this.emit('exit', code, signal));
+        this.worker.on('close', () => this.emit('close'));
+        this.worker.on('error', e => this.emit('error', e));
+    }
+
+    get pid() {
+        return this.worker.pid;
+    }
+
+    get stdout() {
+        return this.worker.stdout;
+    }
+
+    get stderr() {
+        return this.worker.stderr;
+    }
+
+    get killed() {
+        return this.worker.killed;
+    }
+
+    kill(): void {
+        if (!this.worker)
+            return;
+        this.worker.kill('SIGKILL');
+        this.worker.removeAllListeners();
+        this.worker.stdout.removeAllListeners();
+        this.worker.stderr.removeAllListeners();
+        this.worker = undefined;
+    }
+
+    abstract send(message: RpcMessage, reject?: (e: Error) => void): void;
+    abstract setupRpcPeer(peer: RpcPeer): void;
+}

package/src/plugin/runtime/node-fork-worker.ts

@@ -0,0 +1,54 @@
+import { RuntimeWorkerOptions as RuntimeWorkerOptions } from "./runtime-worker";
+import child_process from 'child_process';
+import path from 'path';
+import { RpcMessage, RpcPeer } from "../../rpc";
+import { ChildProcessWorker } from "./child-process-worker";
+import { getPluginNodePath } from "../plugin-npm-dependencies";
+
+export class NodeForkWorker extends ChildProcessWorker {
+
+    constructor(pluginId: string, options: RuntimeWorkerOptions) {
+        super(pluginId, options);
+        
+        const {env, pluginDebug} = options;
+        
+        const execArgv: string[] = process.execArgv.slice();
+        if (pluginDebug) {
+            execArgv.push(`--inspect=0.0.0.0:${pluginDebug.inspectPort}`);
+        }
+
+        this.worker = child_process.fork(require.main.filename, ['child', this.pluginId], {
+            stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
+            env: Object.assign({}, process.env, env, {
+                NODE_PATH: path.join(getPluginNodePath(this.pluginId), 'node_modules'),
+            }),
+            serialization: 'advanced',
+            execArgv,
+        });
+
+        this.setupWorker();
+    }
+
+    setupRpcPeer(peer: RpcPeer): void {
+        this.worker.on('message', message => peer.handleMessage(message as any));
+        peer.transportSafeArgumentTypes.add(Buffer.name);
+    }
+
+    send(message: RpcMessage, reject?: (e: Error) => void): void {
+        try {
+            if (!this.worker)
+                throw new Error('worked has been killed');
+            this.worker.send(message, undefined, e => {
+                if (e && reject)
+                    reject(e);
+            });
+        }
+        catch (e) {
+            reject?.(e);
+        }
+    }
+
+    get pid() {
+        return this.worker.pid;
+    }
+}

package/src/plugin/runtime/node-thread-worker.ts

@@ -0,0 +1,76 @@
+import { EventEmitter } from "ws";
+import { RpcMessage, RpcPeer } from "../../rpc";
+import { RuntimeWorker, RuntimeWorkerOptions } from "./runtime-worker";
+import worker_threads from "worker_threads";
+import path from 'path';
+import { getPluginNodePath } from "../plugin-npm-dependencies";
+import v8 from 'v8';
+
+export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
+    terminated: boolean;
+    worker: worker_threads.Worker;
+
+    constructor(public pluginId: string, options: RuntimeWorkerOptions) {
+        super();
+        const { env } = options;
+
+        this.worker = new worker_threads.Worker(require.main.filename, {
+            argv: ['child-thread', this.pluginId],
+            env: Object.assign({}, process.env, env, {
+                NODE_PATH: path.join(getPluginNodePath(this.pluginId), 'node_modules'),
+            }),
+        });
+
+        this.worker.on('exit', () => {
+            this.terminated = true;
+            this.emit('exit');
+        });
+        this.worker.on('error', e => {
+            this.emit('error', e);
+        });
+        this.worker.on('messageerror', e => {
+            this.emit('error', e);
+        });
+    }
+
+    get pid() {
+        return this.worker.threadId;
+    }
+
+    get stdout() {
+        return this.worker.stdout;
+    }
+
+    get stderr() {
+        return this.worker.stderr;
+    }
+
+    get killed() {
+        return this.terminated;
+    }
+
+    kill(): void {
+        if (!this.worker)
+            return;
+        this.worker.terminate();
+        this.worker.removeAllListeners();
+        this.worker.stdout.removeAllListeners();
+        this.worker.stderr.removeAllListeners();
+        this.worker = undefined;
+    }
+
+    send(message: RpcMessage, reject?: (e: Error) => void): void {
+        try {
+            if (!this.worker)
+                throw new Error('worked has been killed');
+            this.worker.postMessage(v8.serialize(message));
+        }
+        catch (e) {
+            reject?.(e);
+        }
+    }
+
+    setupRpcPeer(peer: RpcPeer): void {
+        this.worker.on('message', message => peer.handleMessage(v8.deserialize(message)));
+    }
+}