@scrypted/server 0.0.115 → 0.0.120

package/src/plugin/plugin-lazy-remote.ts

@@ -17,7 +17,7 @@ import { PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
         })();
     }
 
-    async loadZip(packageJson: any, zipData: Buffer, options?: PluginRemoteLoadZipOptions): Promise<any> {
+    async loadZip(packageJson: any, zipData: Buffer|string, options?: PluginRemoteLoadZipOptions): Promise<any> {
         if (!this.remote)
             await this.remoteReadyPromise;
         return this.remote.loadZip(packageJson, zipData, options);

package/src/plugin/plugin-remote-worker.ts

@@ -0,0 +1,271 @@
+import { RpcMessage, RpcPeer } from '../rpc';
+import { SystemManager, DeviceManager, ScryptedNativeId } from '@scrypted/sdk/types'
+import { attachPluginRemote, PluginReader } from './plugin-remote';
+import { PluginAPI } from './plugin-api';
+import { MediaManagerImpl } from './media';
+import { PassThrough } from 'stream';
+import { Console } from 'console'
+import { install as installSourceMapSupport } from 'source-map-support';
+import net from 'net'
+import { installOptionalDependencies } from './plugin-npm-dependencies';
+import { createREPLServer } from './plugin-repl';
+
+export function startSharedPluginRemote() {
+    process.setMaxListeners(100);
+    process.on('message', (message: any) => {
+        if (message.type === 'start')
+            startPluginRemote(message.pluginId)
+    });
+}
+
+export function startPluginRemote(pluginId: string) {
+    let peerSend: (message: RpcMessage, reject?: (e: Error) => void) => void;
+    let peerListener: NodeJS.MessageListener;
+
+    if (process.argv[3] === '@scrypted/shared') {
+        peerSend = (message, reject) => process.send({
+            pluginId,
+            message,
+        }, undefined, {
+            swallowErrors: !reject,
+        }, e => {
+            if (e)
+                reject?.(e);
+        });
+        peerListener = (message: any) => {
+            if (message.type === 'message' && message.pluginId === pluginId)
+                peer.handleMessage(message.message);
+        }
+    }
+    else {
+        peerSend = (message, reject) => process.send(message, undefined, {
+            swallowErrors: !reject,
+        }, e => {
+            if (e)
+                reject?.(e);
+        });
+        peerListener = message => peer.handleMessage(message as RpcMessage);
+    }
+
+    const peer = new RpcPeer('unknown', 'host', peerSend);
+    peer.transportSafeArgumentTypes.add(Buffer.name);
+    process.on('message', peerListener);
+    process.on('disconnect', () => {
+        console.error('peer host disconnected, exiting.');
+        process.exit(1);
+    });
+
+    let systemManager: SystemManager;
+    let deviceManager: DeviceManager;
+    let api: PluginAPI;
+
+    const getConsole = (hook: (stdout: PassThrough, stderr: PassThrough) => Promise<void>,
+        also?: Console, alsoPrefix?: string) => {
+
+        const stdout = new PassThrough();
+        const stderr = new PassThrough();
+
+        hook(stdout, stderr);
+
+        const ret = new Console(stdout, stderr);
+
+        const methods = [
+            'log', 'warn',
+            'dir', 'time',
+            'timeEnd', 'timeLog',
+            'trace', 'assert',
+            'clear', 'count',
+            'countReset', 'group',
+            'groupEnd', 'table',
+            'debug', 'info',
+            'dirxml', 'error',
+            'groupCollapsed',
+        ];
+
+        const printers = ['log', 'info', 'debug', 'trace', 'warn', 'error'];
+        for (const m of methods) {
+            const old = (ret as any)[m].bind(ret);
+            (ret as any)[m] = (...args: any[]) => {
+                // prefer the mixin version for local/remote console dump.
+                if (also && alsoPrefix && printers.includes(m)) {
+                    (also as any)[m](alsoPrefix, ...args);
+                }
+                else {
+                    (console as any)[m](...args);
+                }
+                // call through to old method to ensure it gets written
+                // to log buffer.
+                old(...args);
+            }
+        }
+
+        return ret;
+    }
+
+    let pluginsPromise: Promise<any>;
+    function getPlugins() {
+        if (!pluginsPromise)
+            pluginsPromise = api.getComponent('plugins');
+        return pluginsPromise;
+    }
+
+    const getDeviceConsole = (nativeId?: ScryptedNativeId) => {
+        // the the plugin console is simply the default console
+        // and gets read from stderr/stdout.
+        if (!nativeId)
+            return console;
+
+        return getConsole(async (stdout, stderr) => {
+            const connect = async () => {
+                const plugins = await getPlugins();
+                const port = await plugins.getRemoteServicePort(peer.selfName, 'console-writer');
+                const socket = net.connect(port);
+                socket.write(nativeId + '\n');
+                const writer = (data: Buffer) => {
+                    socket.write(data);
+                };
+                stdout.on('data', writer);
+                stderr.on('data', writer);
+                socket.on('error', () => {
+                    stdout.removeAllListeners();
+                    stderr.removeAllListeners();
+                    stdout.pause();
+                    stderr.pause();
+                    setTimeout(connect, 10000);
+                });
+            };
+            connect();
+        }, undefined, undefined);
+    }
+
+    const getMixinConsole = (mixinId: string, nativeId: ScryptedNativeId) => {
+        return getConsole(async (stdout, stderr) => {
+            if (!mixinId) {
+                return;
+            }
+            // todo: fix this. a mixin provider can mixin another device to make it a mixin provider itself.
+            // so the mixin id in the mixin table will be incorrect.
+            // there's no easy way to fix this from the remote.
+            // if (!systemManager.getDeviceById(mixinId).mixins.includes(idForNativeId(nativeId))) {
+            //     return;
+            // }
+            const reconnect = () => {
+                stdout.removeAllListeners();
+                stderr.removeAllListeners();
+                stdout.pause();
+                stderr.pause();
+                setTimeout(tryConnect, 10000);
+            };
+
+            const connect = async () => {
+                const ds = deviceManager.getDeviceState(nativeId);
+                if (!ds) {
+                    // deleted?
+                    return;
+                }
+
+                const plugins = await getPlugins();
+                const { pluginId, nativeId: mixinNativeId } = await plugins.getDeviceInfo(mixinId);
+                const port = await plugins.getRemoteServicePort(pluginId, 'console-writer');
+                const socket = net.connect(port);
+                socket.write(mixinNativeId + '\n');
+                const writer = (data: Buffer) => {
+                    let str = data.toString().trim();
+                    str = str.replaceAll('\n', `\n[${ds.name}]: `);
+                    str = `[${ds.name}]: ` + str + '\n';
+                    socket.write(str);
+                };
+                stdout.on('data', writer);
+                stderr.on('data', writer);
+                socket.on('close', reconnect);
+            };
+
+            const tryConnect = async () => {
+                try {
+                    await connect();
+                }
+                catch (e) {
+                    reconnect();
+                }
+            }
+            tryConnect();
+        }, getDeviceConsole(nativeId), `[${systemManager.getDeviceById(mixinId)?.name}]`);
+    }
+
+    let lastCpuUsage: NodeJS.CpuUsage;
+    setInterval(() => {
+        const cpuUsage = process.cpuUsage(lastCpuUsage);
+        lastCpuUsage = cpuUsage;
+        peer.sendOob({
+            type: 'stats',
+            cpu: cpuUsage,
+            memoryUsage: process.memoryUsage(),
+        });
+    }, 10000);
+
+    let replPort: Promise<number>;
+
+    let _pluginConsole: Console;
+    const getPluginConsole = () => {
+        if (!_pluginConsole)
+            _pluginConsole = getDeviceConsole(undefined);
+        return _pluginConsole;
+    }
+
+    attachPluginRemote(peer, {
+        createMediaManager: async (sm) => {
+            systemManager = sm;
+            return new MediaManagerImpl(systemManager, getPluginConsole());
+        },
+        onGetRemote: async (_api, _pluginId) => {
+            api = _api;
+            peer.selfName = pluginId;
+        },
+        onPluginReady: async (scrypted, params, plugin) => {
+            replPort = createREPLServer(scrypted, params, plugin);
+        },
+        getPluginConsole,
+        getDeviceConsole,
+        getMixinConsole,
+        async getServicePort(name, ...args: any[]) {
+            if (name === 'repl') {
+                if (!replPort)
+                    throw new Error('REPL unavailable: Plugin not loaded.')
+                return replPort;
+            }
+            throw new Error(`unknown service ${name}`);
+        },
+        async onLoadZip(pluginReader: PluginReader, packageJson: any) {
+            const entry = pluginReader('main.nodejs.js.map')
+            const map = entry?.toString();
+
+            installSourceMapSupport({
+                environment: 'node',
+                retrieveSourceMap(source) {
+                    if (source === '/plugin/main.nodejs.js' || source === `/${pluginId}/main.nodejs.js`) {
+                        if (!map)
+                            return null;
+                        return {
+                            url: '/plugin/main.nodejs.js',
+                            map,
+                        }
+                    }
+                    return null;
+                }
+            });
+            await installOptionalDependencies(getPluginConsole(), packageJson);
+        }
+    }).then(scrypted => {
+        systemManager = scrypted.systemManager;
+        deviceManager = scrypted.deviceManager;
+
+        process.on('uncaughtException', e => {
+            getPluginConsole().error('uncaughtException', e);
+            scrypted.log.e('uncaughtException ' + e?.toString());
+        });
+        process.on('unhandledRejection', e => {
+            getPluginConsole().error('unhandledRejection', e);
+            scrypted.log.e('unhandledRejection ' + e?.toString());
+        });
+    })
+}

package/src/plugin/plugin-remote.ts

@@ -7,6 +7,8 @@ import { SystemManagerImpl } from './system';
 import { RpcPeer, RPCResultError, PROPERTY_PROXY_ONEWAY_METHODS, PROPERTY_JSON_DISABLE_SERIALIZATION } from '../rpc';
 import { BufferSerializer } from './buffer-serializer';
 import { createWebSocketClass, WebSocketConnectCallbacks, WebSocketMethods } from './plugin-remote-websocket';
+import fs from 'fs';
+const {link} = require('linkfs');
 
 class DeviceLogger implements Logger {
     nativeId: ScryptedNativeId;
@@ -306,13 +308,15 @@ export interface WebSocketCustomHandler {
     methods: WebSocketMethods;
 }
 
+export type PluginReader = (name: string) => Buffer;
+
 export interface PluginRemoteAttachOptions {
     createMediaManager?: (systemManager: SystemManager) => Promise<MediaManager>;
     getServicePort?: (name: string, ...args: any[]) => Promise<number>;
     getDeviceConsole?: (nativeId?: ScryptedNativeId) => Console;
     getPluginConsole?: () => Console;
     getMixinConsole?: (id: string, nativeId?: ScryptedNativeId) => Console;
-    onLoadZip?: (zip: AdmZip, packageJson: any) => Promise<void>;
+    onLoadZip?: (pluginReader: PluginReader, packageJson: any) => Promise<void>;
     onGetRemote?: (api: PluginAPI, pluginId: string) => Promise<void>;
     onPluginReady?: (scrypted: ScryptedStatic, params: any, plugin: any) => Promise<void>;
 }
@@ -434,26 +438,51 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
                 done(ret);
             },
 
-            async loadZip(packageJson: any, zipData: Buffer, zipOptions?: PluginRemoteLoadZipOptions) {
+            async loadZip(packageJson: any, zipData: Buffer | string, zipOptions?: PluginRemoteLoadZipOptions) {
                 const pluginConsole = getPluginConsole?.();
-                const zip = new AdmZip(zipData);
-                await options?.onLoadZip?.(zip, packageJson);
-                const main = zip.getEntry('main.nodejs.js');
-                const script = main.getData().toString();
+
+                let volume: any;
+                let pluginReader: PluginReader;
+                if (zipOptions?.unzippedPath && fs.existsSync(zipOptions?.unzippedPath)) {
+                    volume = link(fs, ['', path.join(zipOptions.unzippedPath, 'fs')]);
+                    pluginReader = name => {
+                        const filename = path.join(zipOptions.unzippedPath, name);
+                        if (!fs.existsSync(filename))
+                            return;
+                        return fs.readFileSync(filename);
+                    };
+                }
+                else {
+                    const admZip = new AdmZip(zipData);
+                    volume = new Volume();
+                    for (const entry of admZip.getEntries()) {
+                        if (entry.isDirectory)
+                            continue;
+                        if (!entry.entryName.startsWith('fs/'))
+                            continue;
+                        const name = entry.entryName.substring('fs/'.length);
+                        volume.mkdirpSync(path.dirname(name));
+                        const data = entry.getData();
+                        volume.writeFileSync(name, data);
+                    }
+
+                    pluginReader = name => {
+                        const entry = admZip.getEntry(name);
+                        if (!entry)
+                            return;
+                        return entry.getData();
+                    }
+                }
+                zipData = undefined;
+
+                await options?.onLoadZip?.(pluginReader, packageJson);
+                const main = pluginReader('main.nodejs.js');
+                pluginReader = undefined;
+                const script = main.toString();
                 const window: any = {};
                 const exports: any = window;
                 window.exports = exports;
 
-                const volume = new Volume();
-                for (const entry of zip.getEntries()) {
-                    if (entry.isDirectory)
-                        continue;
-                    if (!entry.entryName.startsWith('fs/'))
-                        continue;
-                    const name = entry.entryName.substr('fs/'.length);
-                    volume.mkdirpSync(path.dirname(name));
-                    volume.writeFileSync(name, entry.getData());
-                }
 
                 function websocketConnect(url: string, protocols: any, callbacks: WebSocketConnectCallbacks) {
                     if (url.startsWith('io://') || url.startsWith('ws://')) {
@@ -476,7 +505,7 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
                     exports,
                     window,
                     require: (name: string) => {
-                        if (name === 'fs' && !packageJson.scrypted.realfs) {
+                        if (name === 'fakefs' || (name === 'fs' && !packageJson.scrypted.realfs)) {
                             return volume;
                         }
                         if (name === 'realfs') {

package/src/plugin/plugin-repl.ts

@@ -1,4 +1,3 @@
-import { EventEmitter } from 'ws';
 import { listenZero } from './listen-zero';
 import { Server } from 'net';
 import { once } from 'events';
@@ -33,8 +32,11 @@ export async function createREPLServer(scrypted: ScryptedStatic, params: any, pl
             device = await device.getDevice(c);
         }
 
+        const realDevice = systemManager.getDeviceById(device.id);
+
         const ctx = Object.assign(params, {
-            device
+            device,
+            realDevice,
         });
         delete ctx.console;
         delete ctx.window;

package/src/plugin/system.ts

@@ -12,7 +12,7 @@ function newDeviceProxy(id: string, systemManager: SystemManagerImpl) {
 
 
 class DeviceProxyHandler implements PrimitiveProxyHandler<any>, ScryptedDevice {
-    device: ScryptedDevice;
+    device: Promise<ScryptedDevice>;
     constructor(public id: string, public systemManager: SystemManagerImpl) {
     }
 
@@ -41,19 +41,20 @@ class DeviceProxyHandler implements PrimitiveProxyHandler<any>, ScryptedDevice {
         return new Proxy(() => p, this);
     }
 
-    async ensureDevice() {
+     ensureDevice() {
         if (!this.device)
-            this.device = await this.systemManager.api.getDeviceById(this.id);
+            this.device = this.systemManager.api.getDeviceById(this.id);
+        return this.device;
     }
 
     async apply(target: any, thisArg: any, argArray?: any) {
         const method = target();
-        await this.ensureDevice();
+        const device = await this.ensureDevice();
         if (false && method === 'refresh') {
             const name = this.systemManager.state[this.id]?.[ScryptedInterfaceProperty.name].value;
             this.systemManager.log.i(`requested refresh ${name}`);
         }
-        return (this.device as any)[method](...argArray);
+        return (device as any)[method](...argArray);
     }
 
     listen(event: string | EventListenerOptions, callback: (eventSource: ScryptedDevice, eventDetails: EventDetails, eventData: any) => void): EventListenerRegister {
@@ -84,7 +85,8 @@ class EventListenerRegisterImpl implements EventListenerRegister {
     async removeListener(): Promise<void> {
         try {
             const register = await this.promise;
-            register.removeListener();
+            this.promise = undefined;
+            register?.removeListener();
         }
         catch (e) {
             console.error('removeListener', e);

package/src/rpc.ts

@@ -12,6 +12,16 @@ function getDefaultTransportSafeArgumentTypes() {
     return jsonSerializable;
 }
 
+export function startPeriodicGarbageCollection() {
+    if (!global.gc) {
+        console.warn('rpc peer garbage collection not available: global.gc is not exposed.');
+        return;
+    }
+    return setInterval(() => {
+        global?.gc();
+    }, 10000);
+}
+
 export interface RpcMessage {
     type: string;
 }
@@ -135,6 +145,9 @@ class RpcProxy implements PrimitiveProxyHandler<any> {
     }
 
     apply(target: any, thisArg: any, argArray?: any): any {
+        if (Object.isFrozen(this.peer.pendingResults))
+            return Promise.reject(new RPCResultError(this.peer, 'RpcPeer has been killed'));
+
         // rpc objects can be functions. if the function is a oneway method,
         // it will have a null in the oneway method list. this is because
         // undefined is not JSON serializable.
@@ -429,7 +442,6 @@ export class RpcPeer {
         const weakref = new WeakRef(proxy);
         this.remoteWeakProxies[proxyId] = weakref;
         this.finalizers.register(rpc, localProxiedEntry);
-        global.gc?.();
         return proxy;
     }
 
@@ -509,7 +521,6 @@ export class RpcPeer {
                         const localProxiedEntry = this.localProxied.get(local);
                         // if a finalizer id is specified, it must match.
                         if (rpcFinalize.__local_proxy_finalizer_id && rpcFinalize.__local_proxy_finalizer_id !== localProxiedEntry?.finalizerId) {
-                            console.error(this.selfName, this.peerName, 'finalizer mismatch')
                             break;
                         }
                         delete this.localProxyMap[rpcFinalize.__local_proxy_id];

package/src/runtime.ts

@@ -30,6 +30,7 @@ import { spawn as ptySpawn } from 'node-pty';
 import rimraf from 'rimraf';
 import { getPluginVolume } from './plugin/plugin-volume';
 import { PluginHttp } from './plugin/plugin-http';
+import AdmZip from 'adm-zip';
 
 interface DeviceProxyPair {
     handler: PluginDeviceProxyHandler;
@@ -312,7 +313,7 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
         }
     }
 
-    async handleEngineIOEndpoint(req: Request, res: ServerResponse, endpointRequest: HttpRequest, pluginData: HttpPluginData) {
+    handleEngineIOEndpoint(req: Request, res: ServerResponse, endpointRequest: HttpRequest, pluginData: HttpPluginData) {
         const { pluginHost, pluginDevice } = pluginData;
 
         (req as any).scrypted = {
@@ -325,7 +326,7 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
             pluginHost.io.handleRequest(req, res);
     }
 
-    async handleRequestEndpoint(req: Request, res: Response, endpointRequest: HttpRequest, pluginData: HttpPluginData) {
+    handleRequestEndpoint(req: Request, res: Response, endpointRequest: HttpRequest, pluginData: HttpPluginData) {
         const { pluginHost, pluginDevice } = pluginData;
         const handler = this.getDevice<HttpRequestHandler>(pluginDevice._id);
         if (handler.interfaces.includes(ScryptedInterface.EngineIOHandler) && req.headers.connection === 'upgrade' && req.headers.upgrade?.toLowerCase() === 'websocket') {
@@ -333,7 +334,7 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
                 console.log(ws);
             });
         }
-        handler.onRequest(endpointRequest, createResponseInterface(res, pluginHost.zip));
+        handler.onRequest(endpointRequest, createResponseInterface(res, pluginHost.unzippedPath));
     }
 
     killPlugin(plugin: Plugin) {
@@ -459,11 +460,25 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
     }
 
     async installPlugin(plugin: Plugin, pluginDebug?: PluginDebug): Promise<PluginHost> {
-        await this.upsertDevice(plugin._id, plugin.packageJson.scrypted);
+        const device: Device = Object.assign({}, plugin.packageJson.scrypted);
+        try {
+            if (!device.interfaces.includes(ScryptedInterface.Readme)) {
+                const zipData = Buffer.from(plugin.zip, 'base64');
+                const adm = new AdmZip(zipData);
+                const entry = adm.getEntry('README.md');
+                if (entry) {
+                    device.interfaces = device.interfaces.slice();
+                    device.interfaces.push(ScryptedInterface.Readme);
+                }
+            }
+        }
+        catch (e) {
+        }
+        await this.upsertDevice(plugin._id, device);
         return this.runPlugin(plugin, pluginDebug);
     }
 
-    async runPlugin(plugin: Plugin, pluginDebug?: PluginDebug) {
+    runPlugin(plugin: Plugin, pluginDebug?: PluginDebug) {
         this.killPlugin(plugin);
 
         const pluginDevices = this.findPluginDevices(plugin._id);
@@ -490,7 +505,12 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
                     return;
                 }
 
-                this.runPlugin(plugin).catch(e => console.error('error restarting plugin', plugin._id, e));
+                try {
+                    this.runPlugin(plugin);
+                }
+                catch (e) {
+                    console.error('error restarting plugin', plugin._id, e);
+                }
             }, 60000);
         })
         this.plugins[plugin._id] = pluginHost;
@@ -675,7 +695,12 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
         }
 
         for await (const plugin of this.datastore.getAll(Plugin)) {
-            this.runPlugin(plugin).catch(e => console.error('error starting plugin', plugin._id, e));
+            try {
+                this.runPlugin(plugin);
+            }
+            catch (e) {
+                console.error('error starting plugin', plugin._id, e);
+            }
         }
     }
 }