@scrypted/server 0.1.16 → 0.2.1

package/src/plugin/plugin-device.ts

@@ -216,15 +216,24 @@ export class PluginDeviceProxyHandler implements PrimitiveProxyHandler<any>, Scr
         let { allInterfaces } = await previousEntry;
         try {
             const mixinProvider = this.scrypted.getDevice(mixinId) as ScryptedDevice & MixinProvider;
-            const interfaces = mixinProvider?.interfaces?.includes(ScryptedInterface.MixinProvider) && await mixinProvider?.canMixin(type, allInterfaces) as any as ScryptedInterface[];
+            const isMixinProvider = mixinProvider?.interfaces?.includes(ScryptedInterface.MixinProvider);
+            const interfaces = isMixinProvider && await mixinProvider?.canMixin(type, allInterfaces) as any as ScryptedInterface[];
             if (!interfaces) {
                 // this is not an error
                 // do not advertise interfaces so it is skipped during
                 // vtable lookup.
-                console.log(`mixin provider ${mixinId} can no longer mixin ${this.id}`);
-                const mixins: string[] = getState(pluginDevice, ScryptedInterfaceProperty.mixins) || [];
-                this.scrypted.stateManager.setPluginDeviceState(pluginDevice, ScryptedInterfaceProperty.mixins, mixins.filter(mid => mid !== mixinId))
-                this.scrypted.datastore.upsert(pluginDevice);
+                if (!mixinProvider || (isMixinProvider && !interfaces)) {
+                    console.log(`Mixin provider ${mixinId} can no longer mixin ${this.id}. Removing.`, {
+                        mixinProvider: !!mixinProvider,
+                        interfaces,
+                    });
+                    const mixins: string[] = getState(pluginDevice, ScryptedInterfaceProperty.mixins) || [];
+                    this.scrypted.stateManager.setPluginDeviceState(pluginDevice, ScryptedInterfaceProperty.mixins, mixins.filter(mid => mid !== mixinId))
+                    this.scrypted.datastore.upsert(pluginDevice);
+                }
+                else {
+                    console.log(`Mixin provider ${mixinId} can not mixin ${this.id}. It is no longer a MixinProvider. This may be temporary. Passing through.`);
+                }
                 return {
                     passthrough: true,
                     allInterfaces,
@@ -353,12 +362,17 @@ export class PluginDeviceProxyHandler implements PrimitiveProxyHandler<any>, Scr
         for (const mixin of this.mixinTable) {
             const entry = await mixin.entry;
             if (!entry.methods) {
-                const pluginDevice = this.scrypted.findPluginDeviceById(mixin.mixinProviderId || this.id);
-                const plugin = this.scrypted.plugins[pluginDevice.pluginId];
-                let methods = new Set<string>(getInterfaceMethods(ScryptedInterfaceDescriptors, entry.interfaces))
-                if (plugin.api.descriptors)
-                    methods = new Set<string>([...methods, ...getInterfaceMethods(plugin.api.descriptors, entry.interfaces)]);
-                entry.methods = methods;
+                if (entry.interfaces.size) {
+                    const pluginDevice = this.scrypted.findPluginDeviceById(mixin.mixinProviderId || this.id);
+                    const plugin = this.scrypted.plugins[pluginDevice.pluginId];
+                    let methods = new Set<string>(getInterfaceMethods(ScryptedInterfaceDescriptors, entry.interfaces))
+                    if (plugin.api.descriptors)
+                        methods = new Set<string>([...methods, ...getInterfaceMethods(plugin.api.descriptors, entry.interfaces)]);
+                    entry.methods = methods;
+                }
+                else {
+                    entry.methods = new Set();
+                }
             }
             if (entry.methods.has(method)) {
                 return { mixin, entry };

package/src/plugin/plugin-host-api.ts

@@ -47,7 +47,7 @@ export class PluginHostAPI extends PluginAPIManagedListeners implements PluginAP
 
     // do we care about mixin validation here?
     // maybe to prevent/notify errant dangling events?
-    async onMixinEvent(id: string, nativeIdOrMixinDevice: ScryptedNativeId | any, eventInterface: any, eventData?: any) {
+    async onMixinEvent(id: string, nativeIdOrMixinDevice: ScryptedNativeId | any, eventInterface: string, eventData?: any) {
         // nativeId code path has been deprecated in favor of mixin object 12/10/2021
         const device = this.scrypted.findPluginDeviceById(id);
 

package/src/plugin/plugin-host.ts

@@ -214,7 +214,6 @@ export class PluginHost {
                 await remote.setNativeId(pluginDevice.nativeId, pluginDevice._id, pluginDevice.storage || {});
             }
 
-            await remote.setSystemState(scrypted.stateManager.getSystemState());
             const waitDebug = pluginDebug?.waitDebug;
             if (waitDebug) {
                 console.info('waiting for debugger...');

package/src/plugin/plugin-remote-worker.ts

@@ -9,18 +9,24 @@ import { install as installSourceMapSupport } from 'source-map-support';
 import { PassThrough } from 'stream';
 import { RpcMessage, RpcPeer } from '../rpc';
 import { MediaManagerImpl } from './media';
-import { PluginAPI, PluginRemoteLoadZipOptions } from './plugin-api';
+import { PluginAPI, PluginAPIProxy, PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
 import { installOptionalDependencies } from './plugin-npm-dependencies';
-import { attachPluginRemote, PluginReader, setupPluginRemote } from './plugin-remote';
+import { attachPluginRemote, DeviceManagerImpl, PluginReader, setupPluginRemote } from './plugin-remote';
 import { createREPLServer } from './plugin-repl';
 import { NodeThreadWorker } from './runtime/node-thread-worker';
 const { link } = require('linkfs');
 
+interface PluginStats {
+    type: 'stats',
+    cpu: NodeJS.CpuUsage;
+    memoryUsage: NodeJS.MemoryUsage;
+}
+
 export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessage, reject?: (e: Error) => void, serializationContext?: any) => void) {
     const peer = new RpcPeer('unknown', 'host', peerSend);
 
     let systemManager: SystemManager;
-    let deviceManager: DeviceManager;
+    let deviceManager: DeviceManagerImpl;
     let api: PluginAPI;
 
     const getConsole = (hook: (stdout: PassThrough, stderr: PassThrough) => Promise<void>,
@@ -126,12 +132,6 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
             if (!mixinId) {
                 return;
             }
-            // todo: fix this. a mixin provider can mixin another device to make it a mixin provider itself.
-            // so the mixin id in the mixin table will be incorrect.
-            // there's no easy way to fix this from the remote.
-            // if (!systemManager.getDeviceById(mixinId).mixins.includes(idForNativeId(nativeId))) {
-            //     return;
-            // }
             const reconnect = () => {
                 stdout.removeAllListeners();
                 stderr.removeAllListeners();
@@ -178,15 +178,35 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
         return ret;
     }
 
-    peer.getParam('updateStats').then((updateStats: (stats: any) => void) => {
-        let lastCpuUsage: NodeJS.CpuUsage;
+    // process.cpuUsage is for the entire process.
+    // process.memoryUsage is per thread.
+    const allMemoryStats = new Map<NodeThreadWorker, NodeJS.MemoryUsage>();
+
+    peer.getParam('updateStats').then((updateStats: (stats: PluginStats) => void) => {
         setInterval(() => {
-            const cpuUsage = process.cpuUsage(lastCpuUsage);
-            lastCpuUsage = cpuUsage;
+            const cpuUsage = process.cpuUsage();
+            allMemoryStats.set(undefined, process.memoryUsage());
+
+            const memoryUsage: NodeJS.MemoryUsage = {
+                rss: 0,
+                heapTotal: 0,
+                heapUsed: 0,
+                external: 0,
+                arrayBuffers: 0,
+            }
+
+            for (const mu of allMemoryStats.values()) {
+                memoryUsage.rss += mu.rss;
+                memoryUsage.heapTotal += mu.heapTotal;
+                memoryUsage.heapUsed += mu.heapUsed;
+                memoryUsage.external += mu.external;
+                memoryUsage.arrayBuffers += mu.arrayBuffers;
+            }
+
             updateStats({
                 type: 'stats',
                 cpu: cpuUsage,
-                memoryUsage: process.memoryUsage(),
+                memoryUsage,
             });
         }, 10000);
     });
@@ -316,35 +336,75 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
             pluginReader = undefined;
             const script = main.toString();
 
-            scrypted.fork = async () => {
+
+            const forks = new Set<PluginRemote>();
+
+            scrypted.fork = () => {
                 const ntw = new NodeThreadWorker(pluginId, {
                     env: process.env,
                     pluginDebug: undefined,
                 });
-                const threadPeer = new RpcPeer('main', 'thread', (message, reject) => ntw.send(message, reject));
-                threadPeer.params.updateStats = (stats: any) => {
-                    // todo: merge.
-                    // this.stats = stats;
-                }
-                ntw.setupRpcPeer(threadPeer);
 
-                const remote = await setupPluginRemote(threadPeer, api, pluginId, () => systemManager.getSystemState());
-                const forkOptions = Object.assign({}, zipOptions);
-                forkOptions.fork = true;
-                return remote.loadZip(packageJson, zipData, forkOptions)
+                const result = (async () => {
+                    const threadPeer = new RpcPeer('main', 'thread', (message, reject) => ntw.send(message, reject));
+                    threadPeer.params.updateStats = (stats: PluginStats) => {
+                        allMemoryStats.set(ntw, stats.memoryUsage);
+                    }
+                    ntw.setupRpcPeer(threadPeer);
+
+                    class PluginForkAPI extends PluginAPIProxy {
+                        [RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS] = (api as any)[RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS];
+
+                        setStorage(nativeId: string, storage: { [key: string]: any; }): Promise<void> {
+                            const id = deviceManager.nativeIds.get(nativeId).id;
+                            (scrypted.pluginRemoteAPI as PluginRemote).setNativeId(nativeId, id, storage);
+                            for (const r of forks) {
+                                if (r === remote)
+                                    continue;
+                                r.setNativeId(nativeId, id, storage);
+                            }
+                            return super.setStorage(nativeId, storage);
+                        }
+                    }
+                    const forkApi = new PluginForkAPI(api);
+
+                    const remote = await setupPluginRemote(threadPeer, forkApi, pluginId, () => systemManager.getSystemState());
+                    forks.add(remote);
+                    ntw.worker.on('exit', () => {
+                        forkApi.removeListeners();
+                        forks.delete(remote);
+                        allMemoryStats.delete(ntw);
+                    });
+
+                    for (const [nativeId, dmd] of deviceManager.nativeIds.entries()) {
+                        await remote.setNativeId(nativeId, dmd.id, dmd.storage);
+                    }
+
+                    const forkOptions = Object.assign({}, zipOptions);
+                    forkOptions.fork = true;
+                    return remote.loadZip(packageJson, zipData, forkOptions)
+                })();
+
+                result.catch(() => ntw.kill());
+
+                return {
+                    worker: ntw.worker,
+                    result,
+                }
             }
 
             try {
                 peer.evalLocal(script, zipOptions?.filename || '/plugin/main.nodejs.js', params);
-                pluginConsole?.log('plugin successfully loaded');
 
                 if (zipOptions?.fork) {
+                    pluginConsole?.log('plugin forked');
                     const fork = exports.fork;
-                    const ret = await fork();
-                    ret[RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION] = true;
-                    return ret;
+                    const forked = await fork();
+                    forked[RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION] = true;
+                    return forked;
                 }
 
+                pluginConsole?.log('plugin loaded');
                 let pluginInstance = exports.default;
                 // support exporting a plugin class, plugin main function,
                 // or a plugin instance
@@ -365,7 +425,7 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
         }
     }).then(scrypted => {
         systemManager = scrypted.systemManager;
-        deviceManager = scrypted.deviceManager;
+        deviceManager = scrypted.deviceManager as DeviceManagerImpl;
     });
 
     return peer;

package/src/plugin/plugin-remote.ts

@@ -107,6 +107,10 @@ class DeviceStateProxyHandler implements ProxyHandler<any> {
     get?(target: any, p: PropertyKey, receiver: any) {
         if (p === 'id')
             return this.id;
+        if (p === RpcPeer.PROPERTY_PROXY_PROPERTIES)
+            return { id: this.id }
+        if (p === 'setState')
+            return this.setState;
         return this.deviceManager.systemManager.state[this.id][p as string]?.value;
     }
 
@@ -128,7 +132,7 @@ interface DeviceManagerDevice {
     storage: { [key: string]: any };
 }
 
-class DeviceManagerImpl implements DeviceManager {
+export class DeviceManagerImpl implements DeviceManager {
     api: PluginAPI;
     nativeIds = new Map<string, DeviceManagerDevice>();
     deviceStorage = new Map<string, StorageImpl>();
@@ -153,6 +157,11 @@ class DeviceManagerImpl implements DeviceManager {
         return new Proxy(handler, handler);
     }
 
+    createDeviceState(id: string, setState: (property: string, value: any) => Promise<void>): DeviceState {
+        const handler = new DeviceStateProxyHandler(this, id, setState);
+        return new Proxy(handler, handler);
+    }
+
     getDeviceStorage(nativeId?: any): StorageImpl {
         let ret = this.deviceStorage.get(nativeId);
         if (!ret) {
@@ -297,7 +306,7 @@ export async function setupPluginRemote(peer: RpcPeer, api: PluginAPI, pluginId:
         if (!peer.constructorSerializerMap.get(Buffer))
             peer.addSerializer(Buffer, 'Buffer', new BufferSerializer());
         const getRemote = await peer.getParam('getRemote');
-        const remote = await getRemote(api, pluginId);
+        const remote = await getRemote(api, pluginId) as PluginRemote;
 
         await remote.setSystemState(getSystemState());
         api.listen((id, eventDetails, eventData) => {
@@ -337,7 +346,7 @@ export interface WebSocketCustomHandler {
 export type PluginReader = (name: string) => Buffer;
 
 export interface PluginRemoteAttachOptions {
-    createMediaManager?: (systemManager: SystemManager, deviceManager: DeviceManager) => Promise<MediaManager>;
+    createMediaManager?: (systemManager: SystemManager, deviceManager: DeviceManagerImpl) => Promise<MediaManager>;
     getServicePort?: (name: string, ...args: any[]) => Promise<number>;
     getDeviceConsole?: (nativeId?: ScryptedNativeId) => Console;
     getPluginConsole?: () => Console;
@@ -381,6 +390,7 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
             mediaManager,
             log,
             pluginHostAPI: api,
+            pluginRemoteAPI: undefined,
         }
 
         delete peer.params.getRemote;
@@ -402,9 +412,8 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
                 'setNativeId',
             ],
             getServicePort,
-            createDeviceState(id: string, setState: (property: string, value: any) => Promise<void>) {
-                const handler = new DeviceStateProxyHandler(deviceManager, id, setState);
-                return new Proxy(handler, handler);
+            async createDeviceState(id: string, setState: (property: string, value: any) => Promise<void>) {
+                return deviceManager.createDeviceState(id, setState);
             },
 
             async ioEvent(id: string, event: string, message?: any) {
@@ -507,6 +516,8 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
             },
         }
 
+        ret.pluginRemoteAPI = remote;
+
         return remote;
     }
 

package/src/rpc.ts

@@ -321,7 +321,8 @@ export class RpcPeer {
         const params = Object.assign({}, this.params, coercedParams);
         let compile: CompileFunction;
         try {
-            compile = require('vm').compileFunction;;
+            // prevent bundlers from trying to include non-existent vm module.
+            compile = module[`require`]('vm').compileFunction;
         }
         catch (e) {
             compile = compileFunction;

package/src/runtime.ts

@@ -8,7 +8,6 @@ import http, { ServerResponse } from 'http';
 import https from 'https';
 import { spawn as ptySpawn } from 'node-pty-prebuilt-multiarch';
 import path from 'path';
-import qs from "query-string";
 import rimraf from 'rimraf';
 import semver from 'semver';
 import { PassThrough } from 'stream';
@@ -184,12 +183,10 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
 
             const url = new URL(callback_url as string);
             if (url.search) {
-                const search = qs.parse(url.search);
-                const state = search.state as string;
+                const state = url.searchParams.get('state');
                 if (state) {
                     const { s, d, r } = JSON.parse(state);
-                    search.state = s;
-                    url.search = '?' + qs.stringify(search);
+                    url.searchParams.set('state', s);
                     const oauthClient: ScryptedDevice & OauthClient = this.getDevice(d);
                     await oauthClient.onOauthCallback(url.toString()).catch();
                     res.redirect(r);
@@ -197,12 +194,12 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
                 }
             }
             if (url.hash) {
-                const hash = qs.parse(url.hash);
-                const state = hash.state as string;
+                const hash = new URLSearchParams(url.hash.substring(1));
+                const state = hash.get('state');
                 if (state) {
                     const { s, d, r } = JSON.parse(state);
-                    hash.state = s;
-                    url.hash = '#' + qs.stringify(hash);
+                    hash.set('state', s);
+                    url.hash = '#' + hash.toString();
                     const oauthClient: ScryptedDevice & OauthClient = this.getDevice(d);
                     await oauthClient.onOauthCallback(url.toString());
                     res.redirect(r);

package/src/scrypted-server-main.ts

@@ -12,7 +12,6 @@ import { getHostAddresses, SCRYPTED_DEBUG_PORT, SCRYPTED_INSECURE_PORT, SCRYPTED
 import crypto from 'crypto';
 import cookieParser from 'cookie-parser';
 import axios from 'axios';
-import qs from 'query-string';
 import { RPCResultError } from './rpc';
 import fs from 'fs';
 import mkdirp from 'mkdirp';
@@ -318,8 +317,8 @@ async function start() {
 
     app.get('/web/component/script/search', async (req, res) => {
         try {
-            const query = qs.stringify({
-                text: req.query.text,
+            const query = new URLSearchParams({
+                text: req.query.text.toString(),
             })
             const response = await axios(`https://registry.npmjs.org/-/v1/search?${query}`);
             res.send(response.data);
@@ -405,7 +404,12 @@ async function start() {
 
     app.get('/logout', (req, res) => {
         res.clearCookie(getLoginUserToken(req.secure));
-        res.send({});
+        if (req.headers['accept']?.startsWith('application/json')) {
+            res.send({});
+        }
+        else {
+            res.redirect('/endpoint/@scrypted/core/public/');
+        }
     });
 
     let hasLogin = await db.getCount(ScryptedUser) > 0;

package/src/state.ts

@@ -116,7 +116,7 @@ export class ScryptedStateManager extends EventRegistry {
         let cb = (eventDetails: EventDetails, eventData: any) => {
             if (denoise && lastData === eventData)
                 return;
-            callback(eventDetails, eventData);
+            callback?.(eventDetails, eventData);
         };
 
         const wrappedRegister = super.listenDevice(id, options, cb);
@@ -124,6 +124,7 @@ export class ScryptedStateManager extends EventRegistry {
         return new EventListenerRegisterImpl(() => {
             wrappedRegister.removeListener();
             cb = undefined;
+            callback = undefined;
             polling = false;
         });
     }