@scrypted/server 0.0.85 → 0.0.89

package/src/plugin/plugin-device.ts

@@ -9,114 +9,220 @@ import { allInterfaceProperties, isValidInterfaceMethod, methodInterfaces } from
 
 interface MixinTable {
     mixinProviderId: string;
-    interfaces: string[];
-    proxy: Promise<any>;
+    entry: Promise<MixinTableEntry>;
+}
+
+interface MixinTableEntry {
+    interfaces: Set<string>
+    allInterfaces: string[];
+    proxy: any;
+    error?: Error;
 }
 
 export class PluginDeviceProxyHandler implements ProxyHandler<any>, ScryptedDevice {
     scrypted: ScryptedRuntime;
     id: string;
-    mixinTable: Promise<MixinTable[]>;
+    mixinTable: MixinTable[];
 
     constructor(scrypted: ScryptedRuntime, id: string) {
         this.scrypted = scrypted;
         this.id = id;
     }
 
+    invalidateEntry(mixinEntry: MixinTable) {
+        if (!mixinEntry.mixinProviderId)
+            return;
+        (async () => {
+            const mixinProvider = this.scrypted.getDevice(mixinEntry.mixinProviderId) as ScryptedDevice & MixinProvider;
+            const { proxy } = await mixinEntry.entry;
+            mixinProvider?.releaseMixin(this.id, proxy);
+        })().catch(() => { });
+    }
+
     // should this be async?
     invalidate() {
         const mixinTable = this.mixinTable;
         this.mixinTable = undefined;
-        (async () => {
-            for (const mixinEntry of (await mixinTable || [])) {
-                if (!mixinEntry.mixinProviderId)
-                    continue;
-                (async () => {
-                    const mixinProvider = this.scrypted.getDevice(mixinEntry.mixinProviderId) as ScryptedDevice & MixinProvider;
-                    mixinProvider?.releaseMixin(this.id, await mixinEntry.proxy);
-                })().catch(() => { });
-            }
-        })().catch(() => { });;
+        for (const mixinEntry of (mixinTable || [])) {
+            this.invalidateEntry(mixinEntry);
+        }
     }
 
-    // this must not be async, because it potentially changes execution order.
-    ensureProxy(): Promise<PluginDevice> {
+    invalidateMixinTable() {
+        if (!this.mixinTable)
+            return this.invalidate();
+
+        let previousMixinIds = this.mixinTable?.map(entry => entry.mixinProviderId) || [];
+        previousMixinIds.pop();
+        previousMixinIds = previousMixinIds.reverse();
+
         const pluginDevice = this.scrypted.findPluginDeviceById(this.id);
         if (!pluginDevice)
-            throw new Error(`device ${this.id} does not exist`);
+            return this.invalidate();
+
+        const mixins = getState(pluginDevice, ScryptedInterfaceProperty.mixins) || [];
+        // iterate the new mixin table to find the last good mixin,
+        // and resume creation from there.
+        let lastValidMixinId: string;
+        for (const mixinId of mixins) {
+            if (!previousMixinIds.length) {
+                // reached of the previous mixin table, meaning
+                // mixins were added.
+                break;
+            }
+            const check = previousMixinIds.shift();
+            if (check !== mixinId)
+                break;
 
-        if (this.mixinTable)
-            return Promise.resolve(pluginDevice);
+            lastValidMixinId = mixinId;
+        }
 
-        this.mixinTable = (async () => {
-            let proxy: any;
-            if (!pluginDevice.nativeId) {
-                const plugin = this.scrypted.plugins[pluginDevice.pluginId];
-                proxy = await plugin.module;
-            }
-            else {
-                const providerId = getState(pluginDevice, ScryptedInterfaceProperty.providerId);
-                const provider = this.scrypted.getDevice(providerId) as ScryptedDevice & DeviceProvider;
-                proxy = await provider.getDevice(pluginDevice.nativeId);
-            }
+        if (!lastValidMixinId)
+            return this.invalidate();
 
-            if (!proxy)
-                console.error('null proxy', this.id);
-            // after creating an actual device, apply all the mixins
-            const type = getDisplayType(pluginDevice);
-            const allInterfaces: ScryptedInterface[] = getState(pluginDevice, ScryptedInterfaceProperty.providedInterfaces) || [];
+        // invalidate and remove everything up to lastValidMixinId
+        while (true) {
+            const entry = this.mixinTable[0];
+            if (entry.mixinProviderId === lastValidMixinId)
+                break;
+            this.mixinTable.shift();
+            this.invalidateEntry(entry);
+        }
 
-            const mixinTable: MixinTable[] = [];
-            mixinTable.unshift({
-                mixinProviderId: undefined,
-                interfaces: allInterfaces.slice(),
-                proxy,
-            })
+        this.ensureProxy(lastValidMixinId);
+    }
 
-            for (const mixinId of getState(pluginDevice, ScryptedInterfaceProperty.mixins) || []) {
-                const mixinProvider = this.scrypted.getDevice(mixinId) as ScryptedDevice & MixinProvider;
+    // this must not be async, because it potentially changes execution order.
+    ensureProxy(lastValidMixinId?: string): Promise<PluginDevice> {
+        const pluginDevice = this.scrypted.findPluginDeviceById(this.id);
+        if (!pluginDevice)
+            throw new Error(`device ${this.id} does not exist`);
+
+        let previousEntry: Promise<MixinTableEntry>;
+        if (!lastValidMixinId) {
+            if (this.mixinTable)
+                return Promise.resolve(pluginDevice);
+
+            this.mixinTable = [];
 
+            previousEntry = (async () => {
+                let proxy;
                 try {
-                    const interfaces = await mixinProvider?.canMixin(type, allInterfaces) as any as ScryptedInterface[];
-                    if (!interfaces) {
-                        console.warn(`mixin provider ${mixinId} can no longer mixin ${this.id}`);
-                        const mixins: string[] = getState(pluginDevice, ScryptedInterfaceProperty.mixins) || [];
-                        this.scrypted.stateManager.setPluginDeviceState(pluginDevice, ScryptedInterfaceProperty.mixins, mixins.filter(mid => mid !== mixinId))
-                        this.scrypted.datastore.upsert(pluginDevice);
-                        continue;
+                    if (!pluginDevice.nativeId) {
+                        const plugin = this.scrypted.plugins[pluginDevice.pluginId];
+                        proxy = await plugin.module;
+                    }
+                    else {
+                        const providerId = getState(pluginDevice, ScryptedInterfaceProperty.providerId);
+                        const provider = this.scrypted.getDevice(providerId) as ScryptedDevice & DeviceProvider;
+                        proxy = await provider.getDevice(pluginDevice.nativeId);
                     }
 
-                    const wrappedHandler = new PluginDeviceProxyHandler(this.scrypted, this.id);
-                    wrappedHandler.mixinTable = Promise.resolve(mixinTable.slice());
-                    const wrappedProxy = new Proxy(wrappedHandler, wrappedHandler);
-
-                    const host = this.scrypted.getPluginHostForDeviceId(mixinId);
-                    const deviceState = await host.remote.createDeviceState(this.id,
-                        async (property, value) => this.scrypted.stateManager.setPluginDeviceState(pluginDevice, property, value));
-                    const mixinProxy = await mixinProvider.getMixin(wrappedProxy, allInterfaces, deviceState);
-                    if (!mixinProxy)
-                        throw new Error(`mixin provider ${mixinId} did not return mixin for ${this.id}`);
-                    allInterfaces.push(...interfaces);
-                    proxy = mixinProxy;
-
-                    mixinTable.unshift({
-                        mixinProviderId: mixinId,
-                        interfaces,
-                        proxy,
-                    })
+                    if (!proxy)
+                        console.warn('no device was returned by the plugin', this.id);
                 }
                 catch (e) {
-                    console.warn("mixin provider failure", mixinId, e);
+                    console.warn('error occured retrieving device from plugin');
                 }
-            }
 
-            const mixinInterfaces = [...new Set(allInterfaces)].sort();
-            this.scrypted.stateManager.setPluginDeviceState(pluginDevice, ScryptedInterfaceProperty.interfaces, mixinInterfaces);
+                const interfaces: ScryptedInterface[] = getState(pluginDevice, ScryptedInterfaceProperty.providedInterfaces) || [];
+                return {
+                    proxy,
+                    interfaces: new Set<string>(interfaces),
+                    allInterfaces: interfaces,
+                }
+            })();
+
+            this.mixinTable.unshift({
+                mixinProviderId: undefined,
+                entry: previousEntry,
+            });
+        }
+        else {
+            if (!this.mixinTable)
+                throw new Error('mixin table partial invalidation was called with empty mixin table');
+            const prevTable = this.mixinTable.find(table => table.mixinProviderId === lastValidMixinId);
+            if (!prevTable)
+                throw new Error('mixin table partial invalidation was called with invalid lastValidMixinId');
+            previousEntry = prevTable.entry;
+        }
+
+        const type = getDisplayType(pluginDevice);
+
+        for (const mixinId of getState(pluginDevice, ScryptedInterfaceProperty.mixins) || []) {
+            if (lastValidMixinId) {
+                if (mixinId === lastValidMixinId)
+                    lastValidMixinId = undefined;
+                continue;
+            }
 
-            return mixinTable;
-        })();
+            const wrappedMixinTable = this.mixinTable.slice();
+
+            this.mixinTable.unshift({
+                mixinProviderId: mixinId,
+                entry: (async () => {
+                    let { allInterfaces } = await previousEntry;
+                    try {
+                        const mixinProvider = this.scrypted.getDevice(mixinId) as ScryptedDevice & MixinProvider;
+                        const interfaces = await mixinProvider?.canMixin(type, allInterfaces) as any as ScryptedInterface[];
+                        if (!interfaces) {
+                            // this is not an error
+                            // do not advertise interfaces so it is skipped during
+                            // vtable lookup.
+                            console.log(`mixin provider ${mixinId} can no longer mixin ${this.id}`);
+                            const mixins: string[] = getState(pluginDevice, ScryptedInterfaceProperty.mixins) || [];
+                            this.scrypted.stateManager.setPluginDeviceState(pluginDevice, ScryptedInterfaceProperty.mixins, mixins.filter(mid => mid !== mixinId))
+                            this.scrypted.datastore.upsert(pluginDevice);
+                            return {
+                                allInterfaces,
+                                interfaces: new Set<string>(),
+                                proxy: undefined as any,
+                            };
+                        }
+
+                        allInterfaces = allInterfaces.slice();
+                        allInterfaces.push(...interfaces);
+                        const combinedInterfaces = [...new Set(allInterfaces)];
+
+                        const wrappedHandler = new PluginDeviceProxyHandler(this.scrypted, this.id);
+                        wrappedHandler.mixinTable = wrappedMixinTable;
+                        const wrappedProxy = new Proxy(wrappedHandler, wrappedHandler);
+
+                        const host = this.scrypted.getPluginHostForDeviceId(mixinId);
+                        const deviceState = await host.remote.createDeviceState(this.id,
+                            async (property, value) => this.scrypted.stateManager.setPluginDeviceState(pluginDevice, property, value));
+                        const mixinProxy = await mixinProvider.getMixin(wrappedProxy, allInterfaces as ScryptedInterface[], deviceState);
+                        if (!mixinProxy)
+                            throw new Error(`mixin provider ${mixinId} did not return mixin for ${this.id}`);
+
+                        return {
+                            interfaces: new Set<string>(interfaces),
+                            allInterfaces: combinedInterfaces,
+                            proxy: mixinProxy,
+                        };
+                    }
+                    catch (e) {
+                        // on any error, do not advertise interfaces
+                        // on this mixin, so as to prevent total failure?
+                        // this has been the behavior for a while,
+                        // but maybe interfaces implemented by that mixin
+                        // should rethrow the error caught here in applyMixin.
+                        console.warn(e);
+                        return {
+                            allInterfaces,
+                            interfaces: new Set<string>(),
+                            error: e,
+                            proxy: undefined as any,
+                        };
+                    }
+                })(),
+            });
+        }
 
-        return this.mixinTable.then(_ => pluginDevice);
+        return this.mixinTable[0].entry.then(entry => {
+            this.scrypted.stateManager.setPluginDeviceState(pluginDevice, ScryptedInterfaceProperty.interfaces, entry.allInterfaces);
+            return pluginDevice;
+        });
     }
 
     get(target: any, p: PropertyKey, receiver: any): any {
@@ -180,12 +286,14 @@ export class PluginDeviceProxyHandler implements ProxyHandler<any>, ScryptedDevi
         if (!iface)
             throw new Error(`unknown method ${method}`);
 
-        const mixinTable = await this.mixinTable;
-        for (const mixin of mixinTable) {
-
+        for (const mixin of this.mixinTable) {
+            const { interfaces, proxy } = await mixin.entry;
             // this could be null?
-            if (mixin.interfaces.includes(iface))
-                return (await mixin.proxy)[method](...argArray);
+            if (interfaces.has(iface)) {
+                if (!proxy)
+                    throw new Error(`device is unavailable ${this.id} (mixin ${mixin.mixinProviderId})`);
+                return proxy[method](...argArray);
+            }
         }
 
         throw new Error(`${method} not implemented`)

package/src/plugin/plugin-host-api.ts

@@ -47,8 +47,9 @@ export class PluginHostAPI extends PluginAPIManagedListeners implements PluginAP
         const mixins: string[] = getState(device, ScryptedInterfaceProperty.mixins) || [];
         if (!mixins.includes(mixinProvider._id))
             throw new Error(`${mixinProvider._id} is not a mixin provider for ${id}`);
-        const tableEntry = (await this.scrypted.devices[device._id].handler.mixinTable).find(entry => entry.mixinProviderId === mixinProvider._id);
-        if (!tableEntry.interfaces.includes(eventInterface))
+        const tableEntry = this.scrypted.devices[device._id].handler.mixinTable.find(entry => entry.mixinProviderId === mixinProvider._id);
+        const { interfaces } = await tableEntry.entry;
+        if (!interfaces.has(eventInterface))
             throw new Error(`${mixinProvider._id} does not mixin ${eventInterface} for ${id}`);
         this.scrypted.stateManager.notifyInterfaceEvent(device, eventInterface, eventData);
     }

package/src/plugin/plugin-host.ts

@@ -297,6 +297,11 @@ export class PluginHost {
         this.worker.stderr.on('data', data => console.error(data.toString()));
         this.consoleServer = createConsoleServer(this.worker.stdout, this.worker.stderr);
 
+        this.consoleServer.then(cs => {
+            const { pluginConsole } = cs;
+            pluginConsole.log('starting plugin', this.pluginId, this.packageJson.version);
+        });
+
         this.worker.on('disconnect', () => {
             connected = false;
             logger.log('e', `${this.pluginName} disconnected`);
@@ -384,6 +389,11 @@ export function startPluginRemote() {
     }
 
     const getDeviceConsole = (nativeId?: ScryptedNativeId) => {
+        // the the plugin console is simply the default console
+        // and gets read from stderr/stdout.
+        if (!nativeId)
+            return console;
+
         return getConsole(async (stdout, stderr) => {
             const plugins = await api.getComponent('plugins');
             const connect = async () => {
@@ -459,9 +469,9 @@ export function startPluginRemote() {
 
     let _pluginConsole: Console;
     const getPluginConsole = () => {
-        if (_pluginConsole)
-            return _pluginConsole;
-        _pluginConsole = getDeviceConsole(undefined);
+        if (!_pluginConsole)
+            _pluginConsole = getDeviceConsole(undefined);
+        return _pluginConsole;
     }
 
     attachPluginRemote(peer, {
@@ -474,7 +484,7 @@ export function startPluginRemote() {
             pluginId = _pluginId;
             peer.selfName = pluginId;
         },
-        onPluginReady: async(scrypted, params, plugin) => {
+        onPluginReady: async (scrypted, params, plugin) => {
             replPort = createREPLServer(scrypted, params, plugin);
         },
         getPluginConsole,

package/src/plugin/plugin-remote.ts

@@ -4,7 +4,7 @@ import path from 'path';
 import { ScryptedNativeId, DeviceManager, Logger, Device, DeviceManifest, DeviceState, EndpointManager, SystemDeviceState, ScryptedStatic, SystemManager, MediaManager, ScryptedMimeTypes, ScryptedInterface, ScryptedInterfaceProperty, HttpRequest } from '@scrypted/sdk/types'
 import { PluginAPI, PluginLogger, PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
 import { SystemManagerImpl } from './system';
-import { RpcPeer, RPCResultError, PROPERTY_PROXY_ONEWAY_METHODS, PROPERTY_JSON_DISABLE_SERIALIZATION  } from '../rpc';
+import { RpcPeer, RPCResultError, PROPERTY_PROXY_ONEWAY_METHODS, PROPERTY_JSON_DISABLE_SERIALIZATION } from '../rpc';
 import { BufferSerializer } from './buffer-serializer';
 import { EventEmitter } from 'events';
 import { createWebSocketClass } from './plugin-remote-websocket';
@@ -407,7 +407,6 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
 
             async loadZip(packageJson: any, zipData: Buffer, zipOptions?: PluginRemoteLoadZipOptions) {
                 const pluginConsole = getPluginConsole?.();
-                pluginConsole?.log('starting plugin', pluginId, packageJson.version);
                 const zip = new AdmZip(zipData);
                 await options?.onLoadZip?.(zip, packageJson);
                 const main = zip.getEntry('main.nodejs.js');

package/src/runtime.ts

@@ -424,6 +424,15 @@ export class ScryptedRuntime {
         return proxyPair;
     }
 
+    // should this be async?
+    invalidatePluginDeviceMixins(id: string) {
+        const proxyPair = this.devices[id];
+        if (!proxyPair)
+            return;
+        proxyPair.handler.invalidateMixinTable();
+        return proxyPair;
+    }
+
     async installNpm(pkg: string, version?: string): Promise<PluginHost> {
         const registry = (await axios(`https://registry.npmjs.org/${pkg}`)).data;
         if (!version) {

package/src/services/plugin.ts

@@ -29,7 +29,7 @@ export class PluginComponent {
         const pluginDevice = this.scrypted.findPluginDeviceById(id);
         this.scrypted.stateManager.setPluginDeviceState(pluginDevice, ScryptedInterfaceProperty.mixins, [...new Set(mixins)] || []);
         await this.scrypted.datastore.upsert(pluginDevice);
-        const device = this.scrypted.invalidatePluginDevice(id);
+        const device = this.scrypted.invalidatePluginDeviceMixins(id);
         await device?.handler.ensureProxy();
     }
     async getMixins(id: string) {