@scrypted/server 0.0.108 → 0.0.112

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrypted/server",
-  "version": "0.0.108",
+  "version": "0.0.112",
   "description": "",
   "dependencies": {
     "@scrypted/sdk": "^0.0.131",

package/python/plugin-remote.py

@@ -21,7 +21,7 @@ from asyncio.futures import Future
 from asyncio.streams import StreamReader, StreamWriter
 import os
 from os import sys
-from sys import stderr, stdout
+import platform
 
 class SystemDeviceState(TypedDict):
     lastEventTime: int
@@ -164,8 +164,8 @@ class PluginRemote:
 
         zip = zipfile.ZipFile(zipPath)
 
-        python_prefix = os.path.join(os.environ.get(
-            'SCRYPTED_PLUGIN_VOLUME'), 'python')
+        plugin_volume = os.environ.get('SCRYPTED_PLUGIN_VOLUME')
+        python_prefix = os.path.join(plugin_volume, 'python-%s-%s' % (platform.system(), platform.machine()))
         if not os.path.exists(python_prefix):
             os.makedirs(python_prefix)
 
@@ -179,7 +179,7 @@ class PluginRemote:
 
             requirementstxt = os.path.join(python_prefix, 'requirements.txt')
             installed_requirementstxt = os.path.join(
-                python_prefix, 'installed-requirements.txt')
+                python_prefix, 'requirements.installed.txt')
 
             need_pip = True
             try:

package/python/rpc.py

@@ -1,8 +1,10 @@
 from asyncio.futures import Future
-from typing import Callable, Mapping, List
+from typing import Any, Callable, Mapping, List
 import traceback
 import inspect
+from typing_extensions import TypedDict
 import weakref
+import sys
 
 jsonSerializable = set()
 jsonSerializable.add(float)
@@ -45,21 +47,35 @@ class RpcProxyMethod:
         return self.__proxy.__apply__(self.__proxy_method_name, args)
 
 
+class LocalProxiedEntry(TypedDict):
+    id: str
+    finalizerId: str
+
+
 class RpcProxy(object):
-    def __init__(self, peer, proxyId: str, proxyConstructorName: str, proxyProps: any, proxyOneWayMethods: List[str]):
-        self.__dict__['__proxy_id'] = proxyId
+    def __init__(self, peer, entry: LocalProxiedEntry, proxyConstructorName: str, proxyProps: any, proxyOneWayMethods: List[str]):
+        self.__dict__['__proxy_id'] = entry['id']
+        self.__dict__['__proxy_entry'] = entry
         self.__dict__['__proxy_constructor'] = proxyConstructorName
         self.__dict__['__proxy_peer'] = peer
         self.__dict__['__proxy_props'] = proxyProps
         self.__dict__['__proxy_oneway_methods'] = proxyOneWayMethods
 
     def __getattr__(self, name):
+        if name == '__proxy_finalizer_id':
+            return self.dict['__proxy_entry']['finalizerId']
         if name in self.__dict__:
             return self.__dict__[name]
         if self.__dict__['__proxy_props'] and name in self.__dict__['__proxy_props']:
             return self.__dict__['__proxy_props'][name]
         return RpcProxyMethod(self, name)
 
+    def __setattr__(self, name: str, value: Any) -> None:
+        if name == '__proxy_finalizer_id':
+            self.dict['__proxy_entry']['finalizerId'] = value
+
+        return super().__setattr__(name, value)
+
     def __call__(self, *args, **kwargs):
         print('call')
         pass
@@ -69,20 +85,18 @@ class RpcProxy(object):
 
 
 class RpcPeer:
-    # todo: these are all class statics lol, fix this.
-    idCounter = 1
-    peerName = 'Unnamed Peer'
-    params: Mapping[str, any] = {}
-    localProxied: Mapping[any, str] = {}
-    localProxyMap: Mapping[str, any] = {}
-    constructorSerializerMap = {}
-    proxyCounter = 1
-    pendingResults: Mapping[str, Future] = {}
-    remoteWeakProxies: Mapping[str, any] = {}
-    nameDeserializerMap: Mapping[str, RpcSerializer] = {}
-
     def __init__(self, send: Callable[[object, Callable[[Exception], None]], None]) -> None:
         self.send = send
+        self.idCounter = 1
+        self.peerName = 'Unnamed Peer'
+        self.params: Mapping[str, any] = {}
+        self.localProxied: Mapping[any, LocalProxiedEntry] = {}
+        self.localProxyMap: Mapping[str, any] = {}
+        self.constructorSerializerMap = {}
+        self.proxyCounter = 1
+        self.pendingResults: Mapping[str, Future] = {}
+        self.remoteWeakProxies: Mapping[str, any] = {}
+        self.nameDeserializerMap: Mapping[str, RpcSerializer] = {}
 
     def __apply__(self, proxyId: str, oneWayMethods: List[str], method: str, args: list):
         serializedArgs = []
@@ -120,12 +134,17 @@ class RpcPeer:
     def serialize(self, value, requireProxy):
         if (not value or (not requireProxy and type(value) in jsonSerializable)):
             return value
+
         __remote_constructor_name = 'Function' if callable(value) else value.__proxy_constructor if hasattr(
             value, '__proxy_constructor') else type(value).__name__
-        proxyId = self.localProxied.get(value, None)
-        if proxyId:
+
+        proxiedEntry = self.localProxied.get(value, None)
+        if proxiedEntry:
+            proxiedEntry['finalizerId'] = str(self.proxyCounter)
+            self.proxyCounter = self.proxyCounter + 1
             ret = {
-                '__remote_proxy_id': proxyId,
+                '__remote_proxy_id': proxiedEntry['id'],
+                '__remote_proxy_finalizer_id': proxiedEntry['finalizerId'],
                 '__remote_constructor_name': __remote_constructor_name,
                 '__remote_proxy_props': getattr(value, '__proxy_props', None),
                 '__remote_proxy_oneway_methods': getattr(value, '__proxy_oneway_methods', None),
@@ -140,13 +159,15 @@ class RpcPeer:
             }
             return ret
 
-        serializerMapName = self.constructorSerializerMap.get(type(value), None)
+        serializerMapName = self.constructorSerializerMap.get(
+            type(value), None)
         if serializerMapName:
             __remote_constructor_name = serializerMapName
             serializer = self.nameDeserializerMap.get(serializerMapName, None)
             serialized = serializer.serialize(value)
             ret = {
                 '__remote_proxy_id': None,
+                '__remote_proxy_finalizer_id': None,
                 '__remote_constructor_name': __remote_constructor_name,
                 '__remote_proxy_props': getattr(value, '__proxy_props', None),
                 '__remote_proxy_oneway_methods': getattr(value, '__proxy_oneway_methods', None),
@@ -156,11 +177,16 @@ class RpcPeer:
 
         proxyId = str(self.proxyCounter)
         self.proxyCounter = self.proxyCounter + 1
-        self.localProxied[value] = proxyId
+        proxiedEntry = {
+            'id': proxyId,
+            'finalizerId': proxyId,
+        }
+        self.localProxied[value] = proxiedEntry
         self.localProxyMap[proxyId] = value
 
         ret = {
             '__remote_proxy_id': proxyId,
+            '__remote_proxy_finalizer_id': proxyId,
             '__remote_constructor_name': __remote_constructor_name,
             '__remote_proxy_props': getattr(value, '__proxy_props', None),
             '__remote_proxy_oneway_methods': getattr(value, '__proxy_oneway_methods', None),
@@ -168,20 +194,26 @@ class RpcPeer:
 
         return ret
 
-    def finalize(self, id: str):
+    def finalize(self, localProxiedEntry: LocalProxiedEntry):
+        id = localProxiedEntry['id']
         self.remoteWeakProxies.pop(id, None)
         rpcFinalize = {
             '__local_proxy_id': id,
+            '__local_proxy_finalizer_id': localProxiedEntry['finalizerId'],
             'type': 'finalize',
         }
         self.send(rpcFinalize)
 
     def newProxy(self, proxyId: str, proxyConstructorName: str, proxyProps: any, proxyOneWayMethods: List[str]):
-        proxy = RpcProxy(self, proxyId, proxyConstructorName,
+        localProxiedEntry: LocalProxiedEntry = {
+            'id': proxyId,
+            'finalizerId': None,
+        }
+        proxy = RpcProxy(self, localProxiedEntry, proxyConstructorName,
                          proxyProps, proxyOneWayMethods)
         wr = weakref.ref(proxy)
         self.remoteWeakProxies[proxyId] = wr
-        weakref.finalize(proxy, lambda: self.finalize(proxyId))
+        weakref.finalize(proxy, lambda: self.finalize(localProxiedEntry))
         return proxy
 
     def deserialize(self, value):
@@ -192,6 +224,8 @@ class RpcPeer:
             return value
 
         __remote_proxy_id = value.get('__remote_proxy_id', None)
+        __remote_proxy_finalizer_id = value.get(
+            '__remote_proxy_finalizer_id', None)
         __local_proxy_id = value.get('__local_proxy_id', None)
         __remote_constructor_name = value.get(
             '__remote_constructor_name', None)
@@ -206,6 +240,7 @@ class RpcPeer:
             if not proxy:
                 proxy = self.newProxy(__remote_proxy_id, __remote_constructor_name,
                                       __remote_proxy_props, __remote_proxy_oneway_methods)
+            proxy.__proxy_finalizer_id = __remote_proxy_finalizer_id
             return proxy
 
         if __local_proxy_id:
@@ -297,9 +332,16 @@ class RpcPeer:
                 future.set_result(self.deserialize(
                     message.get('result', None)))
             elif messageType == 'finalize':
-                local = self.localProxyMap.pop(
-                    message['__local_proxy_id'], None)
-                self.localProxied.pop(local, None)
+                finalizerId = message.get('__local_proxy_finalizer_id', None)
+                proxyId = message['__local_proxy_id']
+                local = self.localProxyMap.get(proxyId, None)
+                if local:
+                    localProxiedEntry = self.localProxied.get(local)
+                    if localProxiedEntry and finalizerId and localProxiedEntry['finalizerId'] != finalizerId:
+                        print('mismatch finalizer id', file=sys.stderr)
+                        return
+                    self.localProxied.pop(local, None)
+                    local = self.localProxyMap.pop(proxyId, None)
             else:
                 raise RpcResultException(
                     None, 'unknown rpc message type %s' % type)

package/src/event-registry.ts

@@ -12,12 +12,12 @@ export class EventListenerRegisterImpl implements EventListenerRegister {
 const allowedEventInterfaces = new Set<string>([ScryptedInterface.ScryptedDevice, 'Logger', 'Storage'])
 
 export class EventRegistry {
-    systemListeners = new Set<(id: string, eventDetails: EventDetails, eventData: object) => void>();
-    listeners: { [token: string]: Set<(eventDetails: EventDetails, eventData: object) => void> } = {};
+    systemListeners = new Set<(id: string, eventDetails: EventDetails, eventData: any) => void>();
+    listeners: { [token: string]: Set<(eventDetails: EventDetails, eventData: any) => void> } = {};
 
-    listen(EventListener: (id: string, eventDetails: EventDetails, eventData: object) => void): EventListenerRegister {
+    listen(callback: (id: string, eventDetails: EventDetails, eventData: any) => void): EventListenerRegister {
         const events = this.systemListeners;
-        let cb = EventListener;
+        let cb = callback;
         events.add(cb);
         return new EventListenerRegisterImpl(() => {
             events.delete(cb);
@@ -25,7 +25,7 @@ export class EventRegistry {
         });
     }
 
-    listenDevice(id: string, options: string | EventListenerOptions, callback: (eventDetails: EventDetails, eventData: object) => void): EventListenerRegister {
+    listenDevice(id: string, options: string | EventListenerOptions, callback: (eventDetails: EventDetails, eventData: any) => void): EventListenerRegister {
         let { event } = (options || {}) as EventListenerOptions;
         if (!event && typeof options === 'string')
             event = options as string;

package/src/plugin/plugin-api.ts

@@ -20,7 +20,7 @@ export interface PluginAPI {
     setDeviceProperty(id: string, property: ScryptedInterfaceProperty, value: any): Promise<void>;
     removeDevice(id: string): Promise<void>;
     listen(EventListener: (id: string, eventDetails: EventDetails, eventData: any) => void): Promise<EventListenerRegister>;
-    listenDevice(id: string, event: string | EventListenerOptions, callback: (eventDetails: EventDetails, eventData: object) => void): Promise<EventListenerRegister>;
+    listenDevice(id: string, event: string | EventListenerOptions, callback: (eventDetails: EventDetails, eventData: any) => void): Promise<EventListenerRegister>;
 
     ioClose(id: string): Promise<void>;
     ioSend(id: string, message: string): Promise<void>;
@@ -101,10 +101,10 @@ export class PluginAPIProxy extends PluginAPIManagedListeners implements PluginA
     removeDevice(id: string): Promise<void> {
         return this.api.removeDevice(id);
     }
-    async listen(EventListener: (id: string, eventDetails: EventDetails, eventData: any) => void): Promise<EventListenerRegister> {
-        return this.manageListener(await this.api.listen(EventListener));
+    async listen(callback: (id: string, eventDetails: EventDetails, eventData: any) => void): Promise<EventListenerRegister> {
+        return this.manageListener(await this.api.listen(callback));
     }
-    async listenDevice(id: string, event: string | EventListenerOptions, callback: (eventDetails: EventDetails, eventData: object) => void): Promise<EventListenerRegister> {
+    async listenDevice(id: string, event: string | EventListenerOptions, callback: (eventDetails: EventDetails, eventData: any) => void): Promise<EventListenerRegister> {
         return this.manageListener(await this.api.listenDevice(id, event, callback));
     }
     ioClose(id: string): Promise<void> {

package/src/plugin/plugin-device.ts

@@ -54,6 +54,7 @@ export class PluginDeviceProxyHandler implements ProxyHandler<any>, ScryptedDevi
     async isMixin(id: string, mixinDevice: any) {
         if (this.releasing.has(mixinDevice))
             return true;
+        await this.scrypted.devices[id].handler.ensureProxy();
         for (const mixin of this.scrypted.devices[id].handler.mixinTable) {
             const { proxy } = await mixin.entry;
             if (proxy === mixinDevice) {
@@ -298,7 +299,7 @@ export class PluginDeviceProxyHandler implements ProxyHandler<any>, ScryptedDevi
         return new Proxy(() => prop, this);
     }
 
-    listen(event: string | EventListenerOptions, callback: (eventSource: ScryptedDevice, eventDetails: EventDetails, eventData: object) => void): EventListenerRegister {
+    listen(event: string | EventListenerOptions, callback: (eventSource: ScryptedDevice, eventDetails: EventDetails, eventData: any) => void): EventListenerRegister {
         return this.scrypted.stateManager.listenDevice(this.id, event, (eventDetails, eventData) => callback(this.scrypted.getDevice(this.id), eventDetails, eventData));
     }
     async setName(name: string): Promise<void> {

package/src/plugin/plugin-host-api.ts

@@ -47,22 +47,23 @@ export class PluginHostAPI extends PluginAPIManagedListeners implements PluginAP
         const device = this.scrypted.findPluginDeviceById(id);
 
         if (!nativeIdOrMixinDevice || typeof nativeIdOrMixinDevice === 'string') {
+            const nativeId: string = nativeIdOrMixinDevice;
             // todo: deprecate this code path
-            const mixinProvider = this.scrypted.findPluginDevice(this.pluginId, nativeIdOrMixinDevice);
+            const mixinProvider = this.scrypted.findPluginDevice(this.pluginId, nativeId);
             const mixins: string[] = getState(device, ScryptedInterfaceProperty.mixins) || [];
             if (!mixins.includes(mixinProvider._id))
                 throw new Error(`${mixinProvider._id} is not a mixin provider for ${id}`);
 
-            this.scrypted.findPluginDevice(this.pluginId, nativeIdOrMixinDevice);
+            this.scrypted.findPluginDevice(this.pluginId, nativeId);
             const tableEntry = this.scrypted.devices[device._id].handler.mixinTable.find(entry => entry.mixinProviderId === mixinProvider._id);
             const { interfaces } = await tableEntry.entry;
             if (!interfaces.has(eventInterface))
                 throw new Error(`${mixinProvider._id} does not mixin ${eventInterface} for ${id}`);
         }
         else {
-            if (!await this.scrypted.devices[device._id]?.handler?.isMixin(id, nativeIdOrMixinDevice)) {
-                const mixinProvider = this.scrypted.findPluginDevice(this.pluginId, nativeIdOrMixinDevice);
-                throw new Error(`${mixinProvider?._id} does not mixin ${eventInterface} for ${id}`);
+            const mixin: object = nativeIdOrMixinDevice;
+            if (!await this.scrypted.devices[device._id]?.handler?.isMixin(id, mixin)) {
+                throw new Error(`${mixin} does not mixin ${eventInterface} for ${id}`);
             }
         }
         this.scrypted.stateManager.notifyInterfaceEvent(device, eventInterface, eventData);
@@ -121,7 +122,8 @@ export class PluginHostAPI extends PluginAPIManagedListeners implements PluginAP
     }
 
     async onDevicesChanged(deviceManifest: DeviceManifest) {
-        const existing = this.scrypted.findPluginDevices(this.pluginId).filter(p => p.nativeId == deviceManifest.providerNativeId);
+        const provider = this.scrypted.findPluginDevice(this.pluginId, deviceManifest.providerNativeId);
+        const existing = this.scrypted.findPluginDevices(this.pluginId).filter(p => p.state[ScryptedInterfaceProperty.providerId].value === provider._id);
         const newIds = deviceManifest.devices.map(device => device.nativeId);
         const toRemove = existing.filter(e => e.nativeId && !newIds.includes(e.nativeId));
 
@@ -150,10 +152,10 @@ export class PluginHostAPI extends PluginAPIManagedListeners implements PluginAP
     async getDeviceById<T>(id: string): Promise<T & ScryptedDevice> {
         return this.scrypted.getDevice(id);
     }
-    async listen(listener: (id: string, eventDetails: EventDetails, eventData: any) => void): Promise<EventListenerRegister> {
-        return this.manageListener(this.scrypted.stateManager.listen(listener));
+    async listen(callback: (id: string, eventDetails: EventDetails, eventData: any) => void): Promise<EventListenerRegister> {
+        return this.manageListener(this.scrypted.stateManager.listen(callback));
     }
-    async listenDevice(id: string, event: string | EventListenerOptions, callback: (eventDetails: EventDetails, eventData: object) => void): Promise<EventListenerRegister> {
+    async listenDevice(id: string, event: string | EventListenerOptions, callback: (eventDetails: EventDetails, eventData: any) => void): Promise<EventListenerRegister> {
         const device = this.scrypted.findPluginDeviceById(id);
         if (device) {
             const self = this.scrypted.findPluginDevice(this.pluginId);

package/src/plugin/plugin-npm-dependencies.ts

@@ -3,14 +3,18 @@ import fs from 'fs';
 import child_process from 'child_process';
 import path from 'path';
 import { once } from 'events';
+import process from 'process';
+import mkdirp from "mkdirp";
 
 export async function installOptionalDependencies(console: Console, packageJson: any) {
     const pluginVolume = ensurePluginVolume(packageJson.name);
-    const optPj = path.join(pluginVolume, 'package.json');
+    const nodePrefix = path.join(pluginVolume, `${process.platform}-${process.arch}`);
+    const packageJsonPath = path.join(nodePrefix, 'package.json');
+    const currentInstalledPackageJsonPath = path.join(nodePrefix, 'package.installed.json');
 
     let currentPackageJson: any;
     try {
-        currentPackageJson = JSON.parse(fs.readFileSync(optPj).toString());
+        currentPackageJson = JSON.parse(fs.readFileSync(currentInstalledPackageJsonPath).toString());
     }
     catch (e) {
     }
@@ -34,21 +38,18 @@ export async function installOptionalDependencies(console: Console, packageJson:
     delete reduced.optionalDependencies;
     delete reduced.devDependencies;
 
-    try {
-        fs.writeFileSync(optPj, JSON.stringify(reduced));
-
-        const cp = child_process.spawn('npm', ['--prefix', pluginVolume, 'install'], {
-            cwd: pluginVolume,
-            stdio: 'inherit',
-        });
-    
-        await once(cp, 'exit');
-        if (cp.exitCode !== 0)
-            throw new Error('npm installation failed with exit code ' + cp.exitCode);
-    }
-    catch (e) {
-        fs.rmSync(optPj);
-        throw e;
-    }
+    mkdirp.sync(nodePrefix);
+    fs.writeFileSync(packageJsonPath, JSON.stringify(reduced));
+
+    const cp = child_process.spawn('npm', ['--prefix', nodePrefix, 'install'], {
+        cwd: nodePrefix,
+        stdio: 'inherit',
+    });
+
+    await once(cp, 'exit');
+    if (cp.exitCode !== 0)
+        throw new Error('npm installation failed with exit code ' + cp.exitCode);
+
+    fs.writeFileSync(currentInstalledPackageJsonPath, JSON.stringify(reduced));
     console.log('native dependencies installed.');
 }

package/src/plugin/system.ts

@@ -1,6 +1,6 @@
 import { EventListenerOptions, EventDetails, EventListenerRegister, ScryptedDevice, ScryptedInterface, ScryptedInterfaceDescriptors, SystemDeviceState, SystemManager, ScryptedInterfaceProperty, ScryptedDeviceType, Logger } from "@scrypted/sdk/types";
 import { PluginAPI } from "./plugin-api";
-import { handleFunctionInvocations } from '../rpc';
+import { handleFunctionInvocations, PROPERTY_PROXY_ONEWAY_METHODS } from '../rpc';
 import { EventRegistry } from "../event-registry";
 import { allInterfaceProperties, isValidInterfaceMethod } from "./descriptor";
 
@@ -56,7 +56,7 @@ class DeviceProxyHandler implements ProxyHandler<any>, ScryptedDevice {
         return (this.device as any)[method](...argArray);
     }
 
-    listen(event: string | EventListenerOptions, callback: (eventSource: ScryptedDevice, eventDetails: EventDetails, eventData: object) => void): EventListenerRegister {
+    listen(event: string | EventListenerOptions, callback: (eventSource: ScryptedDevice, eventDetails: EventDetails, eventData: any) => void): EventListenerRegister {
         return this.systemManager.listenDevice(this.id, event, callback);
     }
 
@@ -92,6 +92,15 @@ class EventListenerRegisterImpl implements EventListenerRegister {
     }
 }
 
+function makeOneWayCallback<T>(input: T): T {
+    const f: any = input;
+    const oneways: string[] = f[PROPERTY_PROXY_ONEWAY_METHODS] || [];
+    if (!oneways.includes(null))
+        oneways.push(null);
+    f[PROPERTY_PROXY_ONEWAY_METHODS] = oneways;
+    return input;
+}
+
 export class SystemManagerImpl implements SystemManager {
     api: PluginAPI;
     state: {[id: string]: {[property: string]: SystemDeviceState}};
@@ -122,10 +131,10 @@ export class SystemManagerImpl implements SystemManager {
                 return this.getDeviceById(id);
         }
     }
-    listen(EventListener: (eventSource: ScryptedDevice, eventDetails: EventDetails, eventData: object) => void): EventListenerRegister {
-        return this.events.listen((id, eventDetails, eventData) => EventListener(this.getDeviceById(id), eventDetails, eventData));
+    listen(callback: (eventSource: ScryptedDevice, eventDetails: EventDetails, eventData: any) => void): EventListenerRegister {
+        return this.events.listen(makeOneWayCallback((id, eventDetails, eventData) => callback(this.getDeviceById(id), eventDetails, eventData)));
     }
-    listenDevice(id: string, options: string | EventListenerOptions, callback: (eventSource: ScryptedDevice, eventDetails: EventDetails, eventData: object) => void): EventListenerRegister {
+    listenDevice(id: string, options: string | EventListenerOptions, callback: (eventSource: ScryptedDevice, eventDetails: EventDetails, eventData: any) => void): EventListenerRegister {
         let { event, watch } = (options || {}) as EventListenerOptions;
         if (!event && typeof options === 'string')
             event = options as string;
@@ -136,7 +145,7 @@ export class SystemManagerImpl implements SystemManager {
         if (watch)
             return this.events.listenDevice(id, event, (eventDetails, eventData) => callback(this.getDeviceById(id), eventDetails, eventData));
 
-        return new EventListenerRegisterImpl(this.api.listenDevice(id, options, (eventDetails, eventData) => callback(this.getDeviceById(id), eventDetails, eventData)))
+        return new EventListenerRegisterImpl(this.api.listenDevice(id, options, makeOneWayCallback((eventDetails, eventData) => callback(this.getDeviceById(id), eventDetails, eventData))));
     }
 
     async removeDevice(id: string) {