@scrypted/server 0.1.16 → 0.2.5

package/src/plugin/media.ts

@@ -1,7 +1,5 @@
-import { BufferConverter, BufferConvertorOptions, DeviceManager, FFmpegInput, MediaManager, MediaObject, MediaObjectOptions, MediaStreamUrl, ScryptedInterface, ScryptedInterfaceProperty, ScryptedMimeTypes, ScryptedNativeId, SystemDeviceState, SystemManager } from "@scrypted/types";
+import { BufferConverter, DeviceManager, FFmpegInput, MediaManager, MediaObject, MediaObjectOptions, MediaStreamUrl, ScryptedInterface, ScryptedInterfaceProperty, ScryptedMimeTypes, ScryptedNativeId, SystemDeviceState, SystemManager } from "@scrypted/types";
 import axios from 'axios';
-import child_process from 'child_process';
-import { once } from 'events';
 import pathToFfmpeg from 'ffmpeg-static';
 import fs from 'fs';
 import https from 'https';
@@ -11,7 +9,6 @@ import Graph from 'node-dijkstra';
 import os from 'os';
 import path from 'path';
 import MimeType from 'whatwg-mimetype';
-import { safeKillFFmpeg } from '../media-helpers';
 import { MediaObjectRemote } from "./plugin-api";
 
 function typeMatches(target: string, candidate: string): boolean {
@@ -27,15 +24,28 @@ function mimeMatches(target: MimeType, candidate: MimeType) {
 
 const httpsAgent = new https.Agent({
     rejectUnauthorized: false
-})
+});
+
+type IdBufferConverter = BufferConverter & {
+    id: string;
+};
+
+function getBuiltinId(n: number) {
+    return 'builtin-' + n;
+}
+
+function getExtraId(n: number) {
+    return 'extra-' + n;
+}
 
 export abstract class MediaManagerBase implements MediaManager {
-    builtinConverters: BufferConverter[] = [];
-    extraConverters: BufferConverter[] = [];
+    builtinConverters: IdBufferConverter[] = [];
+    extraConverters: IdBufferConverter[] = [];
 
     constructor() {
         for (const h of ['http', 'https']) {
             this.builtinConverters.push({
+                id: getBuiltinId(this.builtinConverters.length),
                 fromMimeType: ScryptedMimeTypes.SchemePrefix + h,
                 toMimeType: ScryptedMimeTypes.MediaObject,
                 convert: async (data, fromMimeType, toMimeType) => {
@@ -51,18 +61,32 @@ export abstract class MediaManagerBase implements MediaManager {
         }
 
         this.builtinConverters.push({
+            id: getBuiltinId(this.builtinConverters.length),
             fromMimeType: ScryptedMimeTypes.SchemePrefix + 'file',
             toMimeType: ScryptedMimeTypes.MediaObject,
             convert: async (data, fromMimeType, toMimeType) => {
-                const filename = data.toString();
+                const url = data.toString();
+                const filename = url.substring('file:'.length);
+
+                if (toMimeType === ScryptedMimeTypes.FFmpegInput) {
+                    const ffmpegInput: FFmpegInput = {
+                        url,
+                        inputArguments: [
+                            '-i', filename,
+                        ]
+                    };
+                    return this.createFFmpegMediaObject(ffmpegInput);
+                }
+
                 const ab = await fs.promises.readFile(filename);
-                const mt = mimeType.lookup(data.toString());
+                const mt = mimeType.getType(data.toString());
                 const mo = this.createMediaObject(ab, mt);
                 return mo;
             }
         });
 
         this.builtinConverters.push({
+            id: getBuiltinId(this.builtinConverters.length),
             fromMimeType: ScryptedMimeTypes.Url,
             toMimeType: ScryptedMimeTypes.FFmpegInput,
             async convert(data, fromMimeType): Promise<Buffer> {
@@ -79,6 +103,7 @@ export abstract class MediaManagerBase implements MediaManager {
         });
 
         this.builtinConverters.push({
+            id: getBuiltinId(this.builtinConverters.length),
             fromMimeType: ScryptedMimeTypes.FFmpegInput,
             toMimeType: ScryptedMimeTypes.MediaStreamUrl,
             async convert(data: Buffer, fromMimeType: string): Promise<Buffer> {
@@ -87,6 +112,7 @@ export abstract class MediaManagerBase implements MediaManager {
         });
 
         this.builtinConverters.push({
+            id: getBuiltinId(this.builtinConverters.length),
             fromMimeType: ScryptedMimeTypes.MediaStreamUrl,
             toMimeType: ScryptedMimeTypes.FFmpegInput,
             async convert(data, fromMimeType: string): Promise<Buffer> {
@@ -111,66 +137,19 @@ export abstract class MediaManagerBase implements MediaManager {
             }
         });
 
+        // todo: move this to snapshot plugin
         this.builtinConverters.push({
+            id: getBuiltinId(this.builtinConverters.length),
             fromMimeType: 'image/*',
             toMimeType: 'image/*',
             convert: async (data, fromMimeType: string): Promise<Buffer> => {
                 return data as Buffer;
             }
         });
-
-        this.builtinConverters.push({
-            fromMimeType: ScryptedMimeTypes.FFmpegInput,
-            toMimeType: 'image/jpeg',
-            convert: async (data, fromMimeType: string, toMimeType: string, options?: BufferConvertorOptions): Promise<Buffer> => {
-                const console = this.getMixinConsole(options?.sourceId, undefined);
-
-                const mt = new MimeType(toMimeType);
-
-                const ffInput: FFmpegInput = JSON.parse(data.toString());
-
-                const args = [
-                    '-hide_banner',
-                    '-y',
-                ];
-                args.push(...ffInput.inputArguments);
-
-                const width = parseInt(mt.parameters.get('width'));
-                const height = parseInt(mt.parameters.get('height'));
-
-                if (mt.parameters.get('width') || mt.parameters.get('height')) {
-                    args.push(
-                        '-vf', `scale=${width || -1}:${height || -1}`,
-                    );
-                }
-
-                args.push("-vframes", "1", '-f', 'image2', 'pipe:3');
-
-                const buffers: Buffer[] = [];
-
-                const cp = child_process.spawn(await this.getFFmpegPath(), args, {
-                    stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
-                });
-                console.log('converting ffmpeg input to image.');
-                // ffmpegLogInitialOutput(console, cp);
-                cp.on('error', (code) => {
-                    console.error('ffmpeg error code', code);
-                });
-                cp.stdio[3].on('data', data => buffers.push(data));
-                const to = setTimeout(() => {
-                    console.log('ffmpeg stream to image convesion timed out.');
-                    safeKillFFmpeg(cp);
-                }, 10000);
-                const [exitCode] = await once(cp, 'exit');
-                clearTimeout(to);
-                if (exitCode)
-                    throw new Error(`ffmpeg stream to image convesion failed with exit code: ${exitCode}`);
-                return Buffer.concat(buffers);
-            }
-        });
     }
 
-    async addConverter(converter: BufferConverter): Promise<void> {
+    async addConverter(converter: IdBufferConverter): Promise<void> {
+        converter.id = getExtraId(this.extraConverters.length);
         this.extraConverters.push(converter);
     }
 
@@ -219,10 +198,10 @@ export abstract class MediaManagerBase implements MediaManager {
         return ret;
     }
 
-    getConverters(): BufferConverter[] {
+    getConverters(): IdBufferConverter[] {
         const converters = Object.entries(this.getSystemState())
             .filter(([id, state]) => state[ScryptedInterfaceProperty.interfaces]?.value?.includes(ScryptedInterface.BufferConverter))
-            .map(([id]) => this.getDeviceById<BufferConverter>(id));
+            .map(([id]) => this.getDeviceById<IdBufferConverter>(id));
 
         // builtins should be after system converters. these should not be overriden by system,
         // as it could cause system instability with misconfiguration.
@@ -236,7 +215,7 @@ export abstract class MediaManagerBase implements MediaManager {
 
     ensureMediaObjectRemote(mediaObject: string | MediaObject): MediaObjectRemote {
         if (typeof mediaObject === 'string') {
-            const mime = mimeType.lookup(mediaObject);
+            const mime = mimeType.getType(mediaObject);
             return this.createMediaObjectRemote(mediaObject, mime);
         }
         return mediaObject as MediaObjectRemote;
@@ -327,7 +306,7 @@ export abstract class MediaManagerBase implements MediaManager {
         return this.createMediaObjectRemote(data, mimeType, options);
     }
 
-    async convert(converters: BufferConverter[], mediaObject: MediaObjectRemote, toMimeType: string): Promise<{ data: Buffer | string | any, mimeType: string }> {
+    async convert(converters: IdBufferConverter[], mediaObject: MediaObjectRemote, toMimeType: string): Promise<{ data: Buffer | string | any, mimeType: string }> {
         // console.log('converting', mediaObject.mimeType, toMimeType);
         const mediaMime = new MimeType(mediaObject.mimeType);
         const outputMime = new MimeType(toMimeType);
@@ -344,13 +323,9 @@ export abstract class MediaManagerBase implements MediaManager {
             sourceId = this.getPluginDeviceId();
         const console = this.getMixinConsole(sourceId, undefined);
 
-        const converterIds = new Map<BufferConverter, string>();
-        const converterReverseids = new Map<string, BufferConverter>();
-        let id = 0;
-        for (const converter of converters) {
-            const cid = (id++).toString();
-            converterIds.set(converter, cid);
-            converterReverseids.set(cid, converter);
+        const converterMap = new Map<string, IdBufferConverter>();
+        for (const c of converters) {
+            converterMap.set(c.id, c);
         }
 
         const nodes: any = {};
@@ -361,33 +336,36 @@ export abstract class MediaManagerBase implements MediaManager {
             try {
                 const inputMime = new MimeType(converter.fromMimeType);
                 const convertedMime = new MimeType(converter.toMimeType);
-                const targetId = converterIds.get(converter);
+                // catch all converters should be heavily weighted so as not to use them.
+                const inputWeight = parseFloat(inputMime.parameters.get('converter-weight')) || (inputMime.essence === '*/*' ? 1000 : 1);
+                // const convertedWeight = parseFloat(convertedMime.parameters.get('converter-weight')) || (convertedMime.essence === ScryptedMimeTypes.MediaObject ? 1000 : 1);
+                // const conversionWeight = inputWeight + convertedWeight;
+                const targetId = converter.id;
                 const node: any = nodes[targetId] = {};
+
+                // edge matches
                 for (const candidate of converters) {
                     try {
                         const candidateMime = new MimeType(candidate.fromMimeType);
                         if (!mimeMatches(convertedMime, candidateMime))
                             continue;
-                        const candidateId = converterIds.get(candidate);
-                        node[candidateId] = 1;
+                        const outputWeight = parseFloat(candidateMime.parameters.get('converter-weight')) || (candidateMime.essence === '*/*' ? 1000 : 1);
+                        const candidateId = candidate.id;
+                        node[candidateId] = inputWeight + outputWeight;
                     }
                     catch (e) {
                         console.warn('skipping converter due to error', e)
                     }
                 }
 
-                // edge matches
+                // source matches
                 if (mimeMatches(mediaMime, inputMime)) {
-                    const weight = parseFloat(inputMime.parameters.get('converter-weight'));
-                    // catch all converters should be heavily weighted so as not to use them.
-                    mediaNode[targetId] = weight || (inputMime.essence === '*/*' ? 1000 : 1);
+                    mediaNode[targetId] = inputWeight;
                 }
 
                 // target output matches
                 if (mimeMatches(outputMime, convertedMime) || converter.toMimeType === ScryptedMimeTypes.MediaObject) {
-                    const weight = parseFloat(inputMime.parameters.get('converter-weight'));
-                    // catch all converters should be heavily weighted so as not to use them.
-                    node['output'] = weight || (convertedMime.essence === ScryptedMimeTypes.MediaObject ? 1000 : 1);
+                    node['output'] = inputWeight;
                 }
             }
             catch (e) {
@@ -412,7 +390,7 @@ export abstract class MediaManagerBase implements MediaManager {
 
         while (route.length) {
             const node = route.shift();
-            const converter = converterReverseids.get(node);
+            const converter = converterMap.get(node);
             const converterToMimeType = new MimeType(converter.toMimeType);
             const converterFromMimeType = new MimeType(converter.fromMimeType);
             const type = converterToMimeType.type === '*' ? valueMime.type : converterToMimeType.type;
@@ -449,6 +427,15 @@ export abstract class MediaManagerBase implements MediaManager {
 export class MediaManagerImpl extends MediaManagerBase {
     constructor(public systemManager: SystemManager, public deviceManager: DeviceManager) {
         super();
+
+        this.builtinConverters.push({
+            id: getBuiltinId(this.builtinConverters.length),
+            fromMimeType: ScryptedMimeTypes.ScryptedDeviceId,
+            toMimeType: ScryptedMimeTypes.ScryptedDevice,
+            convert: async (data, fromMimeType, toMimeType) => {
+                return this.getDeviceById(data.toString());
+            }
+        });
     }
 
     getSystemState(): { [id: string]: { [property: string]: SystemDeviceState; }; } {

package/src/plugin/plugin-api.ts

@@ -11,8 +11,8 @@ export interface PluginAPI {
     setState(nativeId: ScryptedNativeId, key: string, value: any): Promise<void>;
     onDevicesChanged(deviceManifest: DeviceManifest): Promise<void>;
     onDeviceDiscovered(device: Device): Promise<string>;
-    onDeviceEvent(nativeId: ScryptedNativeId, eventInterface: any, eventData?: any): Promise<void>;
-    onMixinEvent(id: string, nativeId: ScryptedNativeId, eventInterface: any, eventData?: any): Promise<void>;
+    onDeviceEvent(nativeId: ScryptedNativeId, eventInterface: string, eventData?: any): Promise<void>;
+    onMixinEvent(id: string, nativeId: ScryptedNativeId, eventInterface: string, eventData?: any): Promise<void>;
     onDeviceRemoved(nativeId: string): Promise<void>;
     setStorage(nativeId: string, storage: {[key: string]: any}): Promise<void>;
 
@@ -89,8 +89,8 @@ export class PluginAPIProxy extends PluginAPIManagedListeners implements PluginA
     onDeviceEvent(nativeId: ScryptedNativeId, eventInterface: any, eventData?: any): Promise<void> {
         return this.api.onDeviceEvent(nativeId, eventInterface, eventData);
     }
-    onMixinEvent(id: string, nativeId: ScryptedNativeId, eventInterface: any, eventData?: any): Promise<void> {
-        return this.api.onMixinEvent(nativeId, eventInterface, eventData);
+    onMixinEvent(id: string, nativeId: ScryptedNativeId, eventInterface: string, eventData?: any): Promise<void> {
+        return this.api.onMixinEvent(id, nativeId, eventInterface, eventData);
     }
     onDeviceRemoved(nativeId: string): Promise<void> {
         return this.api.onDeviceRemoved(nativeId);

package/src/plugin/plugin-device.ts

@@ -202,7 +202,9 @@ export class PluginDeviceProxyHandler implements PrimitiveProxyHandler<any>, Scr
         }
 
         return this.mixinTable[0].entry.then(entry => {
-            this.scrypted.stateManager.setPluginDeviceState(pluginDevice, ScryptedInterfaceProperty.interfaces, PluginDeviceProxyHandler.sortInterfaces(entry.allInterfaces));
+            const changed = this.scrypted.stateManager.setPluginDeviceState(pluginDevice, ScryptedInterfaceProperty.interfaces, PluginDeviceProxyHandler.sortInterfaces(entry.allInterfaces));
+            if (changed)
+                this.scrypted.notifyPluginDeviceDescriptorChanged(pluginDevice);
             return pluginDevice;
         });
     }
@@ -216,15 +218,25 @@ export class PluginDeviceProxyHandler implements PrimitiveProxyHandler<any>, Scr
         let { allInterfaces } = await previousEntry;
         try {
             const mixinProvider = this.scrypted.getDevice(mixinId) as ScryptedDevice & MixinProvider;
-            const interfaces = mixinProvider?.interfaces?.includes(ScryptedInterface.MixinProvider) && await mixinProvider?.canMixin(type, allInterfaces) as any as ScryptedInterface[];
+            const isMixinProvider = mixinProvider?.interfaces?.includes(ScryptedInterface.MixinProvider);
+            const interfaces = isMixinProvider && await mixinProvider?.canMixin(type, allInterfaces) as any as ScryptedInterface[];
             if (!interfaces) {
                 // this is not an error
                 // do not advertise interfaces so it is skipped during
                 // vtable lookup.
-                console.log(`mixin provider ${mixinId} can no longer mixin ${this.id}`);
-                const mixins: string[] = getState(pluginDevice, ScryptedInterfaceProperty.mixins) || [];
-                this.scrypted.stateManager.setPluginDeviceState(pluginDevice, ScryptedInterfaceProperty.mixins, mixins.filter(mid => mid !== mixinId))
-                this.scrypted.datastore.upsert(pluginDevice);
+                if (!mixinProvider || (isMixinProvider && !interfaces)) {
+                    console.log(`Mixin provider ${mixinId} can no longer mixin ${this.id}. Removing.`, {
+                        mixinProvider: !!mixinProvider,
+                        interfaces,
+                    });
+                    const mixins: string[] = getState(pluginDevice, ScryptedInterfaceProperty.mixins) || [];
+                    this.scrypted.stateManager.setPluginDeviceState(pluginDevice, ScryptedInterfaceProperty.mixins, mixins.filter(mid => mid !== mixinId));
+                    this.scrypted.notifyPluginDeviceDescriptorChanged(pluginDevice);
+                    this.scrypted.datastore.upsert(pluginDevice);
+                }
+                else {
+                    console.log(`Mixin provider ${mixinId} can not mixin ${this.id}. It is no longer a MixinProvider. This may be temporary. Passing through.`);
+                }
                 return {
                     passthrough: true,
                     allInterfaces,
@@ -353,12 +365,17 @@ export class PluginDeviceProxyHandler implements PrimitiveProxyHandler<any>, Scr
         for (const mixin of this.mixinTable) {
             const entry = await mixin.entry;
             if (!entry.methods) {
-                const pluginDevice = this.scrypted.findPluginDeviceById(mixin.mixinProviderId || this.id);
-                const plugin = this.scrypted.plugins[pluginDevice.pluginId];
-                let methods = new Set<string>(getInterfaceMethods(ScryptedInterfaceDescriptors, entry.interfaces))
-                if (plugin.api.descriptors)
-                    methods = new Set<string>([...methods, ...getInterfaceMethods(plugin.api.descriptors, entry.interfaces)]);
-                entry.methods = methods;
+                if (entry.interfaces.size) {
+                    const pluginDevice = this.scrypted.findPluginDeviceById(mixin.mixinProviderId || this.id);
+                    const plugin = this.scrypted.plugins[pluginDevice.pluginId];
+                    let methods = new Set<string>(getInterfaceMethods(ScryptedInterfaceDescriptors, entry.interfaces))
+                    if (plugin.api.descriptors)
+                        methods = new Set<string>([...methods, ...getInterfaceMethods(plugin.api.descriptors, entry.interfaces)]);
+                    entry.methods = methods;
+                }
+                else {
+                    entry.methods = new Set();
+                }
             }
             if (entry.methods.has(method)) {
                 return { mixin, entry };

package/src/plugin/plugin-host-api.ts

@@ -47,7 +47,7 @@ export class PluginHostAPI extends PluginAPIManagedListeners implements PluginAP
 
     // do we care about mixin validation here?
     // maybe to prevent/notify errant dangling events?
-    async onMixinEvent(id: string, nativeIdOrMixinDevice: ScryptedNativeId | any, eventInterface: any, eventData?: any) {
+    async onMixinEvent(id: string, nativeIdOrMixinDevice: ScryptedNativeId | any, eventInterface: string, eventData?: any) {
         // nativeId code path has been deprecated in favor of mixin object 12/10/2021
         const device = this.scrypted.findPluginDeviceById(id);
 

package/src/plugin/plugin-host.ts

@@ -214,7 +214,6 @@ export class PluginHost {
                 await remote.setNativeId(pluginDevice.nativeId, pluginDevice._id, pluginDevice.storage || {});
             }
 
-            await remote.setSystemState(scrypted.stateManager.getSystemState());
             const waitDebug = pluginDebug?.waitDebug;
             if (waitDebug) {
                 console.info('waiting for debugger...');

package/src/plugin/plugin-remote-worker.ts

@@ -9,18 +9,24 @@ import { install as installSourceMapSupport } from 'source-map-support';
 import { PassThrough } from 'stream';
 import { RpcMessage, RpcPeer } from '../rpc';
 import { MediaManagerImpl } from './media';
-import { PluginAPI, PluginRemoteLoadZipOptions } from './plugin-api';
+import { PluginAPI, PluginAPIProxy, PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
 import { installOptionalDependencies } from './plugin-npm-dependencies';
-import { attachPluginRemote, PluginReader, setupPluginRemote } from './plugin-remote';
+import { attachPluginRemote, DeviceManagerImpl, PluginReader, setupPluginRemote } from './plugin-remote';
 import { createREPLServer } from './plugin-repl';
 import { NodeThreadWorker } from './runtime/node-thread-worker';
 const { link } = require('linkfs');
 
+interface PluginStats {
+    type: 'stats',
+    cpu: NodeJS.CpuUsage;
+    memoryUsage: NodeJS.MemoryUsage;
+}
+
 export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessage, reject?: (e: Error) => void, serializationContext?: any) => void) {
     const peer = new RpcPeer('unknown', 'host', peerSend);
 
     let systemManager: SystemManager;
-    let deviceManager: DeviceManager;
+    let deviceManager: DeviceManagerImpl;
     let api: PluginAPI;
 
     const getConsole = (hook: (stdout: PassThrough, stderr: PassThrough) => Promise<void>,
@@ -126,12 +132,6 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
             if (!mixinId) {
                 return;
             }
-            // todo: fix this. a mixin provider can mixin another device to make it a mixin provider itself.
-            // so the mixin id in the mixin table will be incorrect.
-            // there's no easy way to fix this from the remote.
-            // if (!systemManager.getDeviceById(mixinId).mixins.includes(idForNativeId(nativeId))) {
-            //     return;
-            // }
             const reconnect = () => {
                 stdout.removeAllListeners();
                 stderr.removeAllListeners();
@@ -178,15 +178,35 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
         return ret;
     }
 
-    peer.getParam('updateStats').then((updateStats: (stats: any) => void) => {
-        let lastCpuUsage: NodeJS.CpuUsage;
+    // process.cpuUsage is for the entire process.
+    // process.memoryUsage is per thread.
+    const allMemoryStats = new Map<NodeThreadWorker, NodeJS.MemoryUsage>();
+
+    peer.getParam('updateStats').then((updateStats: (stats: PluginStats) => void) => {
         setInterval(() => {
-            const cpuUsage = process.cpuUsage(lastCpuUsage);
-            lastCpuUsage = cpuUsage;
+            const cpuUsage = process.cpuUsage();
+            allMemoryStats.set(undefined, process.memoryUsage());
+
+            const memoryUsage: NodeJS.MemoryUsage = {
+                rss: 0,
+                heapTotal: 0,
+                heapUsed: 0,
+                external: 0,
+                arrayBuffers: 0,
+            }
+
+            for (const mu of allMemoryStats.values()) {
+                memoryUsage.rss += mu.rss;
+                memoryUsage.heapTotal += mu.heapTotal;
+                memoryUsage.heapUsed += mu.heapUsed;
+                memoryUsage.external += mu.external;
+                memoryUsage.arrayBuffers += mu.arrayBuffers;
+            }
+
             updateStats({
                 type: 'stats',
                 cpu: cpuUsage,
-                memoryUsage: process.memoryUsage(),
+                memoryUsage,
             });
         }, 10000);
     });
@@ -316,35 +336,75 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
             pluginReader = undefined;
             const script = main.toString();
 
-            scrypted.fork = async () => {
+
+            const forks = new Set<PluginRemote>();
+
+            scrypted.fork = () => {
                 const ntw = new NodeThreadWorker(pluginId, {
                     env: process.env,
                     pluginDebug: undefined,
                 });
-                const threadPeer = new RpcPeer('main', 'thread', (message, reject) => ntw.send(message, reject));
-                threadPeer.params.updateStats = (stats: any) => {
-                    // todo: merge.
-                    // this.stats = stats;
-                }
-                ntw.setupRpcPeer(threadPeer);
 
-                const remote = await setupPluginRemote(threadPeer, api, pluginId, () => systemManager.getSystemState());
-                const forkOptions = Object.assign({}, zipOptions);
-                forkOptions.fork = true;
-                return remote.loadZip(packageJson, zipData, forkOptions)
+                const result = (async () => {
+                    const threadPeer = new RpcPeer('main', 'thread', (message, reject) => ntw.send(message, reject));
+                    threadPeer.params.updateStats = (stats: PluginStats) => {
+                        allMemoryStats.set(ntw, stats.memoryUsage);
+                    }
+                    ntw.setupRpcPeer(threadPeer);
+
+                    class PluginForkAPI extends PluginAPIProxy {
+                        [RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS] = (api as any)[RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS];
+
+                        setStorage(nativeId: string, storage: { [key: string]: any; }): Promise<void> {
+                            const id = deviceManager.nativeIds.get(nativeId).id;
+                            (scrypted.pluginRemoteAPI as PluginRemote).setNativeId(nativeId, id, storage);
+                            for (const r of forks) {
+                                if (r === remote)
+                                    continue;
+                                r.setNativeId(nativeId, id, storage);
+                            }
+                            return super.setStorage(nativeId, storage);
+                        }
+                    }
+                    const forkApi = new PluginForkAPI(api);
+
+                    const remote = await setupPluginRemote(threadPeer, forkApi, pluginId, () => systemManager.getSystemState());
+                    forks.add(remote);
+                    ntw.worker.on('exit', () => {
+                        forkApi.removeListeners();
+                        forks.delete(remote);
+                        allMemoryStats.delete(ntw);
+                    });
+
+                    for (const [nativeId, dmd] of deviceManager.nativeIds.entries()) {
+                        await remote.setNativeId(nativeId, dmd.id, dmd.storage);
+                    }
+
+                    const forkOptions = Object.assign({}, zipOptions);
+                    forkOptions.fork = true;
+                    return remote.loadZip(packageJson, zipData, forkOptions)
+                })();
+
+                result.catch(() => ntw.kill());
+
+                return {
+                    worker: ntw.worker,
+                    result,
+                }
             }
 
             try {
                 peer.evalLocal(script, zipOptions?.filename || '/plugin/main.nodejs.js', params);
-                pluginConsole?.log('plugin successfully loaded');
 
                 if (zipOptions?.fork) {
+                    pluginConsole?.log('plugin forked');
                     const fork = exports.fork;
-                    const ret = await fork();
-                    ret[RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION] = true;
-                    return ret;
+                    const forked = await fork();
+                    forked[RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION] = true;
+                    return forked;
                 }
 
+                pluginConsole?.log('plugin loaded');
                 let pluginInstance = exports.default;
                 // support exporting a plugin class, plugin main function,
                 // or a plugin instance
@@ -365,7 +425,7 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
         }
     }).then(scrypted => {
         systemManager = scrypted.systemManager;
-        deviceManager = scrypted.deviceManager;
+        deviceManager = scrypted.deviceManager as DeviceManagerImpl;
     });
 
     return peer;

package/src/plugin/plugin-remote.ts

@@ -107,6 +107,10 @@ class DeviceStateProxyHandler implements ProxyHandler<any> {
     get?(target: any, p: PropertyKey, receiver: any) {
         if (p === 'id')
             return this.id;
+        if (p === RpcPeer.PROPERTY_PROXY_PROPERTIES)
+            return { id: this.id }
+        if (p === 'setState')
+            return this.setState;
         return this.deviceManager.systemManager.state[this.id][p as string]?.value;
     }
 
@@ -128,7 +132,7 @@ interface DeviceManagerDevice {
     storage: { [key: string]: any };
 }
 
-class DeviceManagerImpl implements DeviceManager {
+export class DeviceManagerImpl implements DeviceManager {
     api: PluginAPI;
     nativeIds = new Map<string, DeviceManagerDevice>();
     deviceStorage = new Map<string, StorageImpl>();
@@ -153,6 +157,11 @@ class DeviceManagerImpl implements DeviceManager {
         return new Proxy(handler, handler);
     }
 
+    createDeviceState(id: string, setState: (property: string, value: any) => Promise<void>): DeviceState {
+        const handler = new DeviceStateProxyHandler(this, id, setState);
+        return new Proxy(handler, handler);
+    }
+
     getDeviceStorage(nativeId?: any): StorageImpl {
         let ret = this.deviceStorage.get(nativeId);
         if (!ret) {
@@ -297,7 +306,7 @@ export async function setupPluginRemote(peer: RpcPeer, api: PluginAPI, pluginId:
         if (!peer.constructorSerializerMap.get(Buffer))
             peer.addSerializer(Buffer, 'Buffer', new BufferSerializer());
         const getRemote = await peer.getParam('getRemote');
-        const remote = await getRemote(api, pluginId);
+        const remote = await getRemote(api, pluginId) as PluginRemote;
 
         await remote.setSystemState(getSystemState());
         api.listen((id, eventDetails, eventData) => {
@@ -337,7 +346,7 @@ export interface WebSocketCustomHandler {
 export type PluginReader = (name: string) => Buffer;
 
 export interface PluginRemoteAttachOptions {
-    createMediaManager?: (systemManager: SystemManager, deviceManager: DeviceManager) => Promise<MediaManager>;
+    createMediaManager?: (systemManager: SystemManager, deviceManager: DeviceManagerImpl) => Promise<MediaManager>;
     getServicePort?: (name: string, ...args: any[]) => Promise<number>;
     getDeviceConsole?: (nativeId?: ScryptedNativeId) => Console;
     getPluginConsole?: () => Console;
@@ -381,6 +390,7 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
             mediaManager,
             log,
             pluginHostAPI: api,
+            pluginRemoteAPI: undefined,
         }
 
         delete peer.params.getRemote;
@@ -402,9 +412,8 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
                 'setNativeId',
             ],
             getServicePort,
-            createDeviceState(id: string, setState: (property: string, value: any) => Promise<void>) {
-                const handler = new DeviceStateProxyHandler(deviceManager, id, setState);
-                return new Proxy(handler, handler);
+            async createDeviceState(id: string, setState: (property: string, value: any) => Promise<void>) {
+                return deviceManager.createDeviceState(id, setState);
             },
 
             async ioEvent(id: string, event: string, message?: any) {
@@ -507,6 +516,8 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
             },
         }
 
+        ret.pluginRemoteAPI = remote;
+
         return remote;
     }
 

package/src/plugin/runtime/node-fork-worker.ts

@@ -64,6 +64,6 @@ export class NodeForkWorker extends ChildProcessWorker {
     }
 
     get pid() {
-        return this.worker.pid;
+        return this.worker?.pid;
     }
 }