@scrypted/server 0.1.14 → 0.2.1

package/src/plugin/media.ts

@@ -1,8 +1,6 @@
-import pathToFfmpeg from 'ffmpeg-static';
-import { BufferConverter, BufferConvertorOptions, DeviceManager, FFmpegInput, MediaManager, MediaObject, MediaObjectOptions, MediaStreamUrl, ScryptedInterface, ScryptedInterfaceProperty, ScryptedMimeTypes, ScryptedNativeId, SystemDeviceState, SystemManager } from "@scrypted/types";
+import ffmpegInstaller from '@ffmpeg-installer/ffmpeg';
+import { BufferConverter, DeviceManager, FFmpegInput, MediaManager, MediaObject, MediaObjectOptions, MediaStreamUrl, ScryptedInterface, ScryptedInterfaceProperty, ScryptedMimeTypes, ScryptedNativeId, SystemDeviceState, SystemManager } from "@scrypted/types";
 import axios from 'axios';
-import child_process from 'child_process';
-import { once } from 'events';
 import fs from 'fs';
 import https from 'https';
 import mimeType from 'mime';
@@ -10,8 +8,6 @@ import mkdirp from "mkdirp";
 import Graph from 'node-dijkstra';
 import os from 'os';
 import path from 'path';
-import rimraf from "rimraf";
-import tmp from 'tmp';
 import MimeType from 'whatwg-mimetype';
 import { MediaObjectRemote } from "./plugin-api";
 
@@ -28,14 +24,28 @@ function mimeMatches(target: MimeType, candidate: MimeType) {
 
 const httpsAgent = new https.Agent({
     rejectUnauthorized: false
-})
+});
+
+type IdBufferConverter = BufferConverter & {
+    id: string;
+};
+
+function getBuiltinId(n: number) {
+    return 'builtin-' + n;
+}
+
+function getExtraId(n: number) {
+    return 'extra-' + n;
+}
 
 export abstract class MediaManagerBase implements MediaManager {
-    builtinConverters: BufferConverter[] = [];
+    builtinConverters: IdBufferConverter[] = [];
+    extraConverters: IdBufferConverter[] = [];
 
     constructor() {
         for (const h of ['http', 'https']) {
             this.builtinConverters.push({
+                id: getBuiltinId(this.builtinConverters.length),
                 fromMimeType: ScryptedMimeTypes.SchemePrefix + h,
                 toMimeType: ScryptedMimeTypes.MediaObject,
                 convert: async (data, fromMimeType, toMimeType) => {
@@ -51,18 +61,32 @@ export abstract class MediaManagerBase implements MediaManager {
         }
 
         this.builtinConverters.push({
+            id: getBuiltinId(this.builtinConverters.length),
             fromMimeType: ScryptedMimeTypes.SchemePrefix + 'file',
             toMimeType: ScryptedMimeTypes.MediaObject,
             convert: async (data, fromMimeType, toMimeType) => {
-                const filename = data.toString();
+                const url = data.toString();
+                const filename = url.substring('file:'.length);
+
+                if (toMimeType === ScryptedMimeTypes.FFmpegInput) {
+                    const ffmpegInput: FFmpegInput = {
+                        url,
+                        inputArguments: [
+                            '-i', filename,
+                        ]
+                    };
+                    return this.createFFmpegMediaObject(ffmpegInput);
+                }
+
                 const ab = await fs.promises.readFile(filename);
-                const mt = mimeType.lookup(data.toString());
+                const mt = mimeType.getType(data.toString());
                 const mo = this.createMediaObject(ab, mt);
                 return mo;
             }
         });
 
         this.builtinConverters.push({
+            id: getBuiltinId(this.builtinConverters.length),
             fromMimeType: ScryptedMimeTypes.Url,
             toMimeType: ScryptedMimeTypes.FFmpegInput,
             async convert(data, fromMimeType): Promise<Buffer> {
@@ -79,6 +103,7 @@ export abstract class MediaManagerBase implements MediaManager {
         });
 
         this.builtinConverters.push({
+            id: getBuiltinId(this.builtinConverters.length),
             fromMimeType: ScryptedMimeTypes.FFmpegInput,
             toMimeType: ScryptedMimeTypes.MediaStreamUrl,
             async convert(data: Buffer, fromMimeType: string): Promise<Buffer> {
@@ -87,6 +112,7 @@ export abstract class MediaManagerBase implements MediaManager {
         });
 
         this.builtinConverters.push({
+            id: getBuiltinId(this.builtinConverters.length),
             fromMimeType: ScryptedMimeTypes.MediaStreamUrl,
             toMimeType: ScryptedMimeTypes.FFmpegInput,
             async convert(data, fromMimeType: string): Promise<Buffer> {
@@ -111,52 +137,24 @@ export abstract class MediaManagerBase implements MediaManager {
             }
         });
 
+        // todo: move this to snapshot plugin
         this.builtinConverters.push({
+            id: getBuiltinId(this.builtinConverters.length),
             fromMimeType: 'image/*',
             toMimeType: 'image/*',
             convert: async (data, fromMimeType: string): Promise<Buffer> => {
                 return data as Buffer;
             }
         });
+    }
 
-        this.builtinConverters.push({
-            fromMimeType: ScryptedMimeTypes.FFmpegInput,
-            toMimeType: 'image/jpeg',
-            convert: async (data, fromMimeType: string, toMimeType: string, options?: BufferConvertorOptions): Promise<Buffer> => {
-                const console = this.getMixinConsole(options?.sourceId, undefined);
-
-                const ffInput: FFmpegInput = JSON.parse(data.toString());
+    async addConverter(converter: IdBufferConverter): Promise<void> {
+        converter.id = getExtraId(this.extraConverters.length);
+        this.extraConverters.push(converter);
+    }
 
-                const args = [
-                    '-hide_banner',
-                ];
-                args.push(...ffInput.inputArguments);
-
-                const tmpfile = tmp.fileSync();
-                try {
-                    args.push('-y', "-vframes", "1", '-f', 'image2', tmpfile.name);
-
-                    const cp = child_process.spawn(await this.getFFmpegPath(), args);
-                    console.log('converting ffmpeg input to image.');
-                    // ffmpegLogInitialOutput(console, cp);
-                    cp.on('error', (code) => {
-                        console.error('ffmpeg error code', code);
-                    })
-                    const to = setTimeout(() => {
-                        console.log('ffmpeg stream to image convesion timed out.');
-                        cp.kill('SIGKILL');
-                    }, 10000);
-                    clearTimeout(to);
-                    const [exitCode] = await once(cp, 'exit');
-                    if (exitCode)
-                        throw new Error(`ffmpeg stream to image convesion failed with exit code: ${exitCode}`);
-                    return fs.readFileSync(tmpfile.name);
-                }
-                finally {
-                    rimraf.sync(tmpfile.name);
-                }
-            }
-        });
+    async clearConverters(): Promise<void> {
+        this.extraConverters = [];
     }
 
     async convertMediaObjectToJSON<T>(mediaObject: MediaObject, toMimeType: string): Promise<T> {
@@ -188,7 +186,7 @@ export abstract class MediaManagerBase implements MediaManager {
             return f;
 
         const defaultPath = os.platform() === 'win32' ? 'ffmpeg.exe' : 'ffmpeg';
-        return pathToFfmpeg || defaultPath;
+        return ffmpegInstaller.path || defaultPath;
     }
 
     async getFilesPath(): Promise<string> {
@@ -200,17 +198,24 @@ export abstract class MediaManagerBase implements MediaManager {
         return ret;
     }
 
-    getConverters(): BufferConverter[] {
+    getConverters(): IdBufferConverter[] {
         const converters = Object.entries(this.getSystemState())
             .filter(([id, state]) => state[ScryptedInterfaceProperty.interfaces]?.value?.includes(ScryptedInterface.BufferConverter))
-            .map(([id]) => this.getDeviceById<BufferConverter>(id));
+            .map(([id]) => this.getDeviceById<IdBufferConverter>(id));
+
+        // builtins should be after system converters. these should not be overriden by system,
+        // as it could cause system instability with misconfiguration.
         converters.push(...this.builtinConverters);
+
+        // extra converters are added last and do allow overriding builtins, as
+        // the instability would be confined to a single plugin.
+        converters.push(...this.extraConverters);
         return converters;
     }
 
     ensureMediaObjectRemote(mediaObject: string | MediaObject): MediaObjectRemote {
         if (typeof mediaObject === 'string') {
-            const mime = mimeType.lookup(mediaObject);
+            const mime = mimeType.getType(mediaObject);
             return this.createMediaObjectRemote(mediaObject, mime);
         }
         return mediaObject as MediaObjectRemote;
@@ -301,7 +306,7 @@ export abstract class MediaManagerBase implements MediaManager {
         return this.createMediaObjectRemote(data, mimeType, options);
     }
 
-    async convert(converters: BufferConverter[], mediaObject: MediaObjectRemote, toMimeType: string): Promise<{ data: Buffer | string | any, mimeType: string }> {
+    async convert(converters: IdBufferConverter[], mediaObject: MediaObjectRemote, toMimeType: string): Promise<{ data: Buffer | string | any, mimeType: string }> {
         // console.log('converting', mediaObject.mimeType, toMimeType);
         const mediaMime = new MimeType(mediaObject.mimeType);
         const outputMime = new MimeType(toMimeType);
@@ -318,13 +323,9 @@ export abstract class MediaManagerBase implements MediaManager {
             sourceId = this.getPluginDeviceId();
         const console = this.getMixinConsole(sourceId, undefined);
 
-        const converterIds = new Map<BufferConverter, string>();
-        const converterReverseids = new Map<string, BufferConverter>();
-        let id = 0;
-        for (const converter of converters) {
-            const cid = (id++).toString();
-            converterIds.set(converter, cid);
-            converterReverseids.set(cid, converter);
+        const converterMap = new Map<string, IdBufferConverter>();
+        for (const c of converters) {
+            converterMap.set(c.id, c);
         }
 
         const nodes: any = {};
@@ -335,31 +336,36 @@ export abstract class MediaManagerBase implements MediaManager {
             try {
                 const inputMime = new MimeType(converter.fromMimeType);
                 const convertedMime = new MimeType(converter.toMimeType);
-                const targetId = converterIds.get(converter);
+                // catch all converters should be heavily weighted so as not to use them.
+                const inputWeight = parseFloat(inputMime.parameters.get('converter-weight')) || (inputMime.essence === '*/*' ? 1000 : 1);
+                // const convertedWeight = parseFloat(convertedMime.parameters.get('converter-weight')) || (convertedMime.essence === ScryptedMimeTypes.MediaObject ? 1000 : 1);
+                // const conversionWeight = inputWeight + convertedWeight;
+                const targetId = converter.id;
                 const node: any = nodes[targetId] = {};
+
+                // edge matches
                 for (const candidate of converters) {
                     try {
                         const candidateMime = new MimeType(candidate.fromMimeType);
                         if (!mimeMatches(convertedMime, candidateMime))
                             continue;
-                        const candidateId = converterIds.get(candidate);
-                        node[candidateId] = 1;
+                        const outputWeight = parseFloat(candidateMime.parameters.get('converter-weight')) || (candidateMime.essence === '*/*' ? 1000 : 1);
+                        const candidateId = candidate.id;
+                        node[candidateId] = inputWeight + outputWeight;
                     }
                     catch (e) {
                         console.warn('skipping converter due to error', e)
                     }
                 }
 
-                // edge matches
+                // source matches
                 if (mimeMatches(mediaMime, inputMime)) {
-                    // catch all converters should be heavily weighted so as not to use them.
-                    mediaNode[targetId] = inputMime.essence === '*/*' ? 1000 : 1;
+                    mediaNode[targetId] = inputWeight;
                 }
 
                 // target output matches
                 if (mimeMatches(outputMime, convertedMime) || converter.toMimeType === ScryptedMimeTypes.MediaObject) {
-                    // catch all converters should be heavily weighted so as not to use them.
-                    node['output'] = converter.toMimeType === ScryptedMimeTypes.MediaObject ? 1000 : 1;
+                    node['output'] = inputWeight;
                 }
             }
             catch (e) {
@@ -382,13 +388,21 @@ export abstract class MediaManagerBase implements MediaManager {
         let value = await mediaObject.getData();
         let valueMime = new MimeType(mediaObject.mimeType);
 
-        for (const node of route) {
-            const converter = converterReverseids.get(node);
+        while (route.length) {
+            const node = route.shift();
+            const converter = converterMap.get(node);
             const converterToMimeType = new MimeType(converter.toMimeType);
             const converterFromMimeType = new MimeType(converter.fromMimeType);
             const type = converterToMimeType.type === '*' ? valueMime.type : converterToMimeType.type;
             const subtype = converterToMimeType.subtype === '*' ? valueMime.subtype : converterToMimeType.subtype;
-            const targetMimeType = `${type}/${subtype}`;
+            let targetMimeType = `${type}/${subtype}`;
+            if (!route.length && outputMime.parameters.size) {
+                const withParameters = new MimeType(targetMimeType);
+                for (const k of outputMime.parameters.keys()) {
+                    withParameters.parameters.set(k, outputMime.parameters.get(k));
+                }
+                targetMimeType = outputMime.toString();
+            }
 
             if (converter.toMimeType === ScryptedMimeTypes.MediaObject) {
                 const mo = await converter.convert(value, valueMime.essence, toMimeType, { sourceId }) as MediaObject;
@@ -413,6 +427,15 @@ export abstract class MediaManagerBase implements MediaManager {
 export class MediaManagerImpl extends MediaManagerBase {
     constructor(public systemManager: SystemManager, public deviceManager: DeviceManager) {
         super();
+
+        this.builtinConverters.push({
+            id: getBuiltinId(this.builtinConverters.length),
+            fromMimeType: ScryptedMimeTypes.ScryptedDeviceId,
+            toMimeType: ScryptedMimeTypes.ScryptedDevice,
+            convert: async (data, fromMimeType, toMimeType) => {
+                return this.getDeviceById(data.toString());
+            }
+        });
     }
 
     getSystemState(): { [id: string]: { [property: string]: SystemDeviceState; }; } {
@@ -436,16 +459,12 @@ export class MediaManagerImpl extends MediaManagerBase {
 
 export class MediaManagerHostImpl extends MediaManagerBase {
     constructor(public pluginDeviceId: string,
-        public systemState: { [id: string]: { [property: string]: SystemDeviceState } },
+        public getSystemState: () => { [id: string]: { [property: string]: SystemDeviceState } },
         public console: Console,
         public getDeviceById: (id: string) => any) {
         super();
     }
 
-    getSystemState(): { [id: string]: { [property: string]: SystemDeviceState; }; } {
-        return this.systemState;
-    }
-
     getPluginDeviceId(): string {
         return this.pluginDeviceId;
     }

package/src/plugin/plugin-api.ts

@@ -11,8 +11,8 @@ export interface PluginAPI {
     setState(nativeId: ScryptedNativeId, key: string, value: any): Promise<void>;
     onDevicesChanged(deviceManifest: DeviceManifest): Promise<void>;
     onDeviceDiscovered(device: Device): Promise<string>;
-    onDeviceEvent(nativeId: ScryptedNativeId, eventInterface: any, eventData?: any): Promise<void>;
-    onMixinEvent(id: string, nativeId: ScryptedNativeId, eventInterface: any, eventData?: any): Promise<void>;
+    onDeviceEvent(nativeId: ScryptedNativeId, eventInterface: string, eventData?: any): Promise<void>;
+    onMixinEvent(id: string, nativeId: ScryptedNativeId, eventInterface: string, eventData?: any): Promise<void>;
     onDeviceRemoved(nativeId: string): Promise<void>;
     setStorage(nativeId: string, storage: {[key: string]: any}): Promise<void>;
 
@@ -89,8 +89,8 @@ export class PluginAPIProxy extends PluginAPIManagedListeners implements PluginA
     onDeviceEvent(nativeId: ScryptedNativeId, eventInterface: any, eventData?: any): Promise<void> {
         return this.api.onDeviceEvent(nativeId, eventInterface, eventData);
     }
-    onMixinEvent(id: string, nativeId: ScryptedNativeId, eventInterface: any, eventData?: any): Promise<void> {
-        return this.api.onMixinEvent(nativeId, eventInterface, eventData);
+    onMixinEvent(id: string, nativeId: ScryptedNativeId, eventInterface: string, eventData?: any): Promise<void> {
+        return this.api.onMixinEvent(id, nativeId, eventInterface, eventData);
     }
     onDeviceRemoved(nativeId: string): Promise<void> {
         return this.api.onDeviceRemoved(nativeId);
@@ -147,6 +147,7 @@ export interface PluginRemoteLoadZipOptions {
      * exist on the "remote", if it is not the same machine.
      */
     unzippedPath?: string;
+    fork?: boolean;
 }
 
 export interface PluginRemote {

package/src/plugin/plugin-device.ts

@@ -216,15 +216,24 @@ export class PluginDeviceProxyHandler implements PrimitiveProxyHandler<any>, Scr
         let { allInterfaces } = await previousEntry;
         try {
             const mixinProvider = this.scrypted.getDevice(mixinId) as ScryptedDevice & MixinProvider;
-            const interfaces = mixinProvider?.interfaces?.includes(ScryptedInterface.MixinProvider) && await mixinProvider?.canMixin(type, allInterfaces) as any as ScryptedInterface[];
+            const isMixinProvider = mixinProvider?.interfaces?.includes(ScryptedInterface.MixinProvider);
+            const interfaces = isMixinProvider && await mixinProvider?.canMixin(type, allInterfaces) as any as ScryptedInterface[];
             if (!interfaces) {
                 // this is not an error
                 // do not advertise interfaces so it is skipped during
                 // vtable lookup.
-                console.log(`mixin provider ${mixinId} can no longer mixin ${this.id}`);
-                const mixins: string[] = getState(pluginDevice, ScryptedInterfaceProperty.mixins) || [];
-                this.scrypted.stateManager.setPluginDeviceState(pluginDevice, ScryptedInterfaceProperty.mixins, mixins.filter(mid => mid !== mixinId))
-                this.scrypted.datastore.upsert(pluginDevice);
+                if (!mixinProvider || (isMixinProvider && !interfaces)) {
+                    console.log(`Mixin provider ${mixinId} can no longer mixin ${this.id}. Removing.`, {
+                        mixinProvider: !!mixinProvider,
+                        interfaces,
+                    });
+                    const mixins: string[] = getState(pluginDevice, ScryptedInterfaceProperty.mixins) || [];
+                    this.scrypted.stateManager.setPluginDeviceState(pluginDevice, ScryptedInterfaceProperty.mixins, mixins.filter(mid => mid !== mixinId))
+                    this.scrypted.datastore.upsert(pluginDevice);
+                }
+                else {
+                    console.log(`Mixin provider ${mixinId} can not mixin ${this.id}. It is no longer a MixinProvider. This may be temporary. Passing through.`);
+                }
                 return {
                     passthrough: true,
                     allInterfaces,
@@ -353,12 +362,17 @@ export class PluginDeviceProxyHandler implements PrimitiveProxyHandler<any>, Scr
         for (const mixin of this.mixinTable) {
             const entry = await mixin.entry;
             if (!entry.methods) {
-                const pluginDevice = this.scrypted.findPluginDeviceById(mixin.mixinProviderId || this.id);
-                const plugin = this.scrypted.plugins[pluginDevice.pluginId];
-                let methods = new Set<string>(getInterfaceMethods(ScryptedInterfaceDescriptors, entry.interfaces))
-                if (plugin.api.descriptors)
-                    methods = new Set<string>([...methods, ...getInterfaceMethods(plugin.api.descriptors, entry.interfaces)]);
-                entry.methods = methods;
+                if (entry.interfaces.size) {
+                    const pluginDevice = this.scrypted.findPluginDeviceById(mixin.mixinProviderId || this.id);
+                    const plugin = this.scrypted.plugins[pluginDevice.pluginId];
+                    let methods = new Set<string>(getInterfaceMethods(ScryptedInterfaceDescriptors, entry.interfaces))
+                    if (plugin.api.descriptors)
+                        methods = new Set<string>([...methods, ...getInterfaceMethods(plugin.api.descriptors, entry.interfaces)]);
+                    entry.methods = methods;
+                }
+                else {
+                    entry.methods = new Set();
+                }
             }
             if (entry.methods.has(method)) {
                 return { mixin, entry };

package/src/plugin/plugin-host-api.ts

@@ -47,7 +47,7 @@ export class PluginHostAPI extends PluginAPIManagedListeners implements PluginAP
 
     // do we care about mixin validation here?
     // maybe to prevent/notify errant dangling events?
-    async onMixinEvent(id: string, nativeIdOrMixinDevice: ScryptedNativeId | any, eventInterface: any, eventData?: any) {
+    async onMixinEvent(id: string, nativeIdOrMixinDevice: ScryptedNativeId | any, eventInterface: string, eventData?: any) {
         // nativeId code path has been deprecated in favor of mixin object 12/10/2021
         const device = this.scrypted.findPluginDeviceById(id);
 

package/src/plugin/plugin-host.ts

@@ -135,7 +135,7 @@ export class PluginHost {
         this.io.on('connection', async (socket) => {
             try {
                 try {
-                    if (socket.request.url.indexOf('/api') !== -1) {
+                    if (socket.request.url.indexOf('/engine.io/api') !== -1) {
                         if (socket.request.url.indexOf('/public') !== -1) {
                             socket.close();
                             return;
@@ -179,7 +179,7 @@ export class PluginHost {
 
         const { runtime } = this.packageJson.scrypted;
         const mediaManager = runtime === 'python'
-            ? new MediaManagerHostImpl(pluginDeviceId, scrypted.stateManager.getSystemState(), console, id => scrypted.getDevice(id))
+            ? new MediaManagerHostImpl(pluginDeviceId, () => scrypted.stateManager.getSystemState(), console, id => scrypted.getDevice(id))
             : undefined;
 
         this.api = new PluginHostAPI(scrypted, this.pluginId, this, mediaManager);
@@ -214,7 +214,6 @@ export class PluginHost {
                 await remote.setNativeId(pluginDevice.nativeId, pluginDevice._id, pluginDevice.storage || {});
             }
 
-            await remote.setSystemState(scrypted.stateManager.getSystemState());
             const waitDebug = pluginDebug?.waitDebug;
             if (waitDebug) {
                 console.info('waiting for debugger...');
@@ -374,7 +373,8 @@ export class PluginHost {
         serializer.setupRpcPeer(rpcPeer);
 
         // wrap the host api with a connection specific api that can be torn down on disconnect
-        const api = new PluginAPIProxy(this.api, await this.peer.getParam('mediaManager'));
+        const createMediaManager = await this.peer.getParam('createMediaManager');
+        const api = new PluginAPIProxy(this.api, await createMediaManager());
         const kill = () => {
             serializer.onDisconnected();
             api.removeListeners();
@@ -389,7 +389,8 @@ export class PluginHost {
         const rpcPeer = createDuplexRpcPeer(`api/${this.pluginId}`, 'duplex', duplex, duplex);
 
         // wrap the host api with a connection specific api that can be torn down on disconnect
-        const api = new PluginAPIProxy(this.api, await this.peer.getParam('mediaManager'));
+        const createMediaManager = await this.peer.getParam('createMediaManager');
+        const api = new PluginAPIProxy(this.api, await createMediaManager());
         const kill = () => {
             api.removeListeners();
         };

package/src/plugin/plugin-http.ts

@@ -38,7 +38,7 @@ export abstract class PluginHttp<T> {
 
     abstract handleEngineIOEndpoint(req: Request, res: ServerResponse, endpointRequest: HttpRequest, pluginData: T): void;
     abstract handleRequestEndpoint(req: Request, res: Response, endpointRequest: HttpRequest, pluginData: T): void;
-    abstract getEndpointPluginData(endpoint: string, isUpgrade: boolean, isEngineIOEndpoint: boolean): Promise<T>;
+    abstract getEndpointPluginData(req: Request, endpoint: string, isUpgrade: boolean, isEngineIOEndpoint: boolean): Promise<T>;
     abstract handleWebSocket(endpoint: string, httpRequest: HttpRequest, ws: WebSocket, pluginData: T): Promise<void>;
 
     async endpointHandler(req: Request, res: Response, isPublicEndpoint: boolean, isEngineIOEndpoint: boolean,
@@ -75,7 +75,7 @@ export abstract class PluginHttp<T> {
             return;
         }
 
-        const pluginData = await this.getEndpointPluginData(endpoint, isUpgrade, isEngineIOEndpoint);
+        const pluginData = await this.getEndpointPluginData(req, endpoint, isUpgrade, isEngineIOEndpoint);
         if (!pluginData) {
             end(404, 'Not Found');
             return;