@scrypted/server 0.94.22 → 0.94.24

package/src/plugin/plugin-remote-worker.ts

@@ -1,10 +1,10 @@
 import { ScryptedStatic, SystemManager } from '@scrypted/types';
-import AdmZip from 'adm-zip';
 import { once } from 'events';
 import fs from 'fs';
 import net from 'net';
 import path from 'path';
 import { install as installSourceMapSupport } from 'source-map-support';
+import worker_threads from 'worker_threads';
 import { computeClusterObjectHash } from '../cluster/cluster-hash';
 import { ClusterObject, ConnectRPCObject } from '../cluster/connect-rpc-object';
 import { listenZero } from '../listen-zero';
@@ -14,11 +14,12 @@ import { MediaManagerImpl } from './media';
 import { PluginAPI, PluginAPIProxy, PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
 import { prepareConsoles } from './plugin-console';
 import { getPluginNodePath, installOptionalDependencies } from './plugin-npm-dependencies';
-import { DeviceManagerImpl, PluginReader, attachPluginRemote, setupPluginRemote } from './plugin-remote';
+import { DeviceManagerImpl, attachPluginRemote, setupPluginRemote } from './plugin-remote';
 import { PluginStats, startStatsUpdater } from './plugin-remote-stats';
 import { createREPLServer } from './plugin-repl';
+import { getPluginVolume } from './plugin-volume';
 import { NodeThreadWorker } from './runtime/node-thread-worker';
-import worker_threads from 'worker_threads';
+import { prepareZip } from './runtime/node-worker-common';
 
 const serverVersion = require('../../package.json').version;
 
@@ -75,8 +76,9 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
             }
             throw new Error(`unknown service ${name}`);
         },
-        async onLoadZip(scrypted: ScryptedStatic, params: any, packageJson: any, zipData: Buffer | string, zipOptions: PluginRemoteLoadZipOptions) {
-            const { clusterId, clusterSecret } = zipOptions;
+        async onLoadZip(scrypted: ScryptedStatic, params: any, packageJson: any, getZip: () => Promise<Buffer>, zipOptions: PluginRemoteLoadZipOptions) {
+            const { clusterId, clusterSecret, zipHash } = zipOptions;
+            const { zipFile, unzippedPath } = await prepareZip(getPluginVolume(pluginId), zipHash, getZip);
 
             const onProxySerialization = (value: any, proxyId: string, sourcePeerPort?: number) => {
                 const properties = RpcPeer.prepareProxyProperties(value) || {};
@@ -194,50 +196,20 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
                     return value;
                 }
             }
-
-            // let volume: any;
-            let pluginReader: PluginReader;
-            if (zipOptions?.unzippedPath && fs.existsSync(zipOptions?.unzippedPath)) {
-                if (worker_threads.isMainThread) {
-                    const fsDir = path.join(zipOptions.unzippedPath, 'fs')
-                    if (fs.existsSync(fsDir))
-                        process.chdir(fsDir);
-                    else
-                        process.chdir(zipOptions.unzippedPath);
-                }
-
-                // volume = link(fs, ['', path.join(zipOptions.unzippedPath, 'fs')]);
-                pluginReader = name => {
-                    const filename = path.join(zipOptions.unzippedPath, name);
-                    if (!fs.existsSync(filename))
-                        return;
-                    return fs.readFileSync(filename);
-                };
+            if (worker_threads.isMainThread) {
+                const fsDir = path.join(unzippedPath, 'fs')
+                if (fs.existsSync(fsDir))
+                    process.chdir(fsDir);
+                else
+                    process.chdir(unzippedPath);
             }
-            else {
-                // this code path was used in testing and should be unreachable.
-
-                const admZip = new AdmZip(zipData);
-                // volume = new Volume();
-                // for (const entry of admZip.getEntries()) {
-                //     if (entry.isDirectory)
-                //         continue;
-                //     if (!entry.entryName.startsWith('fs/'))
-                //         continue;
-                //     const name = entry.entryName.substring('fs/'.length);
-                //     volume.mkdirpSync(path.dirname(name));
-                //     const data = entry.getData();
-                //     volume.writeFileSync(name, data);
-                // }
-
-                pluginReader = name => {
-                    const entry = admZip.getEntry(name);
-                    if (!entry)
-                        return;
-                    return entry.getData();
-                }
-            }
-            zipData = undefined;
+
+            const pluginReader = (name: string) => {
+                const filename = path.join(unzippedPath, name);
+                if (!fs.existsSync(filename))
+                    return;
+                return fs.readFileSync(filename);
+            };
 
             const pluginConsole = getPluginConsole?.();
             params.console = pluginConsole;
@@ -248,9 +220,9 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
                     return require('fs');
                 }
                 try {
-                    if (name.startsWith('.') && zipOptions?.unzippedPath) {
+                    if (name.startsWith('.') && unzippedPath) {
                         try {
-                            const c = path.join(zipOptions.unzippedPath, name);
+                            const c = path.join(unzippedPath, name);
                             const module = require(c);
                             return module;
                         }
@@ -314,7 +286,6 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
             peer.getParam('updateStats').then(updateStats => startStatsUpdater(allMemoryStats, updateStats));
 
             const main = pluginReader('main.nodejs.js');
-            pluginReader = undefined;
             const script = main.toString();
 
             scrypted.connect = (socket, options) => {
@@ -328,6 +299,9 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
                     packageJson,
                     env: process.env,
                     pluginDebug: undefined,
+                    zipFile,
+                    unzippedPath,
+                    zipHash,
                 });
 
                 const result = (async () => {
@@ -368,7 +342,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
 
                     const forkOptions = Object.assign({}, zipOptions);
                     forkOptions.fork = true;
-                    return remote.loadZip(packageJson, zipData, forkOptions)
+                    return remote.loadZip(packageJson, getZip, forkOptions)
                 })();
 
                 result.catch(() => ntw.kill());
@@ -380,7 +354,8 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
             }
 
             try {
-                peer.evalLocal(script, zipOptions?.filename || '/plugin/main.nodejs.js', params);
+                const filename = zipOptions?.debug ? '/plugin/main.nodejs.js' : `/${pluginId}/main.nodejs.js`;
+                peer.evalLocal(script, filename, params);
 
                 if (zipOptions?.fork) {
                     // pluginConsole?.log('plugin forked');

package/src/plugin/plugin-remote.ts

@@ -455,15 +455,13 @@ export interface WebSocketCustomHandler {
     methods: WebSocketMethods;
 }
 
-export type PluginReader = (name: string) => Buffer;
-
 export interface PluginRemoteAttachOptions {
     createMediaManager?: (systemManager: SystemManager, deviceManager: DeviceManagerImpl) => Promise<MediaManager>;
     getServicePort?: (name: string, ...args: any[]) => Promise<number>;
     getDeviceConsole?: (nativeId?: ScryptedNativeId) => Console;
     getPluginConsole?: () => Console;
     getMixinConsole?: (id: string, nativeId?: ScryptedNativeId) => Console;
-    onLoadZip?: (scrypted: ScryptedStatic, params: any, packageJson: any, zipData: Buffer | string, zipOptions: PluginRemoteLoadZipOptions) => Promise<any>;
+    onLoadZip?: (scrypted: ScryptedStatic, params: any, packageJson: any, getZip: () => Promise<Buffer>, zipOptions: PluginRemoteLoadZipOptions) => Promise<any>;
     onGetRemote?: (api: PluginAPI, pluginId: string) => Promise<void>;
 }
 
@@ -637,7 +635,7 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
                 done(ret);
             },
 
-            async loadZip(packageJson: any, zipData: Buffer | string, zipOptions?: PluginRemoteLoadZipOptions) {
+            async loadZip(packageJson: any, getZip: () => Promise<Buffer>, zipOptions?: PluginRemoteLoadZipOptions) {
                 const params: any = {
                     __filename: undefined,
                     deviceManager,
@@ -660,7 +658,7 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
                 params.pluginRuntimeAPI = ret;
 
                 try {
-                    return await options.onLoadZip(ret, params, packageJson, zipData, zipOptions);
+                    return await options.onLoadZip(ret, params, packageJson, getZip, zipOptions);
                 }
                 catch (e) {
                     console.error('plugin start/fork failed', e)

package/src/plugin/runtime/custom-worker.ts

@@ -0,0 +1,81 @@
+import { ScryptedRuntimeArguments } from '@scrypted/types';
+import child_process from 'child_process';
+import { Readable, Writable } from 'stream';
+import { RpcMessage, RpcPeer } from "../../rpc";
+import { createRpcDuplexSerializer } from '../../rpc-serializer';
+import type { ScryptedRuntime } from '../../runtime';
+import { ChildProcessWorker } from "./child-process-worker";
+import { RuntimeWorkerOptions } from "./runtime-worker";
+
+export class CustomRuntimeWorker extends ChildProcessWorker {
+    serializer: ReturnType<typeof createRpcDuplexSerializer>;
+    fork: boolean;
+
+    constructor(pluginId: string, options: RuntimeWorkerOptions, runtime: ScryptedRuntime) {
+        super(pluginId, options);
+
+        const pluginDevice = runtime.findPluginDevice(this.pluginId);
+        const scryptedRuntimeArguments: ScryptedRuntimeArguments = pluginDevice.state.scryptedRuntimeArguments?.value;
+        if (!scryptedRuntimeArguments)
+            throw new Error('custom runtime requires scryptedRuntimeArguments');
+
+        const { env, pluginDebug } = options;
+
+        const args = [...scryptedRuntimeArguments.arguments || []];
+
+        const opts: child_process.ForkOptions | child_process.SpawnOptions = {
+            // stdin, stdout, stderr, peer in, peer out
+            stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
+            env: Object.assign({
+                SCRYYPTED_PLUGIN_ID: pluginId,
+                SCRYPTED_DEBUG_PORT: pluginDebug?.inspectPort?.toString(),
+                SCRYPTED_UNZIPPED_PATH: options.unzippedPath,
+                SCRYPTED_ZIP_FILE: options.zipFile,
+                SCRYPTED_ZIP_HASH: options.zipHash,
+            }, process.env, env),
+        };
+
+        if (!scryptedRuntimeArguments.executable) {
+            this.fork = true;
+            const modulePath = args.shift();
+            if (!modulePath)
+                throw new Error('fork runtime requires modulePath in first argument.');
+
+            (opts.stdio as any)[5] = 'ipc';
+            this.worker = child_process.fork(modulePath, args, opts);
+        }
+        else {
+            this.worker = child_process.spawn(scryptedRuntimeArguments.executable, args, opts);
+        }
+
+        this.setupWorker();
+    }
+
+    setupRpcPeer(peer: RpcPeer): void {
+        const peerin = this.worker.stdio[3] as Writable;
+        const peerout = this.worker.stdio[this.fork ? 3 : 4] as Readable;
+
+        const serializer = this.serializer = createRpcDuplexSerializer(peerin);
+        serializer.setupRpcPeer(peer);
+        peerout.on('data', data => serializer.onData(data));
+        peerin.on('error', e => {
+            this.emit('error', e);
+            serializer.onDisconnected();
+        });
+        peerout.on('error', e => {
+            this.emit('error', e)
+            serializer.onDisconnected();
+        });
+    }
+
+    send(message: RpcMessage, reject?: (e: Error) => void, serializationContext?: any): void {
+        try {
+            if (!this.worker)
+                throw new Error('python worker has been killed');
+            this.serializer.sendMessage(message, reject, serializationContext);
+        }
+        catch (e) {
+            reject?.(e);
+        }
+    }
+}

package/src/plugin/runtime/node-thread-worker.ts

@@ -1,10 +1,8 @@
+import v8 from 'v8';
+import worker_threads from "worker_threads";
 import { EventEmitter } from "ws";
 import { RpcMessage, RpcPeer } from "../../rpc";
 import { RuntimeWorker, RuntimeWorkerOptions } from "./runtime-worker";
-import worker_threads from "worker_threads";
-import path from 'path';
-import { getPluginNodePath } from "../plugin-npm-dependencies";
-import v8 from 'v8';
 
 export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
     terminated: boolean;

package/src/plugin/runtime/node-worker-common.ts

@@ -0,0 +1,77 @@
+import AdmZip from 'adm-zip';
+import fs from 'fs';
+import path from 'path';
+
+function createAdmZipHash(hash: string) {
+    const extractVersion = "1-";
+    return extractVersion + hash;
+}
+
+function prep(pluginVolume: string, hash: string) {
+    hash = createAdmZipHash(hash);
+
+    const zipFilename = `${hash}.zip`;
+    const zipDir = path.join(pluginVolume, 'zip');
+    const zipFile = path.join(zipDir, zipFilename);
+    const unzippedPath = path.join(zipDir, 'unzipped')
+    const zipDirTmp = zipDir + '.tmp';
+
+    return {
+        unzippedPath,
+        zipFilename,
+        zipDir,
+        zipFile,
+        zipDirTmp,
+    };
+}
+
+export async function prepareZip(pluginVolume: string, h: string, getZip: () => Promise<Buffer>) {
+    const { zipFile, unzippedPath } = prep(pluginVolume, h);
+    if (fs.existsSync(zipFile)) {
+        return {
+            zipFile,
+            unzippedPath,
+        }
+    }
+
+    const zipBuffer = await getZip();
+    return extractZip(pluginVolume, h, zipBuffer);
+}
+
+export function prepareZipSync(pluginVolume: string, h: string, getZip: () => Buffer) {
+    const { zipFile, unzippedPath } = prep(pluginVolume, h);
+    if (fs.existsSync(zipFile)) {
+        return {
+            zipFile,
+            unzippedPath,
+        }
+    }
+
+    const zipBuffer = getZip();
+    return extractZip(pluginVolume, h, zipBuffer);
+}
+
+export function extractZip(pluginVolume: string, h: string, zipBuffer: Buffer) {
+    const { zipDir, zipDirTmp, zipFilename, zipFile, unzippedPath } = prep(pluginVolume, h);
+
+    fs.rmSync(zipDirTmp, {
+        recursive: true,
+        force: true,
+    });
+    fs.rmSync(zipDir, {
+        recursive: true,
+        force: true,
+    });
+    fs.mkdirSync(zipDirTmp, {
+        recursive: true,
+    });
+    fs.writeFileSync(path.join(zipDirTmp, zipFilename), zipBuffer);
+    const admZip = new AdmZip(zipBuffer);
+    admZip.extractAllTo(path.join(zipDirTmp, 'unzipped'), true);
+    fs.renameSync(zipDirTmp, zipDir);
+
+    return {
+        zipFile,
+        unzippedPath,
+    }
+}

package/src/plugin/runtime/python-worker.ts

@@ -2,12 +2,12 @@ import child_process from 'child_process';
 import fs from "fs";
 import os from "os";
 import path from 'path';
+import type { PortablePython as PortablePythonType } from 'py';
 import { Readable, Writable } from 'stream';
 import { RpcMessage, RpcPeer } from "../../rpc";
 import { createRpcDuplexSerializer } from '../../rpc-serializer';
 import { ChildProcessWorker } from "./child-process-worker";
 import { RuntimeWorkerOptions } from "./runtime-worker";
-import type {PortablePython as PortablePythonType} from 'py'
 
 export class PythonRuntimeWorker extends ChildProcessWorker {
     static {

package/src/plugin/runtime/runtime-worker.ts

@@ -6,6 +6,9 @@ import net from "net";
 export interface RuntimeWorkerOptions {
     packageJson: any;
     pluginDebug: PluginDebug;
+    zipFile: string,
+    unzippedPath: string;
+    zipHash: string;
     env: any;
 }
 

package/src/runtime.ts

@@ -9,7 +9,6 @@ import fs from 'fs';
 import http, { ServerResponse } from 'http';
 import https from 'https';
 import net from 'net';
-import type { spawn as ptySpawn } from '@homebridge/node-pty-prebuilt-multiarch';
 import path from 'path';
 import { ParsedQs } from 'qs';
 import semver from 'semver';
@@ -40,13 +39,14 @@ import { RuntimeWorker, RuntimeWorkerOptions } from './plugin/runtime/runtime-wo
 import { getIpAddress, SCRYPTED_INSECURE_PORT, SCRYPTED_SECURE_PORT } from './server-settings';
 import { AddressSettings } from './services/addresses';
 import { Alerts } from './services/alerts';
+import { Backup } from './services/backup';
 import { CORSControl } from './services/cors';
 import { Info } from './services/info';
 import { getNpmPackageInfo, PluginComponent } from './services/plugin';
 import { ServiceControl } from './services/service-control';
 import { UsersService } from './services/users';
 import { getState, ScryptedStateManager, setState } from './state';
-import { Backup } from './services/backup';
+import { CustomRuntimeWorker } from './plugin/runtime/custom-worker';
 
 interface DeviceProxyPair {
     handler: PluginDeviceProxyHandler;
@@ -61,7 +61,7 @@ interface HttpPluginData {
     pluginDevice: PluginDevice
 }
 
-export type RuntimeHost = (mainFilename: string, pluginId: string, options: RuntimeWorkerOptions) => RuntimeWorker;
+export type RuntimeHost = (mainFilename: string, pluginId: string, options: RuntimeWorkerOptions, runtime: ScryptedRuntime) => RuntimeWorker;
 
 export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
     clusterId = crypto.randomBytes(3).toString('hex');
@@ -74,17 +74,6 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
     devicesLogger = this.logger.getLogger('device', 'Devices');
     wss = new WebSocketServer({ noServer: true });
     wsAtomic = 0;
-    shellio: IOServer = new io.Server({
-        pingTimeout: 120000,
-        perMessageDeflate: true,
-        cors: (req, callback) => {
-            const header = this.getAccessControlAllowOrigin(req.headers);
-            callback(undefined, {
-                origin: header,
-                credentials: true,
-            })
-        },
-    });
     connectRPCObjectIO: IOServer = new io.Server({
         pingTimeout: 120000,
         perMessageDeflate: true,
@@ -111,6 +100,7 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
         // ensure that all the users are loaded from the db.
         this.usersService.getAllUsers();
 
+        this.pluginHosts.set('custom', (_, pluginId, options, runtime) => new CustomRuntimeWorker(pluginId, options, runtime));
         this.pluginHosts.set('python', (_, pluginId, options) => new PythonRuntimeWorker(pluginId, options));
         this.pluginHosts.set('node', (mainFilename, pluginId, options) => new NodeForkWorker(mainFilename, pluginId, options));
 
@@ -118,46 +108,6 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
 
         this.addMiddleware();
 
-        app.all('/engine.io/shell', (req, res) => {
-            if (res.locals.aclId) {
-                res.writeHead(401);
-                res.end();
-                return;
-            }
-            this.shellHandler(req, res);
-        });
-
-        this.shellio.on('connection', connection => {
-            try {
-                const spawn = require('@homebridge/node-pty-prebuilt-multiarch').spawn as typeof ptySpawn;
-                const cp = spawn(process.env.SHELL, [], {
-                });
-                cp.onData(data => connection.send(data));
-                connection.on('message', message => {
-                    if (Buffer.isBuffer(message)) {
-                        cp.write(message.toString());
-                        return;
-                    }
-
-                    try {
-                        const parsed = JSON.parse(message.toString());
-                        if (parsed.dim) {
-                            cp.resize(parsed.dim.cols, parsed.dim.rows);
-                        }
-                    } catch (e) {
-                        // we should only get here if an outdated core plugin
-                        // is sending us string data instead of buffer data
-                        cp.write(message.toString());
-                    }
-                });
-                connection.on('close', () => cp.kill());
-                cp.onExit(() => connection.close());
-            }
-            catch (e) {
-                connection.close();
-            }
-        });
-
         app.all('/engine.io/connectRPCObject', (req, res) => this.connectRPCObjectHandler(req, res));
 
         /*
@@ -305,33 +255,6 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
         };
     }
 
-    async shellHandler(req: Request, res: Response) {
-        const isUpgrade = isConnectionUpgrade(req.headers);
-
-        const end = (code: number, message: string) => {
-            if (isUpgrade) {
-                const socket = res.socket;
-                socket.write(`HTTP/1.1 ${code} ${message}\r\n` +
-                    '\r\n');
-                socket.destroy();
-            }
-            else {
-                res.status(code);
-                res.send(message);
-            }
-        };
-
-        if (!res.locals.username) {
-            end(401, 'Not Authorized');
-            return;
-        }
-
-        if ((req as any).upgradeHead)
-            this.shellio.handleUpgrade(req, res.socket, (req as any).upgradeHead)
-        else
-            this.shellio.handleRequest(req, res);
-    }
-
     async connectRPCObjectHandler(req: Request, res: Response) {
         const isUpgrade = isConnectionUpgrade(req.headers);