@scrypted/server 0.123.0 → 0.123.2

package/src/plugin/plugin-console.ts

@@ -2,8 +2,9 @@ import { DeviceManager, ScryptedNativeId, SystemManager } from '@scrypted/types'
 import { Console } from 'console';
 import { once } from 'events';
 import net, { Server } from 'net';
-import { PassThrough, Readable } from 'stream';
+import { PassThrough, Readable, Writable } from 'stream';
 import { listenZero } from '../listen-zero';
+import { isClusterAddress } from '../scrypted-cluster-common';
 
 export interface ConsoleServer {
     pluginConsole: Console;
@@ -76,8 +77,11 @@ export function prepareConsoles(getConsoleName: () => string, systemManager: ()
         ret = getConsole(async (stdout, stderr) => {
             const connect = async () => {
                 const plugins = await getPlugins();
-                const port = await plugins.getRemoteServicePort(getConsoleName(), 'console-writer');
-                const socket = net.connect(port);
+                const [port,host] = await plugins.getRemoteServicePort(getConsoleName(), 'console-writer');
+                const socket = net.connect({
+                    port,
+                    host,
+                });
                 socket.write(nativeId + '\n');
                 const writer = (data: Buffer) => {
                     socket.write(data);
@@ -136,8 +140,12 @@ export function prepareConsoles(getConsoleName: () => string, systemManager: ()
                 if (!mixin)
                     return;
                 const { pluginId, nativeId: mixinNativeId } = mixin;
-                const port = await plugins.getRemoteServicePort(pluginId, 'console-writer');
-                const socket = net.connect(port);
+                const [port, host] = await plugins.getRemoteServicePort(pluginId, 'console-writer');
+                const socket = net.connect({
+                    port,
+                    host,
+                });
+                socket.on('error', () => { });
                 socket.write(mixinNativeId + '\n');
                 const writer = (data: Buffer) => {
                     let str = data.toString().trim();
@@ -295,8 +303,12 @@ export async function createConsoleServer(remoteStdout: Readable, remoteStderr:
         socket.once('error', cleanup);
         socket.once('end', cleanup);
     });
-    const readPort = await listenZero(readServer, '127.0.0.1');
-    const writePort = await listenZero(writeServer, '127.0.0.1');
+
+    let address = '0.0.0.0';
+    if (isClusterAddress(address))
+        address = '127.0.0.1';
+    const readPort = await listenZero(readServer, address);
+    const writePort = await listenZero(writeServer, address);
 
     return {
         clear(nativeId: ScryptedNativeId) {
@@ -333,3 +345,13 @@ export function pipeWorkerConsole(nativeWorker: { stdout: Readable, stderr: Read
         useConsole.error(data.toString());
     });
 }
+
+export async function writeWorkerGenerator(asyncIterator: AsyncGenerator<Buffer>, writable: Writable) {
+    try {
+        for await (const data of asyncIterator) {
+            writable.write(data);
+        }
+    }
+    catch (e) {
+    }
+}

package/src/plugin/plugin-host-api.ts

@@ -186,7 +186,7 @@ export class PluginHostAPI extends PluginAPIManagedListeners implements PluginAP
 
     async requestRestart() {
         const logger = await this.getLogger(undefined);
-        logger.log('i', 'plugin restart was requested');
+        logger?.log('i', 'plugin restart was requested');
         return this.restartDebounced();
     }
 

package/src/plugin/plugin-host.ts

@@ -2,27 +2,28 @@ import { Device, EngineIOHandler, ScryptedInterface } from '@scrypted/types';
 import crypto from 'crypto';
 import * as io from 'engine.io';
 import fs from 'fs';
-import net from 'net';
 import os from 'os';
-import { Duplex } from 'stream';
 import WebSocket from 'ws';
 import { Plugin } from '../db-types';
 import { IOServer, IOServerSocket } from '../io';
 import { Logger } from '../logger';
-import { RpcPeer } from '../rpc';
-import { createDuplexRpcPeer, createRpcSerializer } from '../rpc-serializer';
+import { RpcPeer, RPCResultError } from '../rpc';
+import { createRpcSerializer } from '../rpc-serializer';
 import { ScryptedRuntime } from '../runtime';
+import { prepareClusterPeer } from '../scrypted-cluster-common';
 import { sleep } from '../sleep';
 import { AccessControls } from './acl';
 import { MediaManagerHostImpl } from './media';
-import { PluginAPIProxy, PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
+import { PluginAPIProxy, PluginRemote, PluginRemoteLoadZipOptions, PluginZipAPI } from './plugin-api';
 import { ConsoleServer, createConsoleServer } from './plugin-console';
 import { PluginDebug } from './plugin-debug';
 import { PluginHostAPI } from './plugin-host-api';
 import { LazyRemote } from './plugin-lazy-remote';
 import { setupPluginRemote } from './plugin-remote';
+import { PluginStats } from './plugin-remote-stats';
 import { WebSocketConnection } from './plugin-remote-websocket';
 import { ensurePluginVolume, getScryptedVolume } from './plugin-volume';
+import { createClusterForkWorker, needsClusterForkWorker } from './runtime/cluster-fork.worker';
 import { prepareZipSync } from './runtime/node-worker-common';
 import { RuntimeWorker } from './runtime/runtime-worker';
 
@@ -59,10 +60,8 @@ export class PluginHost {
     api: PluginHostAPI;
     pluginName: string;
     packageJson: any;
-    stats: {
-        cpuUsage: NodeJS.CpuUsage,
-        memoryUsage: NodeJS.MemoryUsage,
-    };
+    lastStats: number;
+    stats: PluginStats;
     killed = false;
     consoleServer: Promise<ConsoleServer>;
     zipHash: string;
@@ -145,7 +144,7 @@ export class PluginHost {
             this.unzippedPath = unzippedPath;
         }
 
-        this.startPluginHost(logger, {
+        const peerPromise = this.startPluginHost(logger, {
             SCRYPTED_VOLUME: volume,
             SCRYPTED_PLUGIN_VOLUME: pluginVolume,
         }, pluginDebug);
@@ -206,7 +205,7 @@ export class PluginHost {
         const self = this;
 
         const { runtime } = this.packageJson.scrypted;
-        const mediaManager = runtime === 'python'
+        const mediaManager = runtime && runtime !== 'node'
             ? new MediaManagerHostImpl(pluginDeviceId, () => scrypted.stateManager.getSystemState(), console, id => scrypted.getDevice(id))
             : undefined;
 
@@ -215,7 +214,18 @@ export class PluginHost {
         logger.log('i', `loading ${this.pluginName}`);
         logger.log('i', 'pid ' + this.worker?.pid);
 
-        const remotePromise = setupPluginRemote(this.peer, this.api, self.pluginId, { serverVersion }, () => this.scrypted.stateManager.getSystemState());
+        const remotePromise = (async () => {
+            let peer: RpcPeer;
+            try {
+                peer = await peerPromise;
+            }
+            catch (e) {
+                logger.log('e', 'plugin failed to start ' + e);
+                throw new RPCResultError(this.peer, 'cluster plugin start failed', e);
+            }
+            return setupPluginRemote(peer, this.api, self.pluginId, { serverVersion }, () => this.scrypted.stateManager.getSystemState());
+        })();
+
         const init = (async () => {
             const remote = await remotePromise;
 
@@ -248,7 +258,14 @@ export class PluginHost {
                 };
                 // original implementation sent the zipBuffer, sending the zipFile name now.
                 // can switch back for non-local plugins.
-                const modulePromise = remote.loadZip(this.packageJson, async () => fs.promises.readFile(this.zipFile), loadZipOptions);
+                const modulePromise = remote.loadZip(this.packageJson,
+                    // the plugin is expected to send process stats every 10 seconds.
+                    // this can be used as a check for liveness.
+                    new PluginZipAPI(async () => fs.promises.readFile(this.zipFile), async (stats: PluginStats) => {
+                        this.lastStats = Date.now();
+                        this.stats = stats;
+                    }),
+                    loadZipOptions);
                 // allow garbage collection of the zip buffer
                 const module = await modulePromise;
                 logger.log('i', `loaded ${this.pluginName}`);
@@ -288,28 +305,65 @@ export class PluginHost {
         if (!workerHost)
             throw new UnsupportedRuntimeError(this.packageJson.scrypted.runtime);
 
-        this.worker = workerHost(this.scrypted.mainFilename, this.pluginId, {
-            packageJson: this.packageJson,
-            env,
-            pluginDebug,
-            unzippedPath: this.unzippedPath,
-            zipFile: this.zipFile,
-            zipHash: this.zipHash,
-        }, this.scrypted);
-
-        this.peer = new RpcPeer('host', this.pluginId, (message, reject, serializationContext) => {
-            if (connected) {
-                this.worker.send(message, reject, serializationContext);
-            }
-            else if (reject) {
-                reject(new Error('peer disconnected'));
-            }
-        });
+        let peer: Promise<RpcPeer>
+        if (!needsClusterForkWorker(this.packageJson.scrypted)) {
+            this.peer = new RpcPeer('host', this.pluginId, (message, reject, serializationContext) => {
+                if (connected) {
+                    this.worker.send(message, reject, serializationContext);
+                }
+                else if (reject) {
+                    reject(new Error('peer disconnected'));
+                }
+            });
 
-        this.worker.setupRpcPeer(this.peer);
+            peer = Promise.resolve(this.peer);
+
+            this.worker = workerHost(this.scrypted.mainFilename, this.pluginId, {
+                packageJson: this.packageJson,
+                env,
+                pluginDebug,
+                unzippedPath: this.unzippedPath,
+                zipFile: this.zipFile,
+                zipHash: this.zipHash,
+            }, this.scrypted);
+
+            this.worker.setupRpcPeer(this.peer);
+
+            this.worker.stdout.on('data', data => console.log(data.toString()));
+            this.worker.stderr.on('data', data => console.error(data.toString()));
+        }
+        else {
+            this.peer = new RpcPeer('host', this.pluginId, (message, reject, serializationContext) => {
+                if (connected) {
+                    console.warn('unexpected message to cluster fork worker', message);
+                }
+                else if (reject) {
+                    reject(new Error('peer disconnected'));
+                }
+            });
+
+            const clusterSetup = prepareClusterPeer(this.peer);
+            const { runtimeWorker, forkPeer } = createClusterForkWorker((async () => {
+                await clusterSetup.initializeCluster({
+                    clusterId: this.scrypted.clusterId,
+                    clusterSecret: this.scrypted.clusterSecret,
+                });
+                return this.scrypted.clusterFork;
+            })(),
+                this.zipHash, async () => fs.promises.readFile(this.zipFile),
+                this.packageJson.scrypted, this.packageJson, clusterSetup.connectRPCObject);
+
+            forkPeer.then(peer => {
+                const originalPeer = this.peer;
+                originalPeer.killed.then(s => peer.kill(s)).catch(e => peer.kill(e));
+                this.peer = peer;
+                peer.killed.catch(() =>{} ).finally(() => originalPeer.kill());
+            }).catch(() => {});
+
+            this.worker = runtimeWorker;
+            peer = forkPeer;
+        }
 
-        this.worker.stdout.on('data', data => console.log(data.toString()));
-        this.worker.stderr.on('data', data => console.error(data.toString()));
         let consoleHeader = `${os.platform()} ${os.arch()} ${os.version()}\nserver version: ${serverVersion}\nplugin version: ${this.pluginId} ${this.packageJson.version}\n`;
         if (process.env.SCRYPTED_DOCKER_FLAVOR)
             consoleHeader += `${process.env.SCRYPTED_DOCKER_FLAVOR}\n`;
@@ -333,33 +387,7 @@ export class PluginHost {
             disconnect();
         });
 
-        this.worker.on('rpc', async (message, sendHandle) => {
-            const socket = sendHandle as net.Socket;
-            const { pluginId, username } = message;
-            const host = this.scrypted.plugins[pluginId];
-            if (!host) {
-                socket.destroy();
-                return;
-            }
-            try {
-                const accessControls = await this.scrypted.getAccessControls(username)
-                host.createRpcPeer(socket, accessControls);
-            }
-            catch (e) {
-                socket.destroy();
-                return;
-            }
-        });
-
         const startupTime = Date.now();
-        // the plugin is expected to send process stats every 10 seconds.
-        // this can be used as a check for liveness.
-        let lastStats: number;
-        this.peer.params.updateStats = (stats: any) => {
-            lastStats = Date.now();
-            this.stats = stats;
-        }
-
         let lastPong: number;
         this.peer.params.pong = (time: number) => {
             lastPong = time;
@@ -371,7 +399,7 @@ export class PluginHost {
                     await sleep(30000);
                     if (this.killed)
                         return;
-                    pingPromise ||= await this.peer.getParam('ping');
+                    pingPromise ||= peer.then(p => p.getParam('ping'));
                     const ping = await pingPromise;
                     await ping(Date.now());
                 }
@@ -386,7 +414,7 @@ export class PluginHost {
             const now = Date.now();
             // plugin may take a while to install, so wait 10 minutes.
             // after that, require 1 minute checkins.
-            if (!lastStats || !lastPong) {
+            if (!this.lastStats || !lastPong) {
                 if (now - startupTime > 10 * 60 * 1000) {
                     const logger = await this.api.getLogger(undefined);
                     logger.log('e', 'plugin failed to start in a timely manner. restarting.');
@@ -394,7 +422,7 @@ export class PluginHost {
                 }
                 return;
             }
-            if (!pluginDebug && (lastStats + 60000 < now)) {
+            if (!pluginDebug && (this.lastStats + 60000 < now)) {
                 const logger = await this.api.getLogger(undefined);
                 logger.log('e', 'plugin is not reporting stats. restarting.');
                 this.api.requestRestart();
@@ -406,6 +434,8 @@ export class PluginHost {
             }
         }, 60000);
         this.peer.killed.finally(() => clearInterval(healthInterval));
+
+        return peer;
     }
 
     async createRpcIoPeer(socket: IOServerSocket, accessControls: AccessControls) {
@@ -447,19 +477,4 @@ export class PluginHost {
 
         return setupPluginRemote(rpcPeer, api, null, { serverVersion }, () => this.scrypted.stateManager.getSystemState());
     }
-
-    async createRpcPeer(duplex: Duplex, accessControls: AccessControls) {
-        const rpcPeer = createDuplexRpcPeer(`api/${this.pluginId}`, 'duplex', duplex, duplex);
-        rpcPeer.tags.acl = accessControls;
-
-        // wrap the host api with a connection specific api that can be torn down on disconnect
-        const createMediaManager = await this.peer.getParam('createMediaManager');
-        const api = new PluginAPIProxy(this.api, await createMediaManager());
-        const kill = () => {
-            api.removeListeners();
-        };
-        duplex.on('close', kill);
-
-        return setupPluginRemote(rpcPeer, api, null, { serverVersion }, () => this.scrypted.stateManager.getSystemState());
-    }
 }

package/src/plugin/plugin-lazy-remote.ts

@@ -1,5 +1,5 @@
 import { EventDetails, ScryptedNativeId, SystemDeviceState } from '@scrypted/types'
-import { PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
+import { PluginRemote, PluginRemoteLoadZipOptions, PluginZipAPI } from './plugin-api';
 
 /**
  * This remote is necessary as the host needs to create a remote synchronously
@@ -17,10 +17,10 @@ import { PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
         })();
     }
 
-    async loadZip(packageJson: any, getZip: () => Promise<Buffer>, options?: PluginRemoteLoadZipOptions): Promise<any> {
+    async loadZip(packageJson: any, zipAPI: PluginZipAPI, options?: PluginRemoteLoadZipOptions): Promise<any> {
         if (!this.remote)
             await this.remoteReadyPromise;
-        return this.remote.loadZip(packageJson, getZip, options);
+        return this.remote.loadZip(packageJson, zipAPI, options);
     }
     async setSystemState(state: { [id: string]: { [property: string]: SystemDeviceState; }; }): Promise<void> {
         if (!this.remote)
@@ -65,7 +65,7 @@ import { PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
         return this.remote.createDeviceState(id, setState);
     }
 
-    async getServicePort(name: string, ...args: any[]): Promise<number> {
+    async getServicePort(name: string, ...args: any[]): Promise<[number, string]> {
         const remote = await this.remotePromise;
         return remote.getServicePort(name, ...args);
     }

package/src/plugin/plugin-remote-stats.ts

@@ -2,11 +2,13 @@ import { RuntimeWorker } from "./runtime/runtime-worker";
 
 export interface PluginStats {
     type: 'stats',
-    cpu: NodeJS.CpuUsage;
+    cpuUsage: NodeJS.CpuUsage;
     memoryUsage: NodeJS.MemoryUsage;
 }
 
-export function startStatsUpdater(allMemoryStats: Map<RuntimeWorker, NodeJS.MemoryUsage>, updateStats: (stats: PluginStats) => void) {
+export function startStatsUpdater(allMemoryStats: Map<RuntimeWorker, NodeJS.MemoryUsage>, updateStats: (stats: PluginStats) => Promise<void>) {
+    if (!updateStats)
+        console.warn('wtf')
     setInterval(() => {
         let cpuUsage: NodeJS.CpuUsage;
         let memoryUsage: NodeJS.MemoryUsage;
@@ -35,8 +37,8 @@ export function startStatsUpdater(allMemoryStats: Map<RuntimeWorker, NodeJS.Memo
 
         updateStats({
             type: 'stats',
-            cpu: cpuUsage,
+            cpuUsage,
             memoryUsage,
-        });
+        }).catch(() => {});
     }, 10000);
 }