@scrypted/server 0.123.1 → 0.123.2

package/src/scrypted-cluster.ts

@@ -0,0 +1,326 @@
+import type { ForkOptions } from '@scrypted/types';
+import net from 'net';
+import os from 'os';
+import type { Readable } from 'stream';
+import tls from 'tls';
+import type { createSelfSignedCertificate } from './cert';
+import { computeClusterObjectHash } from './cluster/cluster-hash';
+import type { ClusterObject } from './cluster/connect-rpc-object';
+import { getPluginVolume, getScryptedVolume } from './plugin/plugin-volume';
+import { prepareZip } from './plugin/runtime/node-worker-common';
+import { getBuiltinRuntimeHosts } from './plugin/runtime/runtime-host';
+import { RuntimeWorker } from './plugin/runtime/runtime-worker';
+import { RpcPeer } from './rpc';
+import { createRpcDuplexSerializer } from './rpc-serializer';
+import type { ScryptedRuntime } from './runtime';
+import { prepareClusterPeer } from './scrypted-cluster-common';
+import { sleep } from './sleep';
+
+export class PeerLiveness {
+    __proxy_oneway_methods = ['kill'];
+    constructor(private killed: Promise<any>) {
+    }
+    async waitKilled() {
+        return this.killed;
+    }
+}
+
+export class ClusterForkResult extends PeerLiveness {
+    constructor(private peer: RpcPeer, killed: Promise<any>, private result: any) {
+        super(killed);
+    }
+
+    async kill() {
+        this.peer.kill('killed');
+    }
+
+    async getResult() {
+        return this.result;
+    }
+}
+
+export type ClusterForkParam = (peerLiveness: PeerLiveness, runtime: string, packageJson: any, zipHash: string, getZip: () => Promise<Buffer>) => Promise<ClusterForkResult>;
+export type InitializeCluster = (cluster: { clusterId: string, clusterSecret: string }) => Promise<void>;
+
+export interface ClusterWorkerProperties {
+    labels: string[];
+}
+
+export interface ClusterWorker extends ClusterWorkerProperties {
+    peer: RpcPeer;
+}
+
+export function getScryptedClusterMode(): ['server' | 'client', string, number] {
+    const mode = process.env.SCRYPTED_CLUSTER_MODE as 'server' | 'client';
+    if (!mode)
+        return;
+
+    if (!['server', 'client'].includes(mode))
+        throw new Error('SCRYPTED_CLUSTER_MODE must be set to either "server" or "client".');
+
+    const [server, sport] = process.env.SCRYPTED_CLUSTER_SERVER?.split(':') || [];
+    const port = parseInt(sport) || 10556;
+    if (!net.isIP(server)) {
+        if (server)
+            throw new Error('SCRYPTED_CLUSTER_SERVER is set but is not a valid IP address.');
+        if (process.env.SCRYPTED_CLUSTER_SECRET)
+            throw new Error('SCRYPTED_CLUSTER_SECRET is set but SCRYPTED_CLUSTER_SERVER is not set.');
+        return;
+    }
+    if (!process.env.SCRYPTED_CLUSTER_SECRET)
+        throw new Error('SCRYPTED_CLUSTER_SERVER is set but SCRYPTED_CLUSTER_SECRET is not set.');
+
+    const address = process.env.SCRYPTED_CLUSTER_ADDRESS;
+    if (mode === 'server') {
+        if (address && address !== server)
+            throw new Error('SCRYPTED_CLUSTER_ADDRESS does not match server address. This setting should be removed.');
+        process.env.SCRYPTED_CLUSTER_ADDRESS = address || server;
+    }
+    else if (!net.isIP(address)) {
+        throw new Error('SCRYPTED_CLUSTER_ADDRESS is not set to a valid IP address.');
+    }
+
+    return [mode, server, port];
+}
+
+function peerLifecycle(serializer: ReturnType<typeof createRpcDuplexSerializer>, peer: RpcPeer, socket: tls.TLSSocket, type: 'server' | 'client') {
+    serializer.setupRpcPeer(peer);
+
+    socket.on('data', data => serializer.onData(data));
+
+    socket.on('error', e => {
+        peer.kill(e.message);
+    });
+    socket.on('close', () => {
+        peer.kill(`cluster ${type} closed`);
+    });
+    peer.killed.then(() => {
+        socket.destroy();
+    });
+}
+
+function preparePeer(socket: tls.TLSSocket, type: 'server' | 'client') {
+    const serializer = createRpcDuplexSerializer(socket);
+    const peer = new RpcPeer('cluster-remote', 'cluster-host', (message, reject, serializationContext) => {
+        serializer.sendMessage(message, reject, serializationContext);
+    });
+
+    peerLifecycle(serializer, peer, socket, type);
+
+    return peer;
+}
+
+export interface ClusterForkOptions {
+    runtime?: ForkOptions['runtime'];
+    labels?: ForkOptions['labels'];
+}
+
+export function matchesClusterLabels(options: ClusterForkOptions, labels: string[]) {
+    let matched = 0;
+    for (const label of options?.labels?.require || []) {
+        if (!labels.includes(label))
+            return 0;
+    }
+
+    // if there is nothing in the any list, consider it matched
+    let foundAny = !options?.labels?.any?.length;
+    for (const label of options.labels?.any || []) {
+        if (!labels.includes(label)) {
+            matched++;
+            foundAny = true;
+        }
+    }
+    if (!foundAny)
+        return 0;
+
+    for (const label of options?.labels?.prefer || []) {
+        if (labels.includes(label))
+            matched++;
+    }
+    // ensure non zero result.
+    matched++;
+    return matched;
+}
+
+export function getClusterLabels() {
+    let labels = process.env.SCRYPTED_CLUSTER_LABELS?.split(',') || [];
+    labels.push(process.arch, process.platform, os.hostname());
+    labels = [...new Set(labels)];
+    return labels;
+}
+
+type ConnectForkWorker = (auth: ClusterObject, properties: ClusterWorkerProperties) => Promise<{ clusterId: string }>;
+
+export function startClusterClient(mainFilename: string) {
+    const labels = getClusterLabels();
+
+    const clusterSecret = process.env.SCRYPTED_CLUSTER_SECRET;
+    const clusterMode = getScryptedClusterMode();
+    const [, host, port] = clusterMode;
+    (async () => {
+        while (true) {
+            const backoff = sleep(10000);
+            const socket = tls.connect({
+                host,
+                port,
+                rejectUnauthorized: false,
+            });
+
+            const peer = preparePeer(socket, 'client');
+
+            try {
+                const connectForkWorker: ConnectForkWorker = await peer.getParam('connectForkWorker');
+                const auth: ClusterObject = {
+                    address: socket.localAddress,
+                    port: socket.localPort,
+                    id: undefined,
+                    proxyId: undefined,
+                    sourceKey: undefined,
+                    sha256: undefined,
+                };
+                auth.sha256 = computeClusterObjectHash(auth, clusterSecret);
+
+                const properties: ClusterWorkerProperties = {
+                    labels,
+                };
+
+                const { clusterId } = await connectForkWorker(auth, properties);
+                const clusterPeerSetup = prepareClusterPeer(peer);
+                await clusterPeerSetup.initializeCluster({ clusterId, clusterSecret });
+
+                const clusterForkParam: ClusterForkParam = async (
+                    peerLiveness: PeerLiveness,
+                    runtime: string,
+                    packageJson: any,
+                    zipHash: string, 
+                    getZip: () => Promise<Buffer>) => {
+                    let runtimeWorker: RuntimeWorker;
+
+                    const builtins = getBuiltinRuntimeHosts();
+                    const rt = builtins.get(runtime);
+                    if (!rt)
+                        throw new Error('unknown runtime ' + runtime);
+
+                    const pluginId: string = packageJson.name;
+                    const { zipFile, unzippedPath } = await prepareZip(getPluginVolume(pluginId), zipHash, getZip);
+
+                    const volume = getScryptedVolume();
+                    const pluginVolume = getPluginVolume(pluginId);
+
+                    runtimeWorker = rt(mainFilename, pluginId, {
+                        packageJson,
+                        env: {
+                            SCRYPTED_VOLUME: volume,
+                            SCRYPTED_PLUGIN_VOLUME: pluginVolume,
+                        },
+                        pluginDebug: undefined,
+                        zipFile,
+                        unzippedPath,
+                        zipHash,
+                    }, undefined);
+
+                    const threadPeer = new RpcPeer('main', 'thread', (message, reject, serializationContext) => runtimeWorker.send(message, reject, serializationContext));
+                    runtimeWorker.setupRpcPeer(threadPeer);
+                    runtimeWorker.on('exit', () => {
+                        threadPeer.kill('worker exited');
+                    });
+                    runtimeWorker.on('error', e => {
+                        threadPeer.kill('worker error ' + e);
+                    });
+                    threadPeer.killed.catch(() => { }).finally(() => {
+                        runtimeWorker.kill();
+                    });
+                    peerLiveness.waitKilled().catch(() => { }).finally(() => {
+                        threadPeer.kill('peer killed');
+                    });
+                    let getRemote: any;
+                    try {
+                        const initializeCluster: InitializeCluster = await threadPeer.getParam('initializeCluster');
+                        await initializeCluster({ clusterId, clusterSecret });
+                        getRemote = await threadPeer.getParam('getRemote');
+                    }
+                    catch (e) {
+                        threadPeer.kill('cluster fork failed');
+                        throw e;
+                    }
+
+                    const readStream = async function* (stream: Readable) {
+                        for await (const buffer of stream) {
+                            yield buffer;
+                        }
+                    }
+
+                    const timeout = setTimeout(() => {
+                        threadPeer.kill('cluster fork timeout');
+                    }, 10000);
+                    const clusterGetRemote = (...args: any[]) => {
+                        clearTimeout(timeout);
+                        return {
+                            [RpcPeer.PROPERTY_JSON_COPY_SERIALIZE_CHILDREN]: true,
+                            stdout: readStream(runtimeWorker.stdout),
+                            stderr: readStream(runtimeWorker.stderr),
+                            getRemote,
+                        };
+                    };
+
+                    const result = new ClusterForkResult(threadPeer, threadPeer.killed, clusterGetRemote);
+                    return result;
+                };
+
+                peer.params['fork'] = clusterForkParam;
+
+                await peer.killed;
+            }
+            catch (e) {
+                peer.kill(e.message);
+                socket.destroy();
+            }
+            await backoff;
+        }
+    })();
+}
+
+export function createClusterServer(runtime: ScryptedRuntime, certificate: ReturnType<typeof createSelfSignedCertificate>) {
+    const server = tls.createServer({
+        key: certificate.serviceKey,
+        cert: certificate.certificate,
+    }, (socket) => {
+        const peer = preparePeer(socket, 'server');
+
+        const connectForkWorker: ConnectForkWorker = async (auth: ClusterObject, properties: ClusterWorkerProperties) => {
+            try {
+                const sha256 = computeClusterObjectHash(auth, runtime.clusterSecret);
+                if (sha256 !== auth.sha256)
+                    throw new Error('cluster object hash mismatch');
+                // the remote address may be ipv6 prefixed so use a fuzzy match.
+                // eg ::ffff:192.168.2.124
+                if (!process.env.SCRYPTED_DISABLE_CLUSTER_SERVER_TRUST) {
+                    if (auth.port !== socket.remotePort || !socket.remoteAddress.endsWith(auth.address))
+                        throw new Error('cluster object address mismatch');
+                }
+                const worker: ClusterWorker = {
+                    ...properties,
+                    peer,
+                };
+                runtime.clusterWorkers.add(worker);
+                peer.killed.then(() => {
+                    runtime.clusterWorkers.delete(worker);
+                });
+                socket.on('close', () => {
+                    runtime.clusterWorkers.delete(worker);
+                });
+            }
+            catch (e) {
+                peer.kill(e);
+                socket.destroy();
+            }
+
+            return {
+                clusterId: runtime.clusterId,
+            }
+        }
+        peer.params['connectForkWorker'] = connectForkWorker;
+    });
+
+    return server;
+}

package/src/scrypted-main-exports.ts

@@ -8,10 +8,10 @@ import v8 from 'v8';
 import vm from 'vm';
 import { PluginError } from './plugin/plugin-error';
 import { getScryptedVolume } from './plugin/plugin-volume';
+import { isNodePluginWorkerProcess } from './plugin/runtime/node-fork-worker';
 import { RPCResultError, startPeriodicGarbageCollection } from './rpc';
 import type { Runtime } from './scrypted-server-main';
-import { isNodePluginWorkerProcess } from './plugin/runtime/node-fork-worker';
-
+import { getScryptedClusterMode } from './scrypted-cluster';
 
 function start(mainFilename: string, options?: {
     onRuntimeCreated?: (runtime: Runtime) => Promise<void>,
@@ -28,8 +28,8 @@ function start(mainFilename: string, options?: {
         globalThis.gc = vm.runInNewContext("gc");
     }
 
-    if (!semver.gte(process.version, '16.0.0')) {
-        throw new Error('"node" version out of date. Please update node to v16 or higher.')
+    if (!semver.gte(process.version, '18.0.0')) {
+        throw new Error('"node" version out of date. Please update node to v18 or higher.')
     }
 
     // Node 17 changes the dns resolution order to return the record order.
@@ -53,20 +53,26 @@ function start(mainFilename: string, options?: {
         const start = require('./scrypted-plugin-main').default;
         return start(mainFilename);
     }
-    else {
-        // unhandled rejections are allowed if they are from a rpc/plugin call.
-        process.on('unhandledRejection', error => {
-            if (error?.constructor !== RPCResultError && error?.constructor !== PluginError) {
-                console.error('fatal error', error);
-                throw error;
-            }
-            console.warn('unhandled rejection of RPC Result', error);
-        });
 
-        dotenv.config({
-            path: path.join(getScryptedVolume(), '.env'),
-        });
+    // unhandled rejections are allowed if they are from a rpc/plugin call.
+    process.on('unhandledRejection', error => {
+        if (error?.constructor !== RPCResultError && error?.constructor !== PluginError) {
+            console.error('fatal error', error);
+            throw error;
+        }
+        console.warn('unhandled rejection of RPC Result', error);
+    });
+
+    dotenv.config({
+        path: path.join(getScryptedVolume(), '.env'),
+    });
 
+    const clusterMode = getScryptedClusterMode();
+    if (clusterMode?.[0] === 'client') {
+        const start = require('./scrypted-cluster-main').default;
+        return start(mainFilename);
+    }
+    else {
         const start = require('./scrypted-server-main').default;
         return start(mainFilename, options);
     }

package/src/scrypted-server-main.ts

@@ -11,38 +11,24 @@ import net from 'net';
 import os from 'os';
 import path from 'path';
 import process from 'process';
-import semver from 'semver';
 import { install as installSourceMapSupport } from 'source-map-support';
 import { createSelfSignedCertificate, CURRENT_SELF_SIGNED_CERTIFICATE_VERSION } from './cert';
 import { Plugin, ScryptedUser, Settings } from './db-types';
 import { getUsableNetworkAddresses } from './ip';
 import Level from './level';
-import { PluginError } from './plugin/plugin-error';
 import { getScryptedVolume } from './plugin/plugin-volume';
-import { RPCResultError } from './rpc';
 import { ScryptedRuntime } from './runtime';
 import { SCRYPTED_DEBUG_PORT, SCRYPTED_INSECURE_PORT, SCRYPTED_SECURE_PORT } from './server-settings';
 import { getNpmPackageInfo } from './services/plugin';
 import { setScryptedUserPassword, UsersService } from './services/users';
 import { sleep } from './sleep';
 import { ONE_DAY_MILLISECONDS, UserToken } from './usertoken';
+import { createClusterServer, getScryptedClusterMode } from './scrypted-cluster';
 
 export type Runtime = ScryptedRuntime;
 
-if (!semver.gte(process.version, '18.0.0')) {
-    throw new Error('"node" version out of date. Please update node to v18 or higher.')
-}
-
-process.on('unhandledRejection', error => {
-    if (error?.constructor !== RPCResultError && error?.constructor !== PluginError) {
-        console.error('pending crash', error);
-        throw error;
-    }
-    console.warn('unhandled rejection of RPC Result', error);
-});
-
-async function listenServerPort(env: string, port: number, server: any) {
-    server.listen(port);
+async function listenServerPort(env: string, port: number, server: http.Server | https.Server | net.Server, hostname?: string) {
+    server.listen(port, hostname);
     try {
         await once(server, 'listening');
     }
@@ -474,7 +460,7 @@ async function start(mainFilename: string, options?: {
             debugServer.on('connection', resolve);
         });
 
-        waitDebug.catch(() => {});
+        waitDebug.catch(() => { });
 
         workerInspectPort = Math.round(Math.random() * 10000) + 30000;
         try {
@@ -737,6 +723,12 @@ async function start(mainFilename: string, options?: {
     await listenServerPort('SCRYPTED_SECURE_PORT', SCRYPTED_SECURE_PORT, secure);
     await listenServerPort('SCRYPTED_INSECURE_PORT', SCRYPTED_INSECURE_PORT, insecure);
 
+    const clusterMode = getScryptedClusterMode();
+    if (clusterMode?.[0] === 'server') {
+        const clusterServer = createClusterServer(scrypted, keyPair);
+        await listenServerPort('SCRYPTED_CLUSTER_SERVER', clusterMode[2], clusterServer);
+    }
+
     console.log('#######################################################');
     console.log(`Scrypted Volume           : ${volumeDir}`);
     console.log(`Scrypted Server (Local)   : https://localhost:${SCRYPTED_SECURE_PORT}/`);
@@ -746,13 +738,13 @@ async function start(mainFilename: string, options?: {
     console.log(`Version:       : ${await scrypted.info.getVersion()}`);
     console.log('#######################################################');
     console.log('Scrypted insecure http service port:', SCRYPTED_INSECURE_PORT);
-    console.log('Ports can be changed with environment variables.')
-    console.log('https: $SCRYPTED_SECURE_PORT')
-    console.log('http : $SCRYPTED_INSECURE_PORT')
-    console.log('Certificate can be modified via tls.createSecureContext options in')
+    console.log('Ports can be changed with environment variables.');
+    console.log('https: $SCRYPTED_SECURE_PORT');
+    console.log('http : $SCRYPTED_INSECURE_PORT');
+    console.log('Certificate can be modified via tls.createSecureContext options in');
     console.log('JSON file located at SCRYPTED_HTTPS_OPTIONS_FILE environment variable:');
     console.log('export SCRYPTED_HTTPS_OPTIONS_FILE=/path/to/options.json');
-    console.log('https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions')
+    console.log('https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions');
     console.log('#######################################################');
 
     return scrypted;

package/src/server-settings.ts

@@ -3,6 +3,7 @@ import { getUsableNetworkAddresses } from './ip';
 export const SCRYPTED_INSECURE_PORT = parseInt(process.env.SCRYPTED_INSECURE_PORT) || 11080;
 export const SCRYPTED_SECURE_PORT = parseInt(process.env.SCRYPTED_SECURE_PORT) || 10443;
 export const SCRYPTED_DEBUG_PORT = parseInt(process.env.SCRYPTED_DEBUG_PORT) || 10081;
+export const SCRYPTED_CLUSTER_WORKERS = parseInt(process.env.SCRYPTED_CLUSTER_WORKERS) || 32;
 
 export function getIpAddress(): string {
     return getUsableNetworkAddresses()[0];

package/src/services/backup.ts

@@ -5,7 +5,6 @@ import Level from '../level';
 import { getPluginsVolume, getScryptedVolume } from '../plugin/plugin-volume';
 import { ScryptedRuntime } from '../runtime';
 import { sleep } from '../sleep';
-import util from 'util';
 
 export class Backup {
     constructor(public runtime: ScryptedRuntime) {}

package/src/services/cluster-fork.ts

@@ -0,0 +1,22 @@
+import type { ScryptedRuntime } from "../runtime";
+import { ClusterForkOptions, ClusterForkParam, matchesClusterLabels, PeerLiveness } from "../scrypted-cluster";
+
+export class ClusterFork {
+    constructor(public runtime: ScryptedRuntime) { }
+
+    async fork(peerLiveness: PeerLiveness, options: ClusterForkOptions, packageJson: any, zipHash: string, getZip: () => Promise<Buffer>) {
+        const matchingWorkers = [...this.runtime.clusterWorkers].map(worker => ({
+            worker,
+            matches: matchesClusterLabels(options, worker.labels),
+        }))
+        .filter(({ matches }) => matches);
+        matchingWorkers.sort((a, b) => b.worker.labels.length - a.worker.labels.length);
+        const worker = matchingWorkers[0]?.worker;
+
+        if (!worker)
+            throw new Error(`no worker found for cluster labels ${JSON.stringify(options.labels)}`);
+
+        const fork: ClusterForkParam = await worker.peer.getParam('fork');
+        return fork(peerLiveness, options.runtime, packageJson, zipHash, getZip);
+    }
+}

package/src/services/plugin.ts

@@ -174,34 +174,16 @@ export class PluginComponent {
         consoleServer.clear(pluginDevice.nativeId);
     }
 
-    async getRemoteServicePort(pluginId: string, name: string, ...args: any[]): Promise<number> {
+    async getRemoteServicePort(pluginId: string, name: string, ...args: any[]): Promise<[number, string]> {
         if (name === 'console') {
             const consoleServer = await this.scrypted.plugins[pluginId].consoleServer;
-            return consoleServer.readPort;
+            return [consoleServer.readPort, process.env.SCRYPTED_CLUSTER_ADDRESS];
         }
         if (name === 'console-writer') {
             const consoleServer = await this.scrypted.plugins[pluginId].consoleServer;
-            return consoleServer.writePort;
+            return [consoleServer.writePort, process.env.SCRYPTED_CLUSTER_ADDRESS];
         }
 
         return this.scrypted.plugins[pluginId].remote.getServicePort(name, ...args);
     }
-
-    async setHostParam(pluginId: string, name: string, param?: any) {
-        const host = this.scrypted.plugins[pluginId];
-        if (!host)
-            return;
-
-        const key = `oob-param-${name}`;
-        if (param === undefined)
-            delete host.peer.params[key];
-        else
-            host.peer.params[key] = param;
-    }
-
-    async getHostParam(pluginId: string, name: string) {
-        const host = this.scrypted.plugins[pluginId];
-        const key = `oob-param-${name}`;
-        return host?.peer?.params?.[key];
-    }
 }